kprobes.c 34.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 *  Kernel Probes (KProbes)
 *  kernel/kprobes.c
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) IBM Corporation, 2002, 2004
 *
 * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
 *		Probes initial implementation (includes suggestions from
 *		Rusty Russell).
 * 2004-Aug	Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
 *		hlists and exceptions notifier as suggested by Andi Kleen.
 * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
 *		interface to access function arguments.
 * 2004-Sep	Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
 *		exceptions notifier to be first on the priority list.
30 31 32
 * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
 *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi
 *		<prasanna@in.ibm.com> added function-return probes.
L
Linus Torvalds 已提交
33 34 35 36
 */
#include <linux/kprobes.h>
#include <linux/hash.h>
#include <linux/init.h>
T
Tim Schmielau 已提交
37
#include <linux/slab.h>
R
Randy Dunlap 已提交
38
#include <linux/stddef.h>
L
Linus Torvalds 已提交
39
#include <linux/module.h>
40
#include <linux/moduleloader.h>
41
#include <linux/kallsyms.h>
42
#include <linux/freezer.h>
43 44
#include <linux/seq_file.h>
#include <linux/debugfs.h>
45
#include <linux/kdebug.h>
46
#include <linux/memory.h>
47

48
#include <asm-generic/sections.h>
L
Linus Torvalds 已提交
49 50
#include <asm/cacheflush.h>
#include <asm/errno.h>
51
#include <asm/uaccess.h>
L
Linus Torvalds 已提交
52 53 54 55

#define KPROBE_HASH_BITS 6
#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)

56 57 58 59 60 61 62 63 64 65

/*
 * Some oddball architectures like 64bit powerpc have function descriptors
 * so this must be overridable.
 */
#ifndef kprobe_lookup_name
#define kprobe_lookup_name(name, addr) \
	addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
#endif

66
static int kprobes_initialized;
L
Linus Torvalds 已提交
67
static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
68
static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
L
Linus Torvalds 已提交
69

70 71 72
/* NOTE: change this value only with kprobe_mutex held */
static bool kprobe_enabled;

73
static DEFINE_MUTEX(kprobe_mutex);	/* Protects kprobe_table */
74
static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
75
static struct {
76
	spinlock_t lock ____cacheline_aligned_in_smp;
77 78 79 80 81 82
} kretprobe_table_locks[KPROBE_TABLE_SIZE];

static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
{
	return &(kretprobe_table_locks[hash].lock);
}
L
Linus Torvalds 已提交
83

84 85 86 87 88 89 90
/*
 * Normally, functions that we'd want to prohibit kprobes in, are marked
 * __kprobes. But, there are cases where such functions already belong to
 * a different section (__sched for preempt_schedule)
 *
 * For such cases, we now have a blacklist
 */
91
static struct kprobe_blackpoint kprobe_blacklist[] = {
92 93 94 95
	{"preempt_schedule",},
	{NULL}    /* Terminator */
};

96
#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
97 98 99 100 101 102 103 104 105 106 107 108 109
/*
 * kprobe->ainsn.insn points to the copy of the instruction to be
 * single-stepped. x86_64, POWER4 and above have no-exec support and
 * stepping on the instruction on a vmalloced/kmalloced/data page
 * is a recipe for disaster
 */
#define INSNS_PER_PAGE	(PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))

struct kprobe_insn_page {
	struct hlist_node hlist;
	kprobe_opcode_t *insns;		/* Page of instruction slots */
	char slot_used[INSNS_PER_PAGE];
	int nused;
110
	int ngarbage;
111 112
};

113 114 115 116 117 118
enum kprobe_slot_state {
	SLOT_CLEAN = 0,
	SLOT_DIRTY = 1,
	SLOT_USED = 2,
};

119
static DEFINE_MUTEX(kprobe_insn_mutex);	/* Protects kprobe_insn_pages */
120
static struct hlist_head kprobe_insn_pages;
121 122 123 124 125 126
static int kprobe_garbage_slots;
static int collect_garbage_slots(void);

static int __kprobes check_safety(void)
{
	int ret = 0;
127
#if defined(CONFIG_PREEMPT) && defined(CONFIG_FREEZER)
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
	ret = freeze_processes();
	if (ret == 0) {
		struct task_struct *p, *q;
		do_each_thread(p, q) {
			if (p != current && p->state == TASK_RUNNING &&
			    p->pid != 0) {
				printk("Check failed: %s is running\n",p->comm);
				ret = -1;
				goto loop_end;
			}
		} while_each_thread(p, q);
	}
loop_end:
	thaw_processes();
#else
	synchronize_sched();
#endif
	return ret;
}
147 148

/**
149
 * __get_insn_slot() - Find a slot on an executable page for an instruction.
150 151
 * We allocate an executable page if there's no room on existing ones.
 */
152
static kprobe_opcode_t __kprobes *__get_insn_slot(void)
153 154 155 156
{
	struct kprobe_insn_page *kip;
	struct hlist_node *pos;

157
 retry:
158
	hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
159 160 161
		if (kip->nused < INSNS_PER_PAGE) {
			int i;
			for (i = 0; i < INSNS_PER_PAGE; i++) {
162 163
				if (kip->slot_used[i] == SLOT_CLEAN) {
					kip->slot_used[i] = SLOT_USED;
164 165 166 167 168 169 170 171 172
					kip->nused++;
					return kip->insns + (i * MAX_INSN_SIZE);
				}
			}
			/* Surprise!  No unused slots.  Fix kip->nused. */
			kip->nused = INSNS_PER_PAGE;
		}
	}

173 174 175 176 177
	/* If there are any garbage slots, collect it and try again. */
	if (kprobe_garbage_slots && collect_garbage_slots() == 0) {
		goto retry;
	}
	/* All out of space.  Need to allocate a new page. Use slot 0. */
178
	kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
179
	if (!kip)
180 181 182 183 184 185 186 187 188 189 190 191 192 193
		return NULL;

	/*
	 * Use module_alloc so this page is within +/- 2GB of where the
	 * kernel image and loaded module images reside. This is required
	 * so x86_64 can correctly handle the %rip-relative fixups.
	 */
	kip->insns = module_alloc(PAGE_SIZE);
	if (!kip->insns) {
		kfree(kip);
		return NULL;
	}
	INIT_HLIST_NODE(&kip->hlist);
	hlist_add_head(&kip->hlist, &kprobe_insn_pages);
194 195
	memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE);
	kip->slot_used[0] = SLOT_USED;
196
	kip->nused = 1;
197
	kip->ngarbage = 0;
198 199 200
	return kip->insns;
}

201 202 203 204 205 206 207 208 209
kprobe_opcode_t __kprobes *get_insn_slot(void)
{
	kprobe_opcode_t *ret;
	mutex_lock(&kprobe_insn_mutex);
	ret = __get_insn_slot();
	mutex_unlock(&kprobe_insn_mutex);
	return ret;
}

210 211 212
/* Return 1 if all garbages are collected, otherwise 0. */
static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
{
213
	kip->slot_used[idx] = SLOT_CLEAN;
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
	kip->nused--;
	if (kip->nused == 0) {
		/*
		 * Page is no longer in use.  Free it unless
		 * it's the last one.  We keep the last one
		 * so as not to have to set it up again the
		 * next time somebody inserts a probe.
		 */
		hlist_del(&kip->hlist);
		if (hlist_empty(&kprobe_insn_pages)) {
			INIT_HLIST_NODE(&kip->hlist);
			hlist_add_head(&kip->hlist,
				       &kprobe_insn_pages);
		} else {
			module_free(NULL, kip->insns);
			kfree(kip);
		}
		return 1;
	}
	return 0;
}

static int __kprobes collect_garbage_slots(void)
{
	struct kprobe_insn_page *kip;
	struct hlist_node *pos, *next;
240
	int safety;
241 242

	/* Ensure no-one is preepmted on the garbages */
243 244 245 246
	mutex_unlock(&kprobe_insn_mutex);
	safety = check_safety();
	mutex_lock(&kprobe_insn_mutex);
	if (safety != 0)
247 248
		return -EAGAIN;

249
	hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) {
250 251 252 253 254
		int i;
		if (kip->ngarbage == 0)
			continue;
		kip->ngarbage = 0;	/* we will collect all garbages */
		for (i = 0; i < INSNS_PER_PAGE; i++) {
255
			if (kip->slot_used[i] == SLOT_DIRTY &&
256 257 258 259 260 261 262 263 264
			    collect_one_slot(kip, i))
				break;
		}
	}
	kprobe_garbage_slots = 0;
	return 0;
}

void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
265 266 267 268
{
	struct kprobe_insn_page *kip;
	struct hlist_node *pos;

269
	mutex_lock(&kprobe_insn_mutex);
270
	hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
271 272 273
		if (kip->insns <= slot &&
		    slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {
			int i = (slot - kip->insns) / MAX_INSN_SIZE;
274
			if (dirty) {
275
				kip->slot_used[i] = SLOT_DIRTY;
276 277 278
				kip->ngarbage++;
			} else {
				collect_one_slot(kip, i);
279
			}
280
			break;
281 282
		}
	}
283 284

	if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE)
285
		collect_garbage_slots();
286 287

	mutex_unlock(&kprobe_insn_mutex);
288
}
289
#endif
290

291 292 293 294 295 296 297 298 299 300 301
/* We have preemption disabled.. so it is safe to use __ versions */
static inline void set_kprobe_instance(struct kprobe *kp)
{
	__get_cpu_var(kprobe_instance) = kp;
}

static inline void reset_kprobe_instance(void)
{
	__get_cpu_var(kprobe_instance) = NULL;
}

302 303
/*
 * This routine is called either:
304
 * 	- under the kprobe_mutex - during kprobe_[un]register()
305
 * 				OR
306
 * 	- with preemption disabled - from arch/xxx/kernel/kprobes.c
307
 */
308
struct kprobe __kprobes *get_kprobe(void *addr)
L
Linus Torvalds 已提交
309 310 311
{
	struct hlist_head *head;
	struct hlist_node *node;
312
	struct kprobe *p;
L
Linus Torvalds 已提交
313 314

	head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
315
	hlist_for_each_entry_rcu(p, node, head, hlist) {
L
Linus Torvalds 已提交
316 317 318 319 320 321
		if (p->addr == addr)
			return p;
	}
	return NULL;
}

322 323 324 325
/*
 * Aggregate handlers for multiple kprobes support - these handlers
 * take care of invoking the individual kprobe handlers on p->list
 */
326
static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
327 328 329
{
	struct kprobe *kp;

330
	list_for_each_entry_rcu(kp, &p->list, list) {
331
		if (kp->pre_handler && !kprobe_gone(kp)) {
332
			set_kprobe_instance(kp);
333 334
			if (kp->pre_handler(kp, regs))
				return 1;
335
		}
336
		reset_kprobe_instance();
337 338 339 340
	}
	return 0;
}

341 342
static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
					unsigned long flags)
343 344 345
{
	struct kprobe *kp;

346
	list_for_each_entry_rcu(kp, &p->list, list) {
347
		if (kp->post_handler && !kprobe_gone(kp)) {
348
			set_kprobe_instance(kp);
349
			kp->post_handler(kp, regs, flags);
350
			reset_kprobe_instance();
351 352 353 354
		}
	}
}

355 356
static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
					int trapnr)
357
{
358 359
	struct kprobe *cur = __get_cpu_var(kprobe_instance);

360 361 362 363
	/*
	 * if we faulted "during" the execution of a user specified
	 * probe handler, invoke just that probe's fault handler
	 */
364 365
	if (cur && cur->fault_handler) {
		if (cur->fault_handler(cur, regs, trapnr))
366 367 368 369 370
			return 1;
	}
	return 0;
}

371
static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
372
{
373 374 375 376 377 378
	struct kprobe *cur = __get_cpu_var(kprobe_instance);
	int ret = 0;

	if (cur && cur->break_handler) {
		if (cur->break_handler(cur, regs))
			ret = 1;
379
	}
380 381
	reset_kprobe_instance();
	return ret;
382 383
}

384 385 386 387 388 389 390 391 392 393 394 395 396
/* Walks the list and increments nmissed count for multiprobe case */
void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
{
	struct kprobe *kp;
	if (p->pre_handler != aggr_pre_handler) {
		p->nmissed++;
	} else {
		list_for_each_entry_rcu(kp, &p->list, list)
			kp->nmissed++;
	}
	return;
}

397 398
void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
				struct hlist_head *head)
399
{
400 401
	struct kretprobe *rp = ri->rp;

402 403
	/* remove rp inst off the rprobe_inst_table */
	hlist_del(&ri->hlist);
404 405 406 407 408
	INIT_HLIST_NODE(&ri->hlist);
	if (likely(rp)) {
		spin_lock(&rp->lock);
		hlist_add_head(&ri->hlist, &rp->free_instances);
		spin_unlock(&rp->lock);
409 410
	} else
		/* Unregistering */
411
		hlist_add_head(&ri->hlist, head);
412 413
}

414
void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
415 416 417 418 419 420 421 422 423 424
			 struct hlist_head **head, unsigned long *flags)
{
	unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
	spinlock_t *hlist_lock;

	*head = &kretprobe_inst_table[hash];
	hlist_lock = kretprobe_table_lock_ptr(hash);
	spin_lock_irqsave(hlist_lock, *flags);
}

425 426
static void __kprobes kretprobe_table_lock(unsigned long hash,
	unsigned long *flags)
427
{
428 429 430 431
	spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
	spin_lock_irqsave(hlist_lock, *flags);
}

432 433
void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
	unsigned long *flags)
434 435 436 437 438 439 440 441
{
	unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
	spinlock_t *hlist_lock;

	hlist_lock = kretprobe_table_lock_ptr(hash);
	spin_unlock_irqrestore(hlist_lock, *flags);
}

442
void __kprobes kretprobe_table_unlock(unsigned long hash, unsigned long *flags)
443 444 445
{
	spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
	spin_unlock_irqrestore(hlist_lock, *flags);
446 447 448
}

/*
449 450 451 452
 * This function is called from finish_task_switch when task tk becomes dead,
 * so that we can recycle any function-return probe instances associated
 * with this task. These left over instances represent probed functions
 * that have been called but will never return.
453
 */
454
void __kprobes kprobe_flush_task(struct task_struct *tk)
455
{
B
bibo,mao 已提交
456
	struct kretprobe_instance *ri;
457
	struct hlist_head *head, empty_rp;
458
	struct hlist_node *node, *tmp;
459
	unsigned long hash, flags = 0;
460

461 462 463 464 465 466 467
	if (unlikely(!kprobes_initialized))
		/* Early boot.  kretprobe_table_locks not yet initialized. */
		return;

	hash = hash_ptr(tk, KPROBE_HASH_BITS);
	head = &kretprobe_inst_table[hash];
	kretprobe_table_lock(hash, &flags);
B
bibo,mao 已提交
468 469
	hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
		if (ri->task == tk)
470
			recycle_rp_inst(ri, &empty_rp);
B
bibo,mao 已提交
471
	}
472 473
	kretprobe_table_unlock(hash, &flags);
	INIT_HLIST_HEAD(&empty_rp);
474 475 476 477
	hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
		hlist_del(&ri->hlist);
		kfree(ri);
	}
478 479 480 481 482
}

static inline void free_rp_inst(struct kretprobe *rp)
{
	struct kretprobe_instance *ri;
483 484
	struct hlist_node *pos, *next;

485 486
	hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
		hlist_del(&ri->hlist);
487 488 489 490
		kfree(ri);
	}
}

491 492
static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
{
493
	unsigned long flags, hash;
494 495
	struct kretprobe_instance *ri;
	struct hlist_node *pos, *next;
496 497
	struct hlist_head *head;

498
	/* No race here */
499 500 501 502 503 504 505 506
	for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
		kretprobe_table_lock(hash, &flags);
		head = &kretprobe_inst_table[hash];
		hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
			if (ri->rp == rp)
				ri->rp = NULL;
		}
		kretprobe_table_unlock(hash, &flags);
507 508 509 510
	}
	free_rp_inst(rp);
}

511 512 513 514 515 516 517 518 519 520 521 522 523
/*
 * Keep all fields in the kprobe consistent
 */
static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
{
	memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
	memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
}

/*
* Add the new probe to old_p->list. Fail if this is the
* second jprobe at the address - two jprobes can't coexist
*/
524
static int __kprobes add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
525 526
{
	if (p->break_handler) {
527 528
		if (old_p->break_handler)
			return -EEXIST;
529
		list_add_tail_rcu(&p->list, &old_p->list);
530
		old_p->break_handler = aggr_break_handler;
531
	} else
532
		list_add_rcu(&p->list, &old_p->list);
533 534
	if (p->post_handler && !old_p->post_handler)
		old_p->post_handler = aggr_post_handler;
535 536 537
	return 0;
}

538 539 540 541 542 543
/*
 * Fill in the required fields of the "manager kprobe". Replace the
 * earlier kprobe in the hlist with the manager kprobe
 */
static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
{
544
	copy_kprobe(p, ap);
545
	flush_insn_slot(ap);
546 547 548
	ap->addr = p->addr;
	ap->pre_handler = aggr_pre_handler;
	ap->fault_handler = aggr_fault_handler;
549 550
	/* We don't care the kprobe which has gone. */
	if (p->post_handler && !kprobe_gone(p))
551
		ap->post_handler = aggr_post_handler;
552
	if (p->break_handler && !kprobe_gone(p))
553
		ap->break_handler = aggr_break_handler;
554 555

	INIT_LIST_HEAD(&ap->list);
556
	list_add_rcu(&p->list, &ap->list);
557

558
	hlist_replace_rcu(&p->hlist, &ap->hlist);
559 560 561 562 563 564
}

/*
 * This is the second or subsequent kprobe at the address - handle
 * the intricacies
 */
565 566
static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
					  struct kprobe *p)
567 568 569 570
{
	int ret = 0;
	struct kprobe *ap;

571 572 573 574 575 576 577 578 579 580 581
	if (kprobe_gone(old_p)) {
		/*
		 * Attempting to insert new probe at the same location that
		 * had a probe in the module vaddr area which already
		 * freed. So, the instruction slot has already been
		 * released. We need a new slot for the new probe.
		 */
		ret = arch_prepare_kprobe(old_p);
		if (ret)
			return ret;
	}
582 583 584
	if (old_p->pre_handler == aggr_pre_handler) {
		copy_kprobe(old_p, p);
		ret = add_new_kprobe(old_p, p);
585
		ap = old_p;
586
	} else {
587
		ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
588 589 590
		if (!ap) {
			if (kprobe_gone(old_p))
				arch_remove_kprobe(old_p);
591
			return -ENOMEM;
592
		}
593
		add_aggr_kprobe(ap, old_p);
594 595
		copy_kprobe(ap, p);
		ret = add_new_kprobe(ap, p);
596
	}
597 598 599 600 601 602 603 604 605
	if (kprobe_gone(old_p)) {
		/*
		 * If the old_p has gone, its breakpoint has been disarmed.
		 * We have to arm it again after preparing real kprobes.
		 */
		ap->flags &= ~KPROBE_FLAG_GONE;
		if (kprobe_enabled)
			arch_arm_kprobe(ap);
	}
606 607 608
	return ret;
}

609 610
static int __kprobes in_kprobes_functions(unsigned long addr)
{
611 612
	struct kprobe_blackpoint *kb;

613 614
	if (addr >= (unsigned long)__kprobes_text_start &&
	    addr < (unsigned long)__kprobes_text_end)
615
		return -EINVAL;
616 617 618 619 620 621 622 623 624 625 626
	/*
	 * If there exists a kprobe_blacklist, verify and
	 * fail any probe registration in the prohibited area
	 */
	for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
		if (kb->start_addr) {
			if (addr >= kb->start_addr &&
			    addr < (kb->start_addr + kb->range))
				return -EINVAL;
		}
	}
627 628 629
	return 0;
}

630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647
/*
 * If we have a symbol_name argument, look it up and add the offset field
 * to it. This way, we can specify a relative address to a symbol.
 */
static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
{
	kprobe_opcode_t *addr = p->addr;
	if (p->symbol_name) {
		if (addr)
			return NULL;
		kprobe_lookup_name(p->symbol_name, addr);
	}

	if (!addr)
		return NULL;
	return (kprobe_opcode_t *)(((char *)addr) + p->offset);
}

648
int __kprobes register_kprobe(struct kprobe *p)
L
Linus Torvalds 已提交
649 650
{
	int ret = 0;
651
	struct kprobe *old_p;
652
	struct module *probed_mod;
653
	kprobe_opcode_t *addr;
654

655 656
	addr = kprobe_addr(p);
	if (!addr)
657
		return -EINVAL;
658
	p->addr = addr;
659

660 661 662 663
	preempt_disable();
	if (!__kernel_text_address((unsigned long) p->addr) ||
	    in_kprobes_functions((unsigned long) p->addr)) {
		preempt_enable();
664
		return -EINVAL;
665
	}
666

667
	p->flags = 0;
668 669 670
	/*
	 * Check if are we probing a module.
	 */
671
	probed_mod = __module_text_address((unsigned long) p->addr);
672 673
	if (probed_mod) {
		/*
674 675
		 * We must hold a refcount of the probed module while updating
		 * its code to prohibit unexpected unloading.
676
		 */
677 678 679 680
		if (unlikely(!try_module_get(probed_mod))) {
			preempt_enable();
			return -EINVAL;
		}
681 682 683 684 685 686 687 688 689 690
		/*
		 * If the module freed .init.text, we couldn't insert
		 * kprobes in there.
		 */
		if (within_module_init((unsigned long)p->addr, probed_mod) &&
		    probed_mod->state != MODULE_STATE_COMING) {
			module_put(probed_mod);
			preempt_enable();
			return -EINVAL;
		}
691
	}
692
	preempt_enable();
L
Linus Torvalds 已提交
693

694
	p->nmissed = 0;
695
	INIT_LIST_HEAD(&p->list);
I
Ingo Molnar 已提交
696
	mutex_lock(&kprobe_mutex);
697 698 699
	old_p = get_kprobe(p->addr);
	if (old_p) {
		ret = register_aggr_kprobe(old_p, p);
L
Linus Torvalds 已提交
700 701 702
		goto out;
	}

703
	mutex_lock(&text_mutex);
704 705
	ret = arch_prepare_kprobe(p);
	if (ret)
706
		goto out_unlock_text;
707

708
	INIT_HLIST_NODE(&p->hlist);
709
	hlist_add_head_rcu(&p->hlist,
L
Linus Torvalds 已提交
710 711
		       &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);

712
	if (kprobe_enabled)
713
		arch_arm_kprobe(p);
714

715 716
out_unlock_text:
	mutex_unlock(&text_mutex);
L
Linus Torvalds 已提交
717
out:
I
Ingo Molnar 已提交
718
	mutex_unlock(&kprobe_mutex);
719

720
	if (probed_mod)
721
		module_put(probed_mod);
722

L
Linus Torvalds 已提交
723 724 725
	return ret;
}

726 727 728 729
/*
 * Unregister a kprobe without a scheduler synchronization.
 */
static int __kprobes __unregister_kprobe_top(struct kprobe *p)
L
Linus Torvalds 已提交
730
{
731
	struct kprobe *old_p, *list_p;
732 733

	old_p = get_kprobe(p->addr);
734 735 736
	if (unlikely(!old_p))
		return -EINVAL;

737 738 739 740 741
	if (p != old_p) {
		list_for_each_entry_rcu(list_p, &old_p->list, list)
			if (list_p == p)
			/* kprobe p is a valid probe */
				goto valid_p;
742
		return -EINVAL;
743 744
	}
valid_p:
745 746
	if (old_p == p ||
	    (old_p->pre_handler == aggr_pre_handler &&
747
	     list_is_singular(&old_p->list))) {
748 749
		/*
		 * Only probe on the hash list. Disarm only if kprobes are
750 751
		 * enabled and not gone - otherwise, the breakpoint would
		 * already have been removed. We save on flushing icache.
752
		 */
753 754
		if (kprobe_enabled && !kprobe_gone(old_p)) {
			mutex_lock(&text_mutex);
755
			arch_disarm_kprobe(p);
756 757
			mutex_unlock(&text_mutex);
		}
758 759
		hlist_del_rcu(&old_p->hlist);
	} else {
760
		if (p->break_handler && !kprobe_gone(p))
761
			old_p->break_handler = NULL;
762
		if (p->post_handler && !kprobe_gone(p)) {
763 764 765 766 767 768 769
			list_for_each_entry_rcu(list_p, &old_p->list, list) {
				if ((list_p != p) && (list_p->post_handler))
					goto noclean;
			}
			old_p->post_handler = NULL;
		}
noclean:
770 771
		list_del_rcu(&p->list);
	}
772 773
	return 0;
}
774

775 776 777
static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
{
	struct kprobe *old_p;
778

779
	if (list_empty(&p->list))
780
		arch_remove_kprobe(p);
781 782 783 784 785 786
	else if (list_is_singular(&p->list)) {
		/* "p" is the last child of an aggr_kprobe */
		old_p = list_entry(p->list.next, struct kprobe, list);
		list_del(&p->list);
		arch_remove_kprobe(old_p);
		kfree(old_p);
787 788 789
	}
}

790
int __kprobes register_kprobes(struct kprobe **kps, int num)
791 792 793 794 795 796
{
	int i, ret = 0;

	if (num <= 0)
		return -EINVAL;
	for (i = 0; i < num; i++) {
797
		ret = register_kprobe(kps[i]);
798 799 800
		if (ret < 0) {
			if (i > 0)
				unregister_kprobes(kps, i);
801
			break;
802
		}
803
	}
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827
	return ret;
}

void __kprobes unregister_kprobe(struct kprobe *p)
{
	unregister_kprobes(&p, 1);
}

void __kprobes unregister_kprobes(struct kprobe **kps, int num)
{
	int i;

	if (num <= 0)
		return;
	mutex_lock(&kprobe_mutex);
	for (i = 0; i < num; i++)
		if (__unregister_kprobe_top(kps[i]) < 0)
			kps[i]->addr = NULL;
	mutex_unlock(&kprobe_mutex);

	synchronize_sched();
	for (i = 0; i < num; i++)
		if (kps[i]->addr)
			__unregister_kprobe_bottom(kps[i]);
L
Linus Torvalds 已提交
828 829 830
}

static struct notifier_block kprobe_exceptions_nb = {
831 832 833 834
	.notifier_call = kprobe_exceptions_notify,
	.priority = 0x7fffffff /* we need to be notified first */
};

835 836 837 838
unsigned long __weak arch_deref_entry_point(void *entry)
{
	return (unsigned long)entry;
}
L
Linus Torvalds 已提交
839

840
int __kprobes register_jprobes(struct jprobe **jps, int num)
L
Linus Torvalds 已提交
841
{
842 843
	struct jprobe *jp;
	int ret = 0, i;
844

845
	if (num <= 0)
846
		return -EINVAL;
847 848 849 850 851 852 853 854 855 856 857
	for (i = 0; i < num; i++) {
		unsigned long addr;
		jp = jps[i];
		addr = arch_deref_entry_point(jp->entry);

		if (!kernel_text_address(addr))
			ret = -EINVAL;
		else {
			/* Todo: Verify probepoint is a function entry point */
			jp->kp.pre_handler = setjmp_pre_handler;
			jp->kp.break_handler = longjmp_break_handler;
858
			ret = register_kprobe(&jp->kp);
859
		}
860 861 862
		if (ret < 0) {
			if (i > 0)
				unregister_jprobes(jps, i);
863 864 865 866 867
			break;
		}
	}
	return ret;
}
868

869 870
int __kprobes register_jprobe(struct jprobe *jp)
{
871
	return register_jprobes(&jp, 1);
L
Linus Torvalds 已提交
872 873
}

874
void __kprobes unregister_jprobe(struct jprobe *jp)
L
Linus Torvalds 已提交
875
{
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
	unregister_jprobes(&jp, 1);
}

void __kprobes unregister_jprobes(struct jprobe **jps, int num)
{
	int i;

	if (num <= 0)
		return;
	mutex_lock(&kprobe_mutex);
	for (i = 0; i < num; i++)
		if (__unregister_kprobe_top(&jps[i]->kp) < 0)
			jps[i]->kp.addr = NULL;
	mutex_unlock(&kprobe_mutex);

	synchronize_sched();
	for (i = 0; i < num; i++) {
		if (jps[i]->kp.addr)
			__unregister_kprobe_bottom(&jps[i]->kp);
	}
L
Linus Torvalds 已提交
896 897
}

898
#ifdef CONFIG_KRETPROBES
899 900 901 902 903 904 905 906
/*
 * This kprobe pre_handler is registered with every kretprobe. When probe
 * hits it will set up the return probe.
 */
static int __kprobes pre_handler_kretprobe(struct kprobe *p,
					   struct pt_regs *regs)
{
	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
907 908
	unsigned long hash, flags = 0;
	struct kretprobe_instance *ri;
909 910

	/*TODO: consider to only swap the RA after the last pre_handler fired */
911 912
	hash = hash_ptr(current, KPROBE_HASH_BITS);
	spin_lock_irqsave(&rp->lock, flags);
913 914
	if (!hlist_empty(&rp->free_instances)) {
		ri = hlist_entry(rp->free_instances.first,
915 916 917 918
				struct kretprobe_instance, hlist);
		hlist_del(&ri->hlist);
		spin_unlock_irqrestore(&rp->lock, flags);

919 920
		ri->rp = rp;
		ri->task = current;
921

922
		if (rp->entry_handler && rp->entry_handler(ri, regs))
923 924
			return 0;

925 926 927
		arch_prepare_kretprobe(ri, regs);

		/* XXX(hch): why is there no hlist_move_head? */
928 929 930 931 932
		INIT_HLIST_NODE(&ri->hlist);
		kretprobe_table_lock(hash, &flags);
		hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
		kretprobe_table_unlock(hash, &flags);
	} else {
933
		rp->nmissed++;
934 935
		spin_unlock_irqrestore(&rp->lock, flags);
	}
936 937 938
	return 0;
}

939
int __kprobes register_kretprobe(struct kretprobe *rp)
940 941 942 943
{
	int ret = 0;
	struct kretprobe_instance *inst;
	int i;
944
	void *addr;
945 946

	if (kretprobe_blacklist_size) {
947 948 949
		addr = kprobe_addr(&rp->kp);
		if (!addr)
			return -EINVAL;
950 951 952 953 954 955

		for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
			if (kretprobe_blacklist[i].addr == addr)
				return -EINVAL;
		}
	}
956 957

	rp->kp.pre_handler = pre_handler_kretprobe;
958 959 960
	rp->kp.post_handler = NULL;
	rp->kp.fault_handler = NULL;
	rp->kp.break_handler = NULL;
961 962 963 964 965 966 967 968 969

	/* Pre-allocate memory for max kretprobe instances */
	if (rp->maxactive <= 0) {
#ifdef CONFIG_PREEMPT
		rp->maxactive = max(10, 2 * NR_CPUS);
#else
		rp->maxactive = NR_CPUS;
#endif
	}
970
	spin_lock_init(&rp->lock);
971 972
	INIT_HLIST_HEAD(&rp->free_instances);
	for (i = 0; i < rp->maxactive; i++) {
973 974
		inst = kmalloc(sizeof(struct kretprobe_instance) +
			       rp->data_size, GFP_KERNEL);
975 976 977 978
		if (inst == NULL) {
			free_rp_inst(rp);
			return -ENOMEM;
		}
979 980
		INIT_HLIST_NODE(&inst->hlist);
		hlist_add_head(&inst->hlist, &rp->free_instances);
981 982 983 984
	}

	rp->nmissed = 0;
	/* Establish function entry probe point */
985
	ret = register_kprobe(&rp->kp);
986
	if (ret != 0)
987 988 989 990
		free_rp_inst(rp);
	return ret;
}

991
int __kprobes register_kretprobes(struct kretprobe **rps, int num)
992 993 994 995 996 997
{
	int ret = 0, i;

	if (num <= 0)
		return -EINVAL;
	for (i = 0; i < num; i++) {
998
		ret = register_kretprobe(rps[i]);
999 1000 1001
		if (ret < 0) {
			if (i > 0)
				unregister_kretprobes(rps, i);
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
			break;
		}
	}
	return ret;
}

void __kprobes unregister_kretprobe(struct kretprobe *rp)
{
	unregister_kretprobes(&rp, 1);
}

void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
{
	int i;

	if (num <= 0)
		return;
	mutex_lock(&kprobe_mutex);
	for (i = 0; i < num; i++)
		if (__unregister_kprobe_top(&rps[i]->kp) < 0)
			rps[i]->kp.addr = NULL;
	mutex_unlock(&kprobe_mutex);

	synchronize_sched();
	for (i = 0; i < num; i++) {
		if (rps[i]->kp.addr) {
			__unregister_kprobe_bottom(&rps[i]->kp);
			cleanup_rp_inst(rps[i]);
		}
	}
}

1034
#else /* CONFIG_KRETPROBES */
1035
int __kprobes register_kretprobe(struct kretprobe *rp)
1036 1037 1038 1039
{
	return -ENOSYS;
}

1040
int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1041
{
1042
	return -ENOSYS;
1043
}
1044
void __kprobes unregister_kretprobe(struct kretprobe *rp)
1045
{
1046
}
1047

1048 1049 1050
void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
{
}
1051

1052 1053 1054 1055
static int __kprobes pre_handler_kretprobe(struct kprobe *p,
					   struct pt_regs *regs)
{
	return 0;
1056 1057
}

1058 1059
#endif /* CONFIG_KRETPROBES */

1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
/* Set the kprobe gone and remove its instruction buffer. */
static void __kprobes kill_kprobe(struct kprobe *p)
{
	struct kprobe *kp;
	p->flags |= KPROBE_FLAG_GONE;
	if (p->pre_handler == aggr_pre_handler) {
		/*
		 * If this is an aggr_kprobe, we have to list all the
		 * chained probes and mark them GONE.
		 */
		list_for_each_entry_rcu(kp, &p->list, list)
			kp->flags |= KPROBE_FLAG_GONE;
		p->post_handler = NULL;
		p->break_handler = NULL;
	}
	/*
	 * Here, we can remove insn_slot safely, because no thread calls
	 * the original probed function (which will be freed soon) any more.
	 */
	arch_remove_kprobe(p);
}

/* Module notifier call back, checking kprobes on the module */
static int __kprobes kprobes_module_callback(struct notifier_block *nb,
					     unsigned long val, void *data)
{
	struct module *mod = data;
	struct hlist_head *head;
	struct hlist_node *node;
	struct kprobe *p;
	unsigned int i;
1091
	int checkcore = (val == MODULE_STATE_GOING);
1092

1093
	if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1094 1095 1096
		return NOTIFY_DONE;

	/*
1097 1098 1099 1100
	 * When MODULE_STATE_GOING was notified, both of module .text and
	 * .init.text sections would be freed. When MODULE_STATE_LIVE was
	 * notified, only .init.text section would be freed. We need to
	 * disable kprobes which have been inserted in the sections.
1101 1102 1103 1104 1105
	 */
	mutex_lock(&kprobe_mutex);
	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
		head = &kprobe_table[i];
		hlist_for_each_entry_rcu(p, node, head, hlist)
1106 1107 1108
			if (within_module_init((unsigned long)p->addr, mod) ||
			    (checkcore &&
			     within_module_core((unsigned long)p->addr, mod))) {
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
				/*
				 * The vaddr this probe is installed will soon
				 * be vfreed buy not synced to disk. Hence,
				 * disarming the breakpoint isn't needed.
				 */
				kill_kprobe(p);
			}
	}
	mutex_unlock(&kprobe_mutex);
	return NOTIFY_DONE;
}

static struct notifier_block kprobe_module_nb = {
	.notifier_call = kprobes_module_callback,
	.priority = 0
};

L
Linus Torvalds 已提交
1126 1127 1128
static int __init init_kprobes(void)
{
	int i, err = 0;
1129 1130 1131 1132 1133
	unsigned long offset = 0, size = 0;
	char *modname, namebuf[128];
	const char *symbol_name;
	void *addr;
	struct kprobe_blackpoint *kb;
L
Linus Torvalds 已提交
1134 1135 1136

	/* FIXME allocate the probe table, currently defined statically */
	/* initialize all list heads */
1137
	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
L
Linus Torvalds 已提交
1138
		INIT_HLIST_HEAD(&kprobe_table[i]);
1139
		INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1140
		spin_lock_init(&(kretprobe_table_locks[i].lock));
1141
	}
L
Linus Torvalds 已提交
1142

1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
	/*
	 * Lookup and populate the kprobe_blacklist.
	 *
	 * Unlike the kretprobe blacklist, we'll need to determine
	 * the range of addresses that belong to the said functions,
	 * since a kprobe need not necessarily be at the beginning
	 * of a function.
	 */
	for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
		kprobe_lookup_name(kb->name, addr);
		if (!addr)
			continue;

		kb->start_addr = (unsigned long)addr;
		symbol_name = kallsyms_lookup(kb->start_addr,
				&size, &offset, &modname, namebuf);
		if (!symbol_name)
			kb->range = 0;
		else
			kb->range = size;
	}

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
	if (kretprobe_blacklist_size) {
		/* lookup the function address from its name */
		for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
			kprobe_lookup_name(kretprobe_blacklist[i].name,
					   kretprobe_blacklist[i].addr);
			if (!kretprobe_blacklist[i].addr)
				printk("kretprobe: lookup failed: %s\n",
				       kretprobe_blacklist[i].name);
		}
	}

1176 1177 1178
	/* By default, kprobes are enabled */
	kprobe_enabled = true;

1179
	err = arch_init_kprobes();
1180 1181
	if (!err)
		err = register_die_notifier(&kprobe_exceptions_nb);
1182 1183 1184
	if (!err)
		err = register_module_notifier(&kprobe_module_nb);

1185
	kprobes_initialized = (err == 0);
1186

1187 1188
	if (!err)
		init_test_probes();
L
Linus Torvalds 已提交
1189 1190 1191
	return err;
}

1192 1193
#ifdef CONFIG_DEBUG_FS
static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
1194
		const char *sym, int offset,char *modname)
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204
{
	char *kprobe_type;

	if (p->pre_handler == pre_handler_kretprobe)
		kprobe_type = "r";
	else if (p->pre_handler == setjmp_pre_handler)
		kprobe_type = "j";
	else
		kprobe_type = "k";
	if (sym)
1205 1206 1207
		seq_printf(pi, "%p  %s  %s+0x%x  %s %s\n", p->addr, kprobe_type,
			sym, offset, (modname ? modname : " "),
			(kprobe_gone(p) ? "[GONE]" : ""));
1208
	else
1209 1210
		seq_printf(pi, "%p  %s  %p %s\n", p->addr, kprobe_type, p->addr,
			(kprobe_gone(p) ? "[GONE]" : ""));
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
}

static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
{
	return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
}

static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
{
	(*pos)++;
	if (*pos >= KPROBE_TABLE_SIZE)
		return NULL;
	return pos;
}

static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
{
	/* Nothing to do */
}

static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
{
	struct hlist_head *head;
	struct hlist_node *node;
	struct kprobe *p, *kp;
	const char *sym = NULL;
	unsigned int i = *(loff_t *) v;
A
Alexey Dobriyan 已提交
1238
	unsigned long offset = 0;
1239 1240 1241 1242 1243
	char *modname, namebuf[128];

	head = &kprobe_table[i];
	preempt_disable();
	hlist_for_each_entry_rcu(p, node, head, hlist) {
A
Alexey Dobriyan 已提交
1244
		sym = kallsyms_lookup((unsigned long)p->addr, NULL,
1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
					&offset, &modname, namebuf);
		if (p->pre_handler == aggr_pre_handler) {
			list_for_each_entry_rcu(kp, &p->list, list)
				report_probe(pi, kp, sym, offset, modname);
		} else
			report_probe(pi, p, sym, offset, modname);
	}
	preempt_enable();
	return 0;
}

static struct seq_operations kprobes_seq_ops = {
	.start = kprobe_seq_start,
	.next  = kprobe_seq_next,
	.stop  = kprobe_seq_stop,
	.show  = show_kprobe_addr
};

static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
{
	return seq_open(filp, &kprobes_seq_ops);
}

static struct file_operations debugfs_kprobes_operations = {
	.open           = kprobes_open,
	.read           = seq_read,
	.llseek         = seq_lseek,
	.release        = seq_release,
};

1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
static void __kprobes enable_all_kprobes(void)
{
	struct hlist_head *head;
	struct hlist_node *node;
	struct kprobe *p;
	unsigned int i;

	mutex_lock(&kprobe_mutex);

	/* If kprobes are already enabled, just return */
	if (kprobe_enabled)
		goto already_enabled;

1288
	mutex_lock(&text_mutex);
1289 1290 1291
	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
		head = &kprobe_table[i];
		hlist_for_each_entry_rcu(p, node, head, hlist)
1292 1293
			if (!kprobe_gone(p))
				arch_arm_kprobe(p);
1294
	}
1295
	mutex_unlock(&text_mutex);
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319

	kprobe_enabled = true;
	printk(KERN_INFO "Kprobes globally enabled\n");

already_enabled:
	mutex_unlock(&kprobe_mutex);
	return;
}

static void __kprobes disable_all_kprobes(void)
{
	struct hlist_head *head;
	struct hlist_node *node;
	struct kprobe *p;
	unsigned int i;

	mutex_lock(&kprobe_mutex);

	/* If kprobes are already disabled, just return */
	if (!kprobe_enabled)
		goto already_disabled;

	kprobe_enabled = false;
	printk(KERN_INFO "Kprobes globally disabled\n");
1320
	mutex_lock(&text_mutex);
1321 1322 1323
	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
		head = &kprobe_table[i];
		hlist_for_each_entry_rcu(p, node, head, hlist) {
1324
			if (!arch_trampoline_kprobe(p) && !kprobe_gone(p))
1325 1326 1327 1328
				arch_disarm_kprobe(p);
		}
	}

1329
	mutex_unlock(&text_mutex);
1330 1331 1332
	mutex_unlock(&kprobe_mutex);
	/* Allow all currently running kprobes to complete */
	synchronize_sched();
1333
	return;
1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389

already_disabled:
	mutex_unlock(&kprobe_mutex);
	return;
}

/*
 * XXX: The debugfs bool file interface doesn't allow for callbacks
 * when the bool state is switched. We can reuse that facility when
 * available
 */
static ssize_t read_enabled_file_bool(struct file *file,
	       char __user *user_buf, size_t count, loff_t *ppos)
{
	char buf[3];

	if (kprobe_enabled)
		buf[0] = '1';
	else
		buf[0] = '0';
	buf[1] = '\n';
	buf[2] = 0x00;
	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
}

static ssize_t write_enabled_file_bool(struct file *file,
	       const char __user *user_buf, size_t count, loff_t *ppos)
{
	char buf[32];
	int buf_size;

	buf_size = min(count, (sizeof(buf)-1));
	if (copy_from_user(buf, user_buf, buf_size))
		return -EFAULT;

	switch (buf[0]) {
	case 'y':
	case 'Y':
	case '1':
		enable_all_kprobes();
		break;
	case 'n':
	case 'N':
	case '0':
		disable_all_kprobes();
		break;
	}

	return count;
}

static struct file_operations fops_kp = {
	.read =         read_enabled_file_bool,
	.write =        write_enabled_file_bool,
};

1390 1391 1392
static int __kprobes debugfs_kprobe_init(void)
{
	struct dentry *dir, *file;
1393
	unsigned int value = 1;
1394 1395 1396 1397 1398

	dir = debugfs_create_dir("kprobes", NULL);
	if (!dir)
		return -ENOMEM;

R
Randy Dunlap 已提交
1399
	file = debugfs_create_file("list", 0444, dir, NULL,
1400 1401 1402 1403 1404 1405
				&debugfs_kprobes_operations);
	if (!file) {
		debugfs_remove(dir);
		return -ENOMEM;
	}

1406 1407 1408 1409 1410 1411 1412
	file = debugfs_create_file("enabled", 0600, dir,
					&value, &fops_kp);
	if (!file) {
		debugfs_remove(dir);
		return -ENOMEM;
	}

1413 1414 1415 1416 1417 1418 1419
	return 0;
}

late_initcall(debugfs_kprobe_init);
#endif /* CONFIG_DEBUG_FS */

module_init(init_kprobes);
L
Linus Torvalds 已提交
1420 1421 1422

EXPORT_SYMBOL_GPL(register_kprobe);
EXPORT_SYMBOL_GPL(unregister_kprobe);
1423 1424
EXPORT_SYMBOL_GPL(register_kprobes);
EXPORT_SYMBOL_GPL(unregister_kprobes);
L
Linus Torvalds 已提交
1425 1426
EXPORT_SYMBOL_GPL(register_jprobe);
EXPORT_SYMBOL_GPL(unregister_jprobe);
1427 1428
EXPORT_SYMBOL_GPL(register_jprobes);
EXPORT_SYMBOL_GPL(unregister_jprobes);
L
Linus Torvalds 已提交
1429
EXPORT_SYMBOL_GPL(jprobe_return);
1430 1431
EXPORT_SYMBOL_GPL(register_kretprobe);
EXPORT_SYMBOL_GPL(unregister_kretprobe);
1432 1433
EXPORT_SYMBOL_GPL(register_kretprobes);
EXPORT_SYMBOL_GPL(unregister_kretprobes);