hw_breakpoint.c 12.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) 2007 Alan Stern
 * Copyright (C) 2009 IBM Corporation
18
 * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com>
19 20 21 22
 *
 * Authors: Alan Stern <stern@rowland.harvard.edu>
 *          K.Prasad <prasad@linux.vnet.ibm.com>
 *          Frederic Weisbecker <fweisbec@gmail.com>
23 24 25 26 27 28 29
 */

/*
 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
 * using the CPU's debug registers.
 */

30 31
#include <linux/perf_event.h>
#include <linux/hw_breakpoint.h>
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
#include <linux/irqflags.h>
#include <linux/notifier.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/percpu.h>
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/smp.h>

#include <asm/hw_breakpoint.h>
#include <asm/processor.h>
#include <asm/debugreg.h>

48
/* Per cpu debug control register value */
49 50
DEFINE_PER_CPU(unsigned long, cpu_dr7);
EXPORT_PER_CPU_SYMBOL(cpu_dr7);
51 52 53

/* Per cpu debug address registers values */
static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]);
54 55

/*
56 57
 * Stores the breakpoints currently in use on each breakpoint address
 * register for each cpus
58
 */
59
static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]);
60 61


62 63
static inline unsigned long
__encode_dr7(int drnum, unsigned int len, unsigned int type)
64 65 66 67 68
{
	unsigned long bp_info;

	bp_info = (len | type) & 0xf;
	bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE);
69 70
	bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE));

71 72 73
	return bp_info;
}

74 75 76 77 78 79 80 81 82
/*
 * Encode the length, type, Exact, and Enable bits for a particular breakpoint
 * as stored in debug register 7.
 */
unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type)
{
	return __encode_dr7(drnum, len, type) | DR_GLOBAL_SLOWDOWN;
}

83 84 85 86 87
/*
 * Decode the length and type bits for a particular breakpoint as
 * stored in debug register 7.  Return the "enabled" status.
 */
int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type)
88
{
89
	int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE);
90

91 92
	*len = (bp_info & 0xc) | 0x40;
	*type = (bp_info & 0x3) | 0x80;
93

94
	return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3;
95 96 97
}

/*
98 99 100 101 102 103 104
 * Install a perf counter breakpoint.
 *
 * We seek a free debug address register and use it for this
 * breakpoint. Eventually we enable it in the debug control register.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
105
 */
106
int arch_install_hw_breakpoint(struct perf_event *bp)
107
{
108 109 110 111 112 113 114 115 116 117 118
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned long *dr7;
	int i;

	for (i = 0; i < HBP_NUM; i++) {
		struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);

		if (!*slot) {
			*slot = bp;
			break;
		}
119 120
	}

121 122 123 124
	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
		return -EBUSY;

	set_debugreg(info->address, i);
T
Tejun Heo 已提交
125
	__this_cpu_write(cpu_debugreg[i], info->address);
126

127
	dr7 = &__get_cpu_var(cpu_dr7);
128 129 130 131 132
	*dr7 |= encode_dr7(i, info->len, info->type);

	set_debugreg(*dr7, 7);

	return 0;
133 134 135
}

/*
136 137 138 139 140 141 142
 * Uninstall the breakpoint contained in the given counter.
 *
 * First we search the debug address register it uses and then we disable
 * it.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
143
 */
144
void arch_uninstall_hw_breakpoint(struct perf_event *bp)
145
{
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned long *dr7;
	int i;

	for (i = 0; i < HBP_NUM; i++) {
		struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);

		if (*slot == bp) {
			*slot = NULL;
			break;
		}
	}

	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
		return;
161

162
	dr7 = &__get_cpu_var(cpu_dr7);
163
	*dr7 &= ~__encode_dr7(i, info->len, info->type);
164 165

	set_debugreg(*dr7, 7);
166 167 168 169 170 171 172
}

static int get_hbp_len(u8 hbp_len)
{
	unsigned int len_in_bytes = 0;

	switch (hbp_len) {
173
	case X86_BREAKPOINT_LEN_1:
174 175
		len_in_bytes = 1;
		break;
176
	case X86_BREAKPOINT_LEN_2:
177 178
		len_in_bytes = 2;
		break;
179
	case X86_BREAKPOINT_LEN_4:
180 181 182
		len_in_bytes = 4;
		break;
#ifdef CONFIG_X86_64
183
	case X86_BREAKPOINT_LEN_8:
184 185 186 187 188 189 190 191 192 193
		len_in_bytes = 8;
		break;
#endif
	}
	return len_in_bytes;
}

/*
 * Check for virtual address in kernel space.
 */
194
int arch_check_bp_in_kernelspace(struct perf_event *bp)
195 196
{
	unsigned int len;
197 198
	unsigned long va;
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
199

200 201
	va = info->address;
	len = get_hbp_len(info->len);
202 203 204 205

	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
}

206 207
int arch_bp_generic_fields(int x86_len, int x86_type,
			   int *gen_len, int *gen_type)
208
{
209 210 211 212 213 214 215
	/* Type */
	switch (x86_type) {
	case X86_BREAKPOINT_EXECUTE:
		if (x86_len != X86_BREAKPOINT_LEN_X)
			return -EINVAL;

		*gen_type = HW_BREAKPOINT_X;
216
		*gen_len = sizeof(long);
217 218 219
		return 0;
	case X86_BREAKPOINT_WRITE:
		*gen_type = HW_BREAKPOINT_W;
220
		break;
221 222 223 224 225 226 227 228 229
	case X86_BREAKPOINT_RW:
		*gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R;
		break;
	default:
		return -EINVAL;
	}

	/* Len */
	switch (x86_len) {
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
	case X86_BREAKPOINT_LEN_1:
		*gen_len = HW_BREAKPOINT_LEN_1;
		break;
	case X86_BREAKPOINT_LEN_2:
		*gen_len = HW_BREAKPOINT_LEN_2;
		break;
	case X86_BREAKPOINT_LEN_4:
		*gen_len = HW_BREAKPOINT_LEN_4;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
		*gen_len = HW_BREAKPOINT_LEN_8;
		break;
#endif
	default:
		return -EINVAL;
	}
247

248 249 250 251 252 253 254 255 256 257
	return 0;
}


static int arch_build_bp_info(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);

	info->address = bp->attr.bp_addr;

258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
	/* Type */
	switch (bp->attr.bp_type) {
	case HW_BREAKPOINT_W:
		info->type = X86_BREAKPOINT_WRITE;
		break;
	case HW_BREAKPOINT_W | HW_BREAKPOINT_R:
		info->type = X86_BREAKPOINT_RW;
		break;
	case HW_BREAKPOINT_X:
		info->type = X86_BREAKPOINT_EXECUTE;
		/*
		 * x86 inst breakpoints need to have a specific undefined len.
		 * But we still need to check userspace is not trying to setup
		 * an unsupported length, to get a range breakpoint for example.
		 */
		if (bp->attr.bp_len == sizeof(long)) {
			info->len = X86_BREAKPOINT_LEN_X;
			return 0;
		}
	default:
		return -EINVAL;
	}

281 282
	/* Len */
	switch (bp->attr.bp_len) {
283
	case HW_BREAKPOINT_LEN_1:
284
		info->len = X86_BREAKPOINT_LEN_1;
285 286
		break;
	case HW_BREAKPOINT_LEN_2:
287
		info->len = X86_BREAKPOINT_LEN_2;
288 289
		break;
	case HW_BREAKPOINT_LEN_4:
290
		info->len = X86_BREAKPOINT_LEN_4;
291 292 293
		break;
#ifdef CONFIG_X86_64
	case HW_BREAKPOINT_LEN_8:
294 295 296 297 298 299 300 301 302 303 304 305
		info->len = X86_BREAKPOINT_LEN_8;
		break;
#endif
	default:
		return -EINVAL;
	}

	return 0;
}
/*
 * Validate the arch-specific HW Breakpoint register settings
 */
306
int arch_validate_hwbkpt_settings(struct perf_event *bp)
307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned int align;
	int ret;


	ret = arch_build_bp_info(bp);
	if (ret)
		return ret;

	ret = -EINVAL;

	switch (info->len) {
	case X86_BREAKPOINT_LEN_1:
		align = 0;
		break;
	case X86_BREAKPOINT_LEN_2:
		align = 1;
		break;
	case X86_BREAKPOINT_LEN_4:
		align = 3;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
331 332 333 334 335 336 337 338 339 340 341
		align = 7;
		break;
#endif
	default:
		return ret;
	}

	/*
	 * Check that the low-order bits of the address are appropriate
	 * for the alignment implied by len.
	 */
342
	if (info->address & align)
343 344 345 346 347
		return -EINVAL;

	return 0;
}

348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
/*
 * Dump the debug register contents to the user.
 * We can't dump our per cpu values because it
 * may contain cpu wide breakpoint, something that
 * doesn't belong to the current task.
 *
 * TODO: include non-ptrace user breakpoints (perf)
 */
void aout_dump_debugregs(struct user *dump)
{
	int i;
	int dr7 = 0;
	struct perf_event *bp;
	struct arch_hw_breakpoint *info;
	struct thread_struct *thread = &current->thread;

	for (i = 0; i < HBP_NUM; i++) {
		bp = thread->ptrace_bps[i];

		if (bp && !bp->attr.disabled) {
			dump->u_debugreg[i] = bp->attr.bp_addr;
			info = counter_arch_bp(bp);
			dr7 |= encode_dr7(i, info->len, info->type);
		} else {
			dump->u_debugreg[i] = 0;
		}
	}

	dump->u_debugreg[4] = 0;
	dump->u_debugreg[5] = 0;
	dump->u_debugreg[6] = current->thread.debugreg6;

	dump->u_debugreg[7] = dr7;
}
382
EXPORT_SYMBOL_GPL(aout_dump_debugregs);
383

384 385 386 387
/*
 * Release the user breakpoints used by ptrace
 */
void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
388
{
389 390 391 392 393 394 395
	int i;
	struct thread_struct *t = &tsk->thread;

	for (i = 0; i < HBP_NUM; i++) {
		unregister_hw_breakpoint(t->ptrace_bps[i]);
		t->ptrace_bps[i] = NULL;
	}
396 397
}

398
void hw_breakpoint_restore(void)
399
{
T
Tejun Heo 已提交
400 401 402 403
	set_debugreg(__this_cpu_read(cpu_debugreg[0]), 0);
	set_debugreg(__this_cpu_read(cpu_debugreg[1]), 1);
	set_debugreg(__this_cpu_read(cpu_debugreg[2]), 2);
	set_debugreg(__this_cpu_read(cpu_debugreg[3]), 3);
404
	set_debugreg(current->thread.debugreg6, 6);
T
Tejun Heo 已提交
405
	set_debugreg(__this_cpu_read(cpu_dr7), 7);
406
}
407
EXPORT_SYMBOL_GPL(hw_breakpoint_restore);
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424

/*
 * Handle debug exception notifications.
 *
 * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
 *
 * NOTIFY_DONE returned if one of the following conditions is true.
 * i) When the causative address is from user-space and the exception
 * is a valid one, i.e. not triggered as a result of lazy debug register
 * switching
 * ii) When there are more bits than trap<n> set in DR6 register (such
 * as BD, BS or BT) indicating that more than one debug condition is
 * met and requires some more action in do_debug().
 *
 * NOTIFY_STOP returned for all other cases
 *
 */
425
static int __kprobes hw_breakpoint_handler(struct die_args *args)
426 427
{
	int i, cpu, rc = NOTIFY_STOP;
428
	struct perf_event *bp;
429 430 431 432 433 434
	unsigned long dr7, dr6;
	unsigned long *dr6_p;

	/* The DR6 value is pointed by args->err */
	dr6_p = (unsigned long *)ERR_PTR(args->err);
	dr6 = *dr6_p;
435

436 437 438 439
	/* If it's a single step, TRAP bits are random */
	if (dr6 & DR_STEP)
		return NOTIFY_DONE;

440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458
	/* Do an early return if no trap bits are set in DR6 */
	if ((dr6 & DR_TRAP_BITS) == 0)
		return NOTIFY_DONE;

	get_debugreg(dr7, 7);
	/* Disable breakpoints during exception handling */
	set_debugreg(0UL, 7);
	/*
	 * Assert that local interrupts are disabled
	 * Reset the DRn bits in the virtualized register value.
	 * The ptrace trigger routine will add in whatever is needed.
	 */
	current->thread.debugreg6 &= ~DR_TRAP_BITS;
	cpu = get_cpu();

	/* Handle all the breakpoints that were triggered */
	for (i = 0; i < HBP_NUM; ++i) {
		if (likely(!(dr6 & (DR_TRAP0 << i))))
			continue;
459

460
		/*
461 462 463 464
		 * The counter may be concurrently released but that can only
		 * occur from a call_rcu() path. We can then safely fetch
		 * the breakpoint, use its callback, touch its counter
		 * while we are in an rcu_read_lock() path.
465
		 */
466 467 468
		rcu_read_lock();

		bp = per_cpu(bp_per_reg[i], cpu);
469 470 471 472 473
		/*
		 * Reset the 'i'th TRAP bit in dr6 to denote completion of
		 * exception handling
		 */
		(*dr6_p) &= ~(DR_TRAP0 << i);
474 475
		/*
		 * bp can be NULL due to lazy debug register switching
476
		 * or due to concurrent perf counter removing.
477
		 */
478 479 480 481 482
		if (!bp) {
			rcu_read_unlock();
			break;
		}

483
		perf_bp_event(bp, args->regs);
484

485 486 487 488 489 490 491
		/*
		 * Set up resume flag to avoid breakpoint recursion when
		 * returning back to origin.
		 */
		if (bp->hw.info.type == X86_BREAKPOINT_EXECUTE)
			args->regs->flags |= X86_EFLAGS_RF;

492
		rcu_read_unlock();
493
	}
494 495 496 497 498 499 500
	/*
	 * Further processing in do_debug() is needed for a) user-space
	 * breakpoints (to generate signals) and b) when the system has
	 * taken exception due to multiple causes
	 */
	if ((current->thread.debugreg6 & DR_TRAP_BITS) ||
	    (dr6 & (~DR_TRAP_BITS)))
501 502 503
		rc = NOTIFY_DONE;

	set_debugreg(dr7, 7);
504
	put_cpu();
505

506 507 508 509 510 511 512 513 514 515 516 517 518 519
	return rc;
}

/*
 * Handle debug exception notifications.
 */
int __kprobes hw_breakpoint_exceptions_notify(
		struct notifier_block *unused, unsigned long val, void *data)
{
	if (val != DIE_DEBUG)
		return NOTIFY_DONE;

	return hw_breakpoint_handler(data);
}
520 521 522 523 524

void hw_breakpoint_pmu_read(struct perf_event *bp)
{
	/* TODO */
}