process.c 49.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 *  Derived from "arch/i386/kernel/process.c"
 *    Copyright (C) 1995  Linus Torvalds
 *
 *  Updated and modified by Cort Dougan (cort@cs.nmt.edu) and
 *  Paul Mackerras (paulus@cs.anu.edu.au)
 *
 *  PowerPC version
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version
 *  2 of the License, or (at your option) any later version.
 */

#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/stddef.h>
#include <linux/unistd.h>
#include <linux/ptrace.h>
#include <linux/slab.h>
#include <linux/user.h>
#include <linux/elf.h>
#include <linux/prctl.h>
#include <linux/init_task.h>
30
#include <linux/export.h>
31 32 33
#include <linux/kallsyms.h>
#include <linux/mqueue.h>
#include <linux/hardirq.h>
34
#include <linux/utsname.h>
35
#include <linux/ftrace.h>
36
#include <linux/kernel_stat.h>
37 38
#include <linux/personality.h>
#include <linux/random.h>
39
#include <linux/hw_breakpoint.h>
40
#include <linux/uaccess.h>
41
#include <linux/elf-randomize.h>
42 43 44 45 46 47

#include <asm/pgtable.h>
#include <asm/io.h>
#include <asm/processor.h>
#include <asm/mmu.h>
#include <asm/prom.h>
48
#include <asm/machdep.h>
49
#include <asm/time.h>
50
#include <asm/runlatch.h>
51
#include <asm/syscalls.h>
52
#include <asm/switch_to.h>
53
#include <asm/tm.h>
54
#include <asm/debug.h>
55 56 57
#ifdef CONFIG_PPC64
#include <asm/firmware.h>
#endif
58
#include <asm/code-patching.h>
59
#include <asm/exec.h>
60
#include <asm/livepatch.h>
61
#include <asm/cpu_has_feature.h>
62
#include <asm/asm-prototypes.h>
63

64 65
#include <linux/kprobes.h>
#include <linux/kdebug.h>
66

67 68 69 70 71 72 73
/* Transactional Memory debug */
#ifdef TM_DEBUG_SW
#define TM_DEBUG(x...) printk(KERN_INFO x)
#else
#define TM_DEBUG(x...) do { } while(0)
#endif

74 75
extern unsigned long _get_SP(void);

76
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
77
static void check_if_tm_restore_required(struct task_struct *tsk)
78 79 80 81 82 83 84 85 86 87
{
	/*
	 * If we are saving the current thread's registers, and the
	 * thread is in a transactional state, set the TIF_RESTORE_TM
	 * bit so that we know to restore the registers before
	 * returning to userspace.
	 */
	if (tsk == current && tsk->thread.regs &&
	    MSR_TM_ACTIVE(tsk->thread.regs->msr) &&
	    !test_thread_flag(TIF_RESTORE_TM)) {
88
		tsk->thread.ckpt_regs.msr = tsk->thread.regs->msr;
89 90 91
		set_thread_flag(TIF_RESTORE_TM);
	}
}
92 93 94 95 96

static inline bool msr_tm_active(unsigned long msr)
{
	return MSR_TM_ACTIVE(msr);
}
97
#else
98
static inline bool msr_tm_active(unsigned long msr) { return false; }
99
static inline void check_if_tm_restore_required(struct task_struct *tsk) { }
100 101
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */

102 103 104 105 106 107 108 109 110 111 112 113
bool strict_msr_control;
EXPORT_SYMBOL(strict_msr_control);

static int __init enable_strict_msr_control(char *str)
{
	strict_msr_control = true;
	pr_info("Enabling strict facility control\n");

	return 0;
}
early_param("ppc_strict_facility_enable", enable_strict_msr_control);

114
unsigned long msr_check_and_set(unsigned long bits)
115
{
116 117
	unsigned long oldmsr = mfmsr();
	unsigned long newmsr;
118

119
	newmsr = oldmsr | bits;
120 121

#ifdef CONFIG_VSX
122
	if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
123 124
		newmsr |= MSR_VSX;
#endif
125

126 127
	if (oldmsr != newmsr)
		mtmsr_isync(newmsr);
128 129

	return newmsr;
130
}
131

132
void __msr_check_and_clear(unsigned long bits)
133 134 135 136 137 138 139 140 141 142 143 144 145 146
{
	unsigned long oldmsr = mfmsr();
	unsigned long newmsr;

	newmsr = oldmsr & ~bits;

#ifdef CONFIG_VSX
	if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
		newmsr &= ~MSR_VSX;
#endif

	if (oldmsr != newmsr)
		mtmsr_isync(newmsr);
}
147
EXPORT_SYMBOL(__msr_check_and_clear);
148 149

#ifdef CONFIG_PPC_FPU
150 151
void __giveup_fpu(struct task_struct *tsk)
{
152 153
	unsigned long msr;

154
	save_fpu(tsk);
155 156
	msr = tsk->thread.regs->msr;
	msr &= ~MSR_FP;
157 158
#ifdef CONFIG_VSX
	if (cpu_has_feature(CPU_FTR_VSX))
159
		msr &= ~MSR_VSX;
160
#endif
161
	tsk->thread.regs->msr = msr;
162 163
}

164 165 166 167 168
void giveup_fpu(struct task_struct *tsk)
{
	check_if_tm_restore_required(tsk);

	msr_check_and_set(MSR_FP);
169
	__giveup_fpu(tsk);
170
	msr_check_and_clear(MSR_FP);
171 172 173
}
EXPORT_SYMBOL(giveup_fpu);

174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
/*
 * Make sure the floating-point register state in the
 * the thread_struct is up to date for task tsk.
 */
void flush_fp_to_thread(struct task_struct *tsk)
{
	if (tsk->thread.regs) {
		/*
		 * We need to disable preemption here because if we didn't,
		 * another process could get scheduled after the regs->msr
		 * test but before we have finished saving the FP registers
		 * to the thread_struct.  That process could take over the
		 * FPU, and then when we get scheduled again we would store
		 * bogus values for the remaining FP registers.
		 */
		preempt_disable();
		if (tsk->thread.regs->msr & MSR_FP) {
			/*
			 * This should only ever be called for current or
			 * for a stopped child process.  Since we save away
194
			 * the FP register state on context switch,
195 196 197 198
			 * there is something wrong if a stopped child appears
			 * to still have its FP state in the CPU registers.
			 */
			BUG_ON(tsk != current);
199
			giveup_fpu(tsk);
200 201 202 203
		}
		preempt_enable();
	}
}
204
EXPORT_SYMBOL_GPL(flush_fp_to_thread);
205 206 207

void enable_kernel_fp(void)
{
208 209
	unsigned long cpumsr;

210 211
	WARN_ON(preemptible());

212
	cpumsr = msr_check_and_set(MSR_FP);
A
Anton Blanchard 已提交
213

214 215
	if (current->thread.regs && (current->thread.regs->msr & MSR_FP)) {
		check_if_tm_restore_required(current);
216 217 218 219 220 221 222 223 224
		/*
		 * If a thread has already been reclaimed then the
		 * checkpointed registers are on the CPU but have definitely
		 * been saved by the reclaim code. Don't need to and *cannot*
		 * giveup as this would save  to the 'live' structure not the
		 * checkpointed structure.
		 */
		if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
			return;
225
		__giveup_fpu(current);
226
	}
227 228
}
EXPORT_SYMBOL(enable_kernel_fp);
229 230

static int restore_fp(struct task_struct *tsk) {
231
	if (tsk->thread.load_fp || msr_tm_active(tsk->thread.regs->msr)) {
232 233 234 235 236 237 238 239
		load_fp_state(&current->thread.fp_state);
		current->thread.load_fp++;
		return 1;
	}
	return 0;
}
#else
static int restore_fp(struct task_struct *tsk) { return 0; }
240
#endif /* CONFIG_PPC_FPU */
241 242

#ifdef CONFIG_ALTIVEC
243 244
#define loadvec(thr) ((thr).load_vec)

245 246
static void __giveup_altivec(struct task_struct *tsk)
{
247 248
	unsigned long msr;

249
	save_altivec(tsk);
250 251
	msr = tsk->thread.regs->msr;
	msr &= ~MSR_VEC;
252 253
#ifdef CONFIG_VSX
	if (cpu_has_feature(CPU_FTR_VSX))
254
		msr &= ~MSR_VSX;
255
#endif
256
	tsk->thread.regs->msr = msr;
257 258
}

259 260 261 262
void giveup_altivec(struct task_struct *tsk)
{
	check_if_tm_restore_required(tsk);

263
	msr_check_and_set(MSR_VEC);
264
	__giveup_altivec(tsk);
265
	msr_check_and_clear(MSR_VEC);
266 267 268
}
EXPORT_SYMBOL(giveup_altivec);

269 270
void enable_kernel_altivec(void)
{
271 272
	unsigned long cpumsr;

273 274
	WARN_ON(preemptible());

275
	cpumsr = msr_check_and_set(MSR_VEC);
A
Anton Blanchard 已提交
276

277 278
	if (current->thread.regs && (current->thread.regs->msr & MSR_VEC)) {
		check_if_tm_restore_required(current);
279 280 281 282 283 284 285 286 287
		/*
		 * If a thread has already been reclaimed then the
		 * checkpointed registers are on the CPU but have definitely
		 * been saved by the reclaim code. Don't need to and *cannot*
		 * giveup as this would save  to the 'live' structure not the
		 * checkpointed structure.
		 */
		if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
			return;
288
		__giveup_altivec(current);
289
	}
290 291 292 293 294 295 296 297 298 299 300 301 302
}
EXPORT_SYMBOL(enable_kernel_altivec);

/*
 * Make sure the VMX/Altivec register state in the
 * the thread_struct is up to date for task tsk.
 */
void flush_altivec_to_thread(struct task_struct *tsk)
{
	if (tsk->thread.regs) {
		preempt_disable();
		if (tsk->thread.regs->msr & MSR_VEC) {
			BUG_ON(tsk != current);
303
			giveup_altivec(tsk);
304 305 306 307
		}
		preempt_enable();
	}
}
308
EXPORT_SYMBOL_GPL(flush_altivec_to_thread);
309 310 311

static int restore_altivec(struct task_struct *tsk)
{
312 313
	if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
		(tsk->thread.load_vec || msr_tm_active(tsk->thread.regs->msr))) {
314 315 316 317 318 319 320 321 322 323 324
		load_vr_state(&tsk->thread.vr_state);
		tsk->thread.used_vr = 1;
		tsk->thread.load_vec++;

		return 1;
	}
	return 0;
}
#else
#define loadvec(thr) 0
static inline int restore_altivec(struct task_struct *tsk) { return 0; }
325 326
#endif /* CONFIG_ALTIVEC */

327
#ifdef CONFIG_VSX
328
static void __giveup_vsx(struct task_struct *tsk)
329 330 331 332 333
{
	if (tsk->thread.regs->msr & MSR_FP)
		__giveup_fpu(tsk);
	if (tsk->thread.regs->msr & MSR_VEC)
		__giveup_altivec(tsk);
334 335 336 337 338 339 340 341
	tsk->thread.regs->msr &= ~MSR_VSX;
}

static void giveup_vsx(struct task_struct *tsk)
{
	check_if_tm_restore_required(tsk);

	msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
342
	__giveup_vsx(tsk);
343
	msr_check_and_clear(MSR_FP|MSR_VEC|MSR_VSX);
344
}
345 346 347 348 349 350 351 352

static void save_vsx(struct task_struct *tsk)
{
	if (tsk->thread.regs->msr & MSR_FP)
		save_fpu(tsk);
	if (tsk->thread.regs->msr & MSR_VEC)
		save_altivec(tsk);
}
353

354 355
void enable_kernel_vsx(void)
{
356 357
	unsigned long cpumsr;

358 359
	WARN_ON(preemptible());

360
	cpumsr = msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
A
Anton Blanchard 已提交
361

362
	if (current->thread.regs && (current->thread.regs->msr & MSR_VSX)) {
363
		check_if_tm_restore_required(current);
364 365 366 367 368 369 370 371 372
		/*
		 * If a thread has already been reclaimed then the
		 * checkpointed registers are on the CPU but have definitely
		 * been saved by the reclaim code. Don't need to and *cannot*
		 * giveup as this would save  to the 'live' structure not the
		 * checkpointed structure.
		 */
		if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
			return;
373 374 375 376 377
		if (current->thread.regs->msr & MSR_FP)
			__giveup_fpu(current);
		if (current->thread.regs->msr & MSR_VEC)
			__giveup_altivec(current);
		__giveup_vsx(current);
A
Anton Blanchard 已提交
378
	}
379 380 381 382 383 384 385 386 387 388 389 390 391 392
}
EXPORT_SYMBOL(enable_kernel_vsx);

void flush_vsx_to_thread(struct task_struct *tsk)
{
	if (tsk->thread.regs) {
		preempt_disable();
		if (tsk->thread.regs->msr & MSR_VSX) {
			BUG_ON(tsk != current);
			giveup_vsx(tsk);
		}
		preempt_enable();
	}
}
393
EXPORT_SYMBOL_GPL(flush_vsx_to_thread);
394 395 396 397 398 399 400 401 402 403 404 405

static int restore_vsx(struct task_struct *tsk)
{
	if (cpu_has_feature(CPU_FTR_VSX)) {
		tsk->thread.used_vsr = 1;
		return 1;
	}

	return 0;
}
#else
static inline int restore_vsx(struct task_struct *tsk) { return 0; }
406
static inline void save_vsx(struct task_struct *tsk) { }
407 408
#endif /* CONFIG_VSX */

409
#ifdef CONFIG_SPE
410 411 412 413
void giveup_spe(struct task_struct *tsk)
{
	check_if_tm_restore_required(tsk);

414
	msr_check_and_set(MSR_SPE);
415
	__giveup_spe(tsk);
416
	msr_check_and_clear(MSR_SPE);
417 418
}
EXPORT_SYMBOL(giveup_spe);
419 420 421 422 423

void enable_kernel_spe(void)
{
	WARN_ON(preemptible());

424
	msr_check_and_set(MSR_SPE);
A
Anton Blanchard 已提交
425

426 427
	if (current->thread.regs && (current->thread.regs->msr & MSR_SPE)) {
		check_if_tm_restore_required(current);
428
		__giveup_spe(current);
429
	}
430 431 432 433 434 435 436 437 438
}
EXPORT_SYMBOL(enable_kernel_spe);

void flush_spe_to_thread(struct task_struct *tsk)
{
	if (tsk->thread.regs) {
		preempt_disable();
		if (tsk->thread.regs->msr & MSR_SPE) {
			BUG_ON(tsk != current);
439
			tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
440
			giveup_spe(tsk);
441 442 443 444 445 446
		}
		preempt_enable();
	}
}
#endif /* CONFIG_SPE */

A
Anton Blanchard 已提交
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
static unsigned long msr_all_available;

static int __init init_msr_all_available(void)
{
#ifdef CONFIG_PPC_FPU
	msr_all_available |= MSR_FP;
#endif
#ifdef CONFIG_ALTIVEC
	if (cpu_has_feature(CPU_FTR_ALTIVEC))
		msr_all_available |= MSR_VEC;
#endif
#ifdef CONFIG_VSX
	if (cpu_has_feature(CPU_FTR_VSX))
		msr_all_available |= MSR_VSX;
#endif
#ifdef CONFIG_SPE
	if (cpu_has_feature(CPU_FTR_SPE))
		msr_all_available |= MSR_SPE;
#endif

	return 0;
}
early_initcall(init_msr_all_available);

void giveup_all(struct task_struct *tsk)
{
	unsigned long usermsr;

	if (!tsk->thread.regs)
		return;

	usermsr = tsk->thread.regs->msr;

	if ((usermsr & msr_all_available) == 0)
		return;

	msr_check_and_set(msr_all_available);
484
	check_if_tm_restore_required(tsk);
A
Anton Blanchard 已提交
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506

#ifdef CONFIG_PPC_FPU
	if (usermsr & MSR_FP)
		__giveup_fpu(tsk);
#endif
#ifdef CONFIG_ALTIVEC
	if (usermsr & MSR_VEC)
		__giveup_altivec(tsk);
#endif
#ifdef CONFIG_VSX
	if (usermsr & MSR_VSX)
		__giveup_vsx(tsk);
#endif
#ifdef CONFIG_SPE
	if (usermsr & MSR_SPE)
		__giveup_spe(tsk);
#endif

	msr_check_and_clear(msr_all_available);
}
EXPORT_SYMBOL(giveup_all);

507 508 509 510
void restore_math(struct pt_regs *regs)
{
	unsigned long msr;

511 512
	if (!msr_tm_active(regs->msr) &&
		!current->thread.load_fp && !loadvec(current->thread))
513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537
		return;

	msr = regs->msr;
	msr_check_and_set(msr_all_available);

	/*
	 * Only reload if the bit is not set in the user MSR, the bit BEING set
	 * indicates that the registers are hot
	 */
	if ((!(msr & MSR_FP)) && restore_fp(current))
		msr |= MSR_FP | current->thread.fpexc_mode;

	if ((!(msr & MSR_VEC)) && restore_altivec(current))
		msr |= MSR_VEC;

	if ((msr & (MSR_FP | MSR_VEC)) == (MSR_FP | MSR_VEC) &&
			restore_vsx(current)) {
		msr |= MSR_VSX;
	}

	msr_check_and_clear(msr_all_available);

	regs->msr = msr;
}

538 539 540 541 542 543 544 545 546 547 548 549 550 551
void save_all(struct task_struct *tsk)
{
	unsigned long usermsr;

	if (!tsk->thread.regs)
		return;

	usermsr = tsk->thread.regs->msr;

	if ((usermsr & msr_all_available) == 0)
		return;

	msr_check_and_set(msr_all_available);

552 553 554 555 556 557 558 559 560 561 562 563 564
	/*
	 * Saving the way the register space is in hardware, save_vsx boils
	 * down to a save_fpu() and save_altivec()
	 */
	if (usermsr & MSR_VSX) {
		save_vsx(tsk);
	} else {
		if (usermsr & MSR_FP)
			save_fpu(tsk);

		if (usermsr & MSR_VEC)
			save_altivec(tsk);
	}
565 566 567 568 569 570 571

	if (usermsr & MSR_SPE)
		__giveup_spe(tsk);

	msr_check_and_clear(msr_all_available);
}

572 573 574 575 576
void flush_all_to_thread(struct task_struct *tsk)
{
	if (tsk->thread.regs) {
		preempt_disable();
		BUG_ON(tsk != current);
577
		save_all(tsk);
578 579 580 581 582 583 584 585 586 587 588

#ifdef CONFIG_SPE
		if (tsk->thread.regs->msr & MSR_SPE)
			tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
#endif

		preempt_enable();
	}
}
EXPORT_SYMBOL(flush_all_to_thread);

589 590 591 592 593 594
#ifdef CONFIG_PPC_ADV_DEBUG_REGS
void do_send_trap(struct pt_regs *regs, unsigned long address,
		  unsigned long error_code, int signal_code, int breakpt)
{
	siginfo_t info;

595
	current->thread.trap_nr = signal_code;
596 597 598 599 600 601 602 603 604 605 606 607
	if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
			11, SIGSEGV) == NOTIFY_STOP)
		return;

	/* Deliver the signal to userspace */
	info.si_signo = SIGTRAP;
	info.si_errno = breakpt;	/* breakpoint or watchpoint id */
	info.si_code = signal_code;
	info.si_addr = (void __user *)address;
	force_sig_info(SIGTRAP, &info, current);
}
#else	/* !CONFIG_PPC_ADV_DEBUG_REGS */
608
void do_break (struct pt_regs *regs, unsigned long address,
609 610 611 612
		    unsigned long error_code)
{
	siginfo_t info;

613
	current->thread.trap_nr = TRAP_HWBKPT;
614 615 616 617
	if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
			11, SIGSEGV) == NOTIFY_STOP)
		return;

618
	if (debugger_break_match(regs))
619 620
		return;

621 622
	/* Clear the breakpoint */
	hw_breakpoint_disable();
623 624 625 626 627 628 629 630

	/* Deliver the signal to userspace */
	info.si_signo = SIGTRAP;
	info.si_errno = 0;
	info.si_code = TRAP_HWBKPT;
	info.si_addr = (void __user *)address;
	force_sig_info(SIGTRAP, &info, current);
}
631
#endif	/* CONFIG_PPC_ADV_DEBUG_REGS */
632

633
static DEFINE_PER_CPU(struct arch_hw_breakpoint, current_brk);
634

635 636 637 638 639 640
#ifdef CONFIG_PPC_ADV_DEBUG_REGS
/*
 * Set the debug registers back to their default "safe" values.
 */
static void set_debug_reg_defaults(struct thread_struct *thread)
{
641
	thread->debug.iac1 = thread->debug.iac2 = 0;
642
#if CONFIG_PPC_ADV_DEBUG_IACS > 2
643
	thread->debug.iac3 = thread->debug.iac4 = 0;
644
#endif
645
	thread->debug.dac1 = thread->debug.dac2 = 0;
646
#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
647
	thread->debug.dvc1 = thread->debug.dvc2 = 0;
648
#endif
649
	thread->debug.dbcr0 = 0;
650 651 652 653
#ifdef CONFIG_BOOKE
	/*
	 * Force User/Supervisor bits to b11 (user-only MSR[PR]=1)
	 */
654
	thread->debug.dbcr1 = DBCR1_IAC1US | DBCR1_IAC2US |
655 656 657 658 659
			DBCR1_IAC3US | DBCR1_IAC4US;
	/*
	 * Force Data Address Compare User/Supervisor bits to be User-only
	 * (0b11 MSR[PR]=1) and set all other bits in DBCR2 register to be 0.
	 */
660
	thread->debug.dbcr2 = DBCR2_DAC1US | DBCR2_DAC2US;
661
#else
662
	thread->debug.dbcr1 = 0;
663 664 665
#endif
}

666
static void prime_debug_regs(struct debug_reg *debug)
667
{
668 669 670 671 672 673 674
	/*
	 * We could have inherited MSR_DE from userspace, since
	 * it doesn't get cleared on exception entry.  Make sure
	 * MSR_DE is clear before we enable any debug events.
	 */
	mtmsr(mfmsr() & ~MSR_DE);

675 676
	mtspr(SPRN_IAC1, debug->iac1);
	mtspr(SPRN_IAC2, debug->iac2);
677
#if CONFIG_PPC_ADV_DEBUG_IACS > 2
678 679
	mtspr(SPRN_IAC3, debug->iac3);
	mtspr(SPRN_IAC4, debug->iac4);
680
#endif
681 682
	mtspr(SPRN_DAC1, debug->dac1);
	mtspr(SPRN_DAC2, debug->dac2);
683
#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
684 685
	mtspr(SPRN_DVC1, debug->dvc1);
	mtspr(SPRN_DVC2, debug->dvc2);
686
#endif
687 688
	mtspr(SPRN_DBCR0, debug->dbcr0);
	mtspr(SPRN_DBCR1, debug->dbcr1);
689
#ifdef CONFIG_BOOKE
690
	mtspr(SPRN_DBCR2, debug->dbcr2);
691 692 693 694 695 696 697
#endif
}
/*
 * Unless neither the old or new thread are making use of the
 * debug registers, set the debug registers from the values
 * stored in the new thread.
 */
698
void switch_booke_debug_regs(struct debug_reg *new_debug)
699
{
700
	if ((current->thread.debug.dbcr0 & DBCR0_IDM)
701 702
		|| (new_debug->dbcr0 & DBCR0_IDM))
			prime_debug_regs(new_debug);
703
}
704
EXPORT_SYMBOL_GPL(switch_booke_debug_regs);
705
#else	/* !CONFIG_PPC_ADV_DEBUG_REGS */
706
#ifndef CONFIG_HAVE_HW_BREAKPOINT
707 708
static void set_debug_reg_defaults(struct thread_struct *thread)
{
709 710
	thread->hw_brk.address = 0;
	thread->hw_brk.type = 0;
711
	set_breakpoint(&thread->hw_brk);
712
}
713
#endif /* !CONFIG_HAVE_HW_BREAKPOINT */
714 715
#endif	/* CONFIG_PPC_ADV_DEBUG_REGS */

716
#ifdef CONFIG_PPC_ADV_DEBUG_REGS
717 718
static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
{
719
	mtspr(SPRN_DAC1, dabr);
720 721 722
#ifdef CONFIG_PPC_47x
	isync();
#endif
723 724
	return 0;
}
725
#elif defined(CONFIG_PPC_BOOK3S)
726 727
static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
{
728
	mtspr(SPRN_DABR, dabr);
729 730
	if (cpu_has_feature(CPU_FTR_DABRX))
		mtspr(SPRN_DABRX, dabrx);
731
	return 0;
732
}
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
#else
static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
{
	return -EINVAL;
}
#endif

static inline int set_dabr(struct arch_hw_breakpoint *brk)
{
	unsigned long dabr, dabrx;

	dabr = brk->address | (brk->type & HW_BRK_TYPE_DABR);
	dabrx = ((brk->type >> 3) & 0x7);

	if (ppc_md.set_dabr)
		return ppc_md.set_dabr(dabr, dabrx);

	return __set_dabr(dabr, dabrx);
}

753 754
static inline int set_dawr(struct arch_hw_breakpoint *brk)
{
755
	unsigned long dawr, dawrx, mrd;
756 757 758 759 760 761 762 763 764

	dawr = brk->address;

	dawrx  = (brk->type & (HW_BRK_TYPE_READ | HW_BRK_TYPE_WRITE)) \
		                   << (63 - 58); //* read/write bits */
	dawrx |= ((brk->type & (HW_BRK_TYPE_TRANSLATE)) >> 2) \
		                   << (63 - 59); //* translate */
	dawrx |= (brk->type & (HW_BRK_TYPE_PRIV_ALL)) \
		                   >> 3; //* PRIM bits */
765 766 767 768 769 770 771 772
	/* dawr length is stored in field MDR bits 48:53.  Matches range in
	   doublewords (64 bits) baised by -1 eg. 0b000000=1DW and
	   0b111111=64DW.
	   brk->len is in bytes.
	   This aligns up to double word size, shifts and does the bias.
	*/
	mrd = ((brk->len + 7) >> 3) - 1;
	dawrx |= (mrd & 0x3f) << (63 - 53);
773 774 775 776 777 778 779 780

	if (ppc_md.set_dawr)
		return ppc_md.set_dawr(dawr, dawrx);
	mtspr(SPRN_DAWR, dawr);
	mtspr(SPRN_DAWRX, dawrx);
	return 0;
}

781
void __set_breakpoint(struct arch_hw_breakpoint *brk)
782
{
783
	memcpy(this_cpu_ptr(&current_brk), brk, sizeof(*brk));
784

785
	if (cpu_has_feature(CPU_FTR_DAWR))
786 787 788
		set_dawr(brk);
	else
		set_dabr(brk);
789
}
790

791 792 793 794 795 796 797
void set_breakpoint(struct arch_hw_breakpoint *brk)
{
	preempt_disable();
	__set_breakpoint(brk);
	preempt_enable();
}

798 799 800
#ifdef CONFIG_PPC64
DEFINE_PER_CPU(struct cpu_usage, cpu_usage_array);
#endif
801

802 803 804 805 806 807 808 809 810 811 812
static inline bool hw_brk_match(struct arch_hw_breakpoint *a,
			      struct arch_hw_breakpoint *b)
{
	if (a->address != b->address)
		return false;
	if (a->type != b->type)
		return false;
	if (a->len != b->len)
		return false;
	return true;
}
813

814
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
815 816 817 818 819 820

static inline bool tm_enabled(struct task_struct *tsk)
{
	return tsk && tsk->thread.regs && (tsk->thread.regs->msr & MSR_TM);
}

821 822 823
static void tm_reclaim_thread(struct thread_struct *thr,
			      struct thread_info *ti, uint8_t cause)
{
824 825 826 827 828 829 830 831 832 833 834 835 836
	/*
	 * Use the current MSR TM suspended bit to track if we have
	 * checkpointed state outstanding.
	 * On signal delivery, we'd normally reclaim the checkpointed
	 * state to obtain stack pointer (see:get_tm_stackpointer()).
	 * This will then directly return to userspace without going
	 * through __switch_to(). However, if the stack frame is bad,
	 * we need to exit this thread which calls __switch_to() which
	 * will again attempt to reclaim the already saved tm state.
	 * Hence we need to check that we've not already reclaimed
	 * this state.
	 * We do this using the current MSR, rather tracking it in
	 * some specific thread_struct bit, as it has the additional
M
Michael Ellerman 已提交
837
	 * benefit of checking for a potential TM bad thing exception.
838 839 840 841
	 */
	if (!MSR_TM_SUSPENDED(mfmsr()))
		return;

842
	giveup_all(container_of(thr, struct task_struct, thread));
843

844
	tm_reclaim(thr, thr->ckpt_regs.msr, cause);
845 846 847 848 849 850 851 852
}

void tm_reclaim_current(uint8_t cause)
{
	tm_enable();
	tm_reclaim_thread(&current->thread, current_thread_info(), cause);
}

853 854 855 856 857 858 859
static inline void tm_reclaim_task(struct task_struct *tsk)
{
	/* We have to work out if we're switching from/to a task that's in the
	 * middle of a transaction.
	 *
	 * In switching we need to maintain a 2nd register state as
	 * oldtask->thread.ckpt_regs.  We tm_reclaim(oldproc); this saves the
860 861
	 * checkpointed (tbegin) state in ckpt_regs, ckfp_state and
	 * ckvr_state
862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
	 *
	 * We also context switch (save) TFHAR/TEXASR/TFIAR in here.
	 */
	struct thread_struct *thr = &tsk->thread;

	if (!thr->regs)
		return;

	if (!MSR_TM_ACTIVE(thr->regs->msr))
		goto out_and_saveregs;

	TM_DEBUG("--- tm_reclaim on pid %d (NIP=%lx, "
		 "ccr=%lx, msr=%lx, trap=%lx)\n",
		 tsk->pid, thr->regs->nip,
		 thr->regs->ccr, thr->regs->msr,
		 thr->regs->trap);

879
	tm_reclaim_thread(thr, task_thread_info(tsk), TM_CAUSE_RESCHED);
880 881 882 883 884 885 886 887 888 889 890 891 892

	TM_DEBUG("--- tm_reclaim on pid %d complete\n",
		 tsk->pid);

out_and_saveregs:
	/* Always save the regs here, even if a transaction's not active.
	 * This context-switches a thread's TM info SPRs.  We do it here to
	 * be consistent with the restore path (in recheckpoint) which
	 * cannot happen later in _switch().
	 */
	tm_save_sprs(thr);
}

893 894 895 896 897 898 899 900
extern void __tm_recheckpoint(struct thread_struct *thread,
			      unsigned long orig_msr);

void tm_recheckpoint(struct thread_struct *thread,
		     unsigned long orig_msr)
{
	unsigned long flags;

901 902 903
	if (!(thread->regs->msr & MSR_TM))
		return;

904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
	/* We really can't be interrupted here as the TEXASR registers can't
	 * change and later in the trecheckpoint code, we have a userspace R1.
	 * So let's hard disable over this region.
	 */
	local_irq_save(flags);
	hard_irq_disable();

	/* The TM SPRs are restored here, so that TEXASR.FS can be set
	 * before the trecheckpoint and no explosion occurs.
	 */
	tm_restore_sprs(thread);

	__tm_recheckpoint(thread, orig_msr);

	local_irq_restore(flags);
}

921
static inline void tm_recheckpoint_new_task(struct task_struct *new)
922 923 924 925 926 927 928 929 930 931 932
{
	unsigned long msr;

	if (!cpu_has_feature(CPU_FTR_TM))
		return;

	/* Recheckpoint the registers of the thread we're about to switch to.
	 *
	 * If the task was using FP, we non-lazily reload both the original and
	 * the speculative FP register states.  This is because the kernel
	 * doesn't see if/when a TM rollback occurs, so if we take an FP
933
	 * unavailable later, we are unable to determine which set of FP regs
934 935
	 * need to be restored.
	 */
936
	if (!tm_enabled(new))
937 938
		return;

939 940
	if (!MSR_TM_ACTIVE(new->thread.regs->msr)){
		tm_restore_sprs(&new->thread);
941
		return;
942
	}
943
	msr = new->thread.ckpt_regs.msr;
944 945 946 947 948 949 950
	/* Recheckpoint to restore original checkpointed register state. */
	TM_DEBUG("*** tm_recheckpoint of pid %d "
		 "(new->msr 0x%lx, new->origmsr 0x%lx)\n",
		 new->pid, new->thread.regs->msr, msr);

	tm_recheckpoint(&new->thread, msr);

951 952 953 954 955 956
	/*
	 * The checkpointed state has been restored but the live state has
	 * not, ensure all the math functionality is turned off to trigger
	 * restore_math() to reload.
	 */
	new->thread.regs->msr &= ~(MSR_FP | MSR_VEC | MSR_VSX);
957 958 959 960 961 962

	TM_DEBUG("*** tm_recheckpoint of pid %d complete "
		 "(kernel msr 0x%lx)\n",
		 new->pid, mfmsr());
}

963 964
static inline void __switch_to_tm(struct task_struct *prev,
		struct task_struct *new)
965 966
{
	if (cpu_has_feature(CPU_FTR_TM)) {
967 968 969 970 971 972 973 974 975 976
		if (tm_enabled(prev) || tm_enabled(new))
			tm_enable();

		if (tm_enabled(prev)) {
			prev->thread.load_tm++;
			tm_reclaim_task(prev);
			if (!MSR_TM_ACTIVE(prev->thread.regs->msr) && prev->thread.load_tm == 0)
				prev->thread.regs->msr &= ~MSR_TM;
		}

977
		tm_recheckpoint_new_task(new);
978 979
	}
}
980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998

/*
 * This is called if we are on the way out to userspace and the
 * TIF_RESTORE_TM flag is set.  It checks if we need to reload
 * FP and/or vector state and does so if necessary.
 * If userspace is inside a transaction (whether active or
 * suspended) and FP/VMX/VSX instructions have ever been enabled
 * inside that transaction, then we have to keep them enabled
 * and keep the FP/VMX/VSX state loaded while ever the transaction
 * continues.  The reason is that if we didn't, and subsequently
 * got a FP/VMX/VSX unavailable interrupt inside a transaction,
 * we don't know whether it's the same transaction, and thus we
 * don't know which of the checkpointed state and the transactional
 * state to use.
 */
void restore_tm_state(struct pt_regs *regs)
{
	unsigned long msr_diff;

999 1000 1001 1002 1003 1004
	/*
	 * This is the only moment we should clear TIF_RESTORE_TM as
	 * it is here that ckpt_regs.msr and pt_regs.msr become the same
	 * again, anything else could lead to an incorrect ckpt_msr being
	 * saved and therefore incorrect signal contexts.
	 */
1005 1006 1007 1008
	clear_thread_flag(TIF_RESTORE_TM);
	if (!MSR_TM_ACTIVE(regs->msr))
		return;

1009
	msr_diff = current->thread.ckpt_regs.msr & ~regs->msr;
1010
	msr_diff &= MSR_FP | MSR_VEC | MSR_VSX;
1011

1012 1013 1014
	/* Ensure that restore_math() will restore */
	if (msr_diff & MSR_FP)
		current->thread.load_fp = 1;
1015
#ifdef CONFIG_ALTIVEC
1016 1017 1018
	if (cpu_has_feature(CPU_FTR_ALTIVEC) && msr_diff & MSR_VEC)
		current->thread.load_vec = 1;
#endif
1019 1020
	restore_math(regs);

1021 1022 1023
	regs->msr |= msr_diff;
}

1024 1025
#else
#define tm_recheckpoint_new_task(new)
1026
#define __switch_to_tm(prev, new)
1027
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
1028

1029 1030 1031
static inline void save_sprs(struct thread_struct *t)
{
#ifdef CONFIG_ALTIVEC
1032
	if (cpu_has_feature(CPU_FTR_ALTIVEC))
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
		t->vrsave = mfspr(SPRN_VRSAVE);
#endif
#ifdef CONFIG_PPC_BOOK3S_64
	if (cpu_has_feature(CPU_FTR_DSCR))
		t->dscr = mfspr(SPRN_DSCR);

	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
		t->bescr = mfspr(SPRN_BESCR);
		t->ebbhr = mfspr(SPRN_EBBHR);
		t->ebbrr = mfspr(SPRN_EBBRR);

		t->fscr = mfspr(SPRN_FSCR);

		/*
		 * Note that the TAR is not available for use in the kernel.
		 * (To provide this, the TAR should be backed up/restored on
		 * exception entry/exit instead, and be in pt_regs.  FIXME,
		 * this should be in pt_regs anyway (for debug).)
		 */
		t->tar = mfspr(SPRN_TAR);
	}
1054 1055 1056 1057 1058 1059 1060 1061

	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
		/* Conditionally save Load Monitor registers, if enabled */
		if (t->fscr & FSCR_LM) {
			t->lmrr = mfspr(SPRN_LMRR);
			t->lmser = mfspr(SPRN_LMSER);
		}
	}
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
#endif
}

static inline void restore_sprs(struct thread_struct *old_thread,
				struct thread_struct *new_thread)
{
#ifdef CONFIG_ALTIVEC
	if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
	    old_thread->vrsave != new_thread->vrsave)
		mtspr(SPRN_VRSAVE, new_thread->vrsave);
#endif
#ifdef CONFIG_PPC_BOOK3S_64
	if (cpu_has_feature(CPU_FTR_DSCR)) {
		u64 dscr = get_paca()->dscr_default;
1076
		if (new_thread->dscr_inherit)
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
			dscr = new_thread->dscr;

		if (old_thread->dscr != dscr)
			mtspr(SPRN_DSCR, dscr);
	}

	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
		if (old_thread->bescr != new_thread->bescr)
			mtspr(SPRN_BESCR, new_thread->bescr);
		if (old_thread->ebbhr != new_thread->ebbhr)
			mtspr(SPRN_EBBHR, new_thread->ebbhr);
		if (old_thread->ebbrr != new_thread->ebbrr)
			mtspr(SPRN_EBBRR, new_thread->ebbrr);

1091 1092 1093
		if (old_thread->fscr != new_thread->fscr)
			mtspr(SPRN_FSCR, new_thread->fscr);

1094 1095 1096
		if (old_thread->tar != new_thread->tar)
			mtspr(SPRN_TAR, new_thread->tar);
	}
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106

	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
		/* Conditionally restore Load Monitor registers, if enabled */
		if (new_thread->fscr & FSCR_LM) {
			if (old_thread->lmrr != new_thread->lmrr)
				mtspr(SPRN_LMRR, new_thread->lmrr);
			if (old_thread->lmser != new_thread->lmser)
				mtspr(SPRN_LMSER, new_thread->lmser);
		}
	}
1107 1108 1109
#endif
}

1110 1111 1112 1113 1114
struct task_struct *__switch_to(struct task_struct *prev,
	struct task_struct *new)
{
	struct thread_struct *new_thread, *old_thread;
	struct task_struct *last;
P
Peter Zijlstra 已提交
1115 1116 1117
#ifdef CONFIG_PPC_BOOK3S_64
	struct ppc64_tlb_batch *batch;
#endif
1118

1119 1120 1121
	new_thread = &new->thread;
	old_thread = &current->thread;

1122 1123
	WARN_ON(!irqs_disabled());

1124 1125 1126 1127 1128
#ifdef CONFIG_PPC64
	/*
	 * Collect processor utilization data per process
	 */
	if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
1129
		struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
1130 1131 1132 1133 1134 1135
		long unsigned start_tb, current_tb;
		start_tb = old_thread->start_tb;
		cu->current_tb = current_tb = mfspr(SPRN_PURR);
		old_thread->accum_tb += (current_tb - start_tb);
		new_thread->start_tb = current_tb;
	}
P
Peter Zijlstra 已提交
1136 1137
#endif /* CONFIG_PPC64 */

1138
#ifdef CONFIG_PPC_STD_MMU_64
1139
	batch = this_cpu_ptr(&ppc64_tlb_batch);
P
Peter Zijlstra 已提交
1140 1141 1142 1143 1144 1145
	if (batch->active) {
		current_thread_info()->local_flags |= _TLF_LAZY_MMU;
		if (batch->index)
			__flush_tlb_pending(batch);
		batch->active = 0;
	}
1146
#endif /* CONFIG_PPC_STD_MMU_64 */
1147

A
Anton Blanchard 已提交
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169
#ifdef CONFIG_PPC_ADV_DEBUG_REGS
	switch_booke_debug_regs(&new->thread.debug);
#else
/*
 * For PPC_BOOK3S_64, we use the hw-breakpoint interfaces that would
 * schedule DABR
 */
#ifndef CONFIG_HAVE_HW_BREAKPOINT
	if (unlikely(!hw_brk_match(this_cpu_ptr(&current_brk), &new->thread.hw_brk)))
		__set_breakpoint(&new->thread.hw_brk);
#endif /* CONFIG_HAVE_HW_BREAKPOINT */
#endif

	/*
	 * We need to save SPRs before treclaim/trecheckpoint as these will
	 * change a number of them.
	 */
	save_sprs(&prev->thread);

	/* Save FPU, Altivec, VSX and SPE state */
	giveup_all(prev);

1170 1171
	__switch_to_tm(prev, new);

1172 1173 1174 1175 1176 1177
	/*
	 * We can't take a PMU exception inside _switch() since there is a
	 * window where the kernel stack SLB and the kernel stack are out
	 * of sync. Hard disable here.
	 */
	hard_irq_disable();
1178

1179 1180 1181 1182 1183 1184 1185
	/*
	 * Call restore_sprs() before calling _switch(). If we move it after
	 * _switch() then we miss out on calling it for new tasks. The reason
	 * for this is we manually create a stack frame for new tasks that
	 * directly returns through ret_from_fork() or
	 * ret_from_kernel_thread(). See copy_thread() for details.
	 */
A
Anton Blanchard 已提交
1186 1187
	restore_sprs(old_thread, new_thread);

1188 1189
	last = _switch(old_thread, new_thread);

1190
#ifdef CONFIG_PPC_STD_MMU_64
P
Peter Zijlstra 已提交
1191 1192
	if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
		current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
1193
		batch = this_cpu_ptr(&ppc64_tlb_batch);
P
Peter Zijlstra 已提交
1194 1195
		batch->active = 1;
	}
1196 1197 1198

	if (current_thread_info()->task->thread.regs)
		restore_math(current_thread_info()->task->thread.regs);
1199
#endif /* CONFIG_PPC_STD_MMU_64 */
P
Peter Zijlstra 已提交
1200

1201 1202 1203
	return last;
}

1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
static int instructions_to_print = 16;

static void show_instructions(struct pt_regs *regs)
{
	int i;
	unsigned long pc = regs->nip - (instructions_to_print * 3 / 4 *
			sizeof(int));

	printk("Instruction dump:");

	for (i = 0; i < instructions_to_print; i++) {
		int instr;

		if (!(i % 8))
			printk("\n");

1220 1221 1222 1223 1224 1225 1226 1227
#if !defined(CONFIG_BOOKE)
		/* If executing with the IMMU off, adjust pc rather
		 * than print XXXXXXXX.
		 */
		if (!(regs->msr & MSR_IR))
			pc = (unsigned long)phys_to_virt(pc);
#endif

1228
		if (!__kernel_text_address(pc) ||
1229
		     probe_kernel_address((unsigned int __user *)pc, instr)) {
1230
			printk(KERN_CONT "XXXXXXXX ");
1231 1232
		} else {
			if (regs->nip == pc)
1233
				printk(KERN_CONT "<%08x> ", instr);
1234
			else
1235
				printk(KERN_CONT "%08x ", instr);
1236 1237 1238 1239 1240 1241 1242 1243
		}

		pc += sizeof(int);
	}

	printk("\n");
}

1244
struct regbit {
1245 1246
	unsigned long bit;
	const char *name;
1247 1248 1249
};

static struct regbit msr_bits[] = {
1250 1251 1252 1253 1254 1255 1256 1257 1258
#if defined(CONFIG_PPC64) && !defined(CONFIG_BOOKE)
	{MSR_SF,	"SF"},
	{MSR_HV,	"HV"},
#endif
	{MSR_VEC,	"VEC"},
	{MSR_VSX,	"VSX"},
#ifdef CONFIG_BOOKE
	{MSR_CE,	"CE"},
#endif
1259 1260 1261 1262
	{MSR_EE,	"EE"},
	{MSR_PR,	"PR"},
	{MSR_FP,	"FP"},
	{MSR_ME,	"ME"},
1263
#ifdef CONFIG_BOOKE
1264
	{MSR_DE,	"DE"},
1265 1266 1267 1268
#else
	{MSR_SE,	"SE"},
	{MSR_BE,	"BE"},
#endif
1269 1270
	{MSR_IR,	"IR"},
	{MSR_DR,	"DR"},
1271 1272 1273 1274 1275
	{MSR_PMM,	"PMM"},
#ifndef CONFIG_BOOKE
	{MSR_RI,	"RI"},
	{MSR_LE,	"LE"},
#endif
1276 1277 1278
	{0,		NULL}
};

1279
static void print_bits(unsigned long val, struct regbit *bits, const char *sep)
1280
{
1281
	const char *s = "";
1282 1283 1284

	for (; bits->bit; ++bits)
		if (val & bits->bit) {
1285
			pr_cont("%s%s", s, bits->name);
1286
			s = sep;
1287
		}
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
}

#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
static struct regbit msr_tm_bits[] = {
	{MSR_TS_T,	"T"},
	{MSR_TS_S,	"S"},
	{MSR_TM,	"E"},
	{0,		NULL}
};

static void print_tm_bits(unsigned long val)
{
/*
 * This only prints something if at least one of the TM bit is set.
 * Inside the TM[], the output means:
 *   E: Enabled		(bit 32)
 *   S: Suspended	(bit 33)
 *   T: Transactional	(bit 34)
 */
	if (val & (MSR_TM | MSR_TS_S | MSR_TS_T)) {
1308
		pr_cont(",TM[");
1309
		print_bits(val, msr_tm_bits, "");
1310
		pr_cont("]");
1311 1312 1313 1314 1315 1316 1317 1318
	}
}
#else
static void print_tm_bits(unsigned long val) {}
#endif

static void print_msr_bits(unsigned long val)
{
1319
	pr_cont("<");
1320 1321
	print_bits(val, msr_bits, ",");
	print_tm_bits(val);
1322
	pr_cont(">");
1323 1324 1325
}

#ifdef CONFIG_PPC64
1326
#define REG		"%016lx"
1327 1328 1329
#define REGS_PER_LINE	4
#define LAST_VOLATILE	13
#else
1330
#define REG		"%08lx"
1331 1332 1333 1334
#define REGS_PER_LINE	8
#define LAST_VOLATILE	12
#endif

1335 1336 1337 1338
void show_regs(struct pt_regs * regs)
{
	int i, trap;

1339 1340
	show_regs_print_info(KERN_DEFAULT);

1341 1342 1343
	printk("NIP: "REG" LR: "REG" CTR: "REG"\n",
	       regs->nip, regs->link, regs->ctr);
	printk("REGS: %p TRAP: %04lx   %s  (%s)\n",
1344
	       regs, regs->trap, print_tainted(), init_utsname()->release);
1345
	printk("MSR: "REG" ", regs->msr);
1346
	print_msr_bits(regs->msr);
1347
	printk("  CR: %08lx  XER: %08lx\n", regs->ccr, regs->xer);
1348
	trap = TRAP(regs);
1349
	if ((regs->trap != 0xc00) && cpu_has_feature(CPU_FTR_CFAR))
1350
		pr_cont("CFAR: "REG" ", regs->orig_gpr3);
1351
	if (trap == 0x200 || trap == 0x300 || trap == 0x600)
1352
#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
1353
		pr_cont("DEAR: "REG" ESR: "REG" ", regs->dar, regs->dsisr);
1354
#else
1355
		pr_cont("DAR: "REG" DSISR: %08lx ", regs->dar, regs->dsisr);
1356 1357
#endif
#ifdef CONFIG_PPC64
1358
	pr_cont("SOFTE: %ld ", regs->softe);
1359 1360
#endif
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1361
	if (MSR_TM_ACTIVE(regs->msr))
1362
		pr_cont("\nPACATMSCRATCH: %016llx ", get_paca()->tm_scratch);
1363
#endif
1364 1365

	for (i = 0;  i < 32;  i++) {
1366
		if ((i % REGS_PER_LINE) == 0)
1367 1368
			pr_cont("\nGPR%02d: ", i);
		pr_cont(REG " ", regs->gpr[i]);
1369
		if (i == LAST_VOLATILE && !FULL_REGS(regs))
1370 1371
			break;
	}
1372
	pr_cont("\n");
1373 1374 1375 1376 1377
#ifdef CONFIG_KALLSYMS
	/*
	 * Lookup NIP late so we have the best change of getting the
	 * above info out without failing
	 */
1378 1379
	printk("NIP ["REG"] %pS\n", regs->nip, (void *)regs->nip);
	printk("LR ["REG"] %pS\n", regs->link, (void *)regs->link);
1380
#endif
1381
	show_stack(current, (unsigned long *) regs->gpr[1]);
1382 1383
	if (!user_mode(regs))
		show_instructions(regs);
1384 1385 1386 1387
}

void flush_thread(void)
{
1388
#ifdef CONFIG_HAVE_HW_BREAKPOINT
1389
	flush_ptrace_hw_breakpoint(current);
1390
#else /* CONFIG_HAVE_HW_BREAKPOINT */
1391
	set_debug_reg_defaults(&current->thread);
1392
#endif /* CONFIG_HAVE_HW_BREAKPOINT */
1393 1394 1395 1396 1397 1398 1399 1400
}

void
release_thread(struct task_struct *t)
{
}

/*
1401 1402
 * this gets called so that we can store coprocessor state into memory and
 * copy the current task into the new thread.
1403
 */
1404
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
1405
{
1406
	flush_all_to_thread(src);
1407 1408 1409 1410 1411 1412
	/*
	 * Flush TM state out so we can copy it.  __switch_to_tm() does this
	 * flush but it removes the checkpointed state from the current CPU and
	 * transitions the CPU out of TM mode.  Hence we need to call
	 * tm_recheckpoint_new_task() (on the same task) to restore the
	 * checkpointed state back and the TM mode.
1413 1414 1415
	 *
	 * Can't pass dst because it isn't ready. Doesn't matter, passing
	 * dst is only important for __switch_to()
1416
	 */
1417
	__switch_to_tm(src, src);
1418

1419
	*dst = *src;
1420 1421 1422

	clear_task_ebb(dst);

1423
	return 0;
1424 1425
}

1426 1427 1428 1429 1430 1431
static void setup_ksp_vsid(struct task_struct *p, unsigned long sp)
{
#ifdef CONFIG_PPC_STD_MMU_64
	unsigned long sp_vsid;
	unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp;

1432 1433 1434
	if (radix_enabled())
		return;

1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
	if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
		sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_1T)
			<< SLB_VSID_SHIFT_1T;
	else
		sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_256M)
			<< SLB_VSID_SHIFT;
	sp_vsid |= SLB_VSID_KERNEL | llp;
	p->thread.ksp_vsid = sp_vsid;
#endif
}

1446 1447 1448
/*
 * Copy a thread..
 */
1449

1450 1451 1452
/*
 * Copy architecture-specific thread state
 */
A
Alexey Dobriyan 已提交
1453
int copy_thread(unsigned long clone_flags, unsigned long usp,
1454
		unsigned long kthread_arg, struct task_struct *p)
1455 1456 1457
{
	struct pt_regs *childregs, *kregs;
	extern void ret_from_fork(void);
A
Al Viro 已提交
1458 1459
	extern void ret_from_kernel_thread(void);
	void (*f)(void);
A
Al Viro 已提交
1460
	unsigned long sp = (unsigned long)task_stack_page(p) + THREAD_SIZE;
1461 1462 1463
	struct thread_info *ti = task_thread_info(p);

	klp_init_thread_info(ti);
1464 1465 1466 1467

	/* Copy registers */
	sp -= sizeof(struct pt_regs);
	childregs = (struct pt_regs *) sp;
1468
	if (unlikely(p->flags & PF_KTHREAD)) {
1469
		/* kernel thread */
A
Al Viro 已提交
1470
		memset(childregs, 0, sizeof(struct pt_regs));
1471
		childregs->gpr[1] = sp + sizeof(struct pt_regs);
1472 1473 1474
		/* function */
		if (usp)
			childregs->gpr[14] = ppc_function_entry((void *)usp);
A
Al Viro 已提交
1475
#ifdef CONFIG_PPC64
A
Al Viro 已提交
1476
		clear_tsk_thread_flag(p, TIF_32BIT);
1477
		childregs->softe = 1;
1478
#endif
1479
		childregs->gpr[15] = kthread_arg;
1480
		p->thread.regs = NULL;	/* no user register state */
1481
		ti->flags |= _TIF_RESTOREALL;
A
Al Viro 已提交
1482
		f = ret_from_kernel_thread;
1483
	} else {
1484
		/* user thread */
1485
		struct pt_regs *regs = current_pt_regs();
A
Al Viro 已提交
1486 1487
		CHECK_FULL_REGS(regs);
		*childregs = *regs;
1488 1489
		if (usp)
			childregs->gpr[1] = usp;
1490
		p->thread.regs = childregs;
A
Al Viro 已提交
1491
		childregs->gpr[3] = 0;  /* Result from fork() */
1492 1493
		if (clone_flags & CLONE_SETTLS) {
#ifdef CONFIG_PPC64
1494
			if (!is_32bit_task())
1495 1496 1497 1498 1499
				childregs->gpr[13] = childregs->gpr[6];
			else
#endif
				childregs->gpr[2] = childregs->gpr[6];
		}
A
Al Viro 已提交
1500 1501

		f = ret_from_fork;
1502
	}
1503
	childregs->msr &= ~(MSR_FP|MSR_VEC|MSR_VSX);
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
	sp -= STACK_FRAME_OVERHEAD;

	/*
	 * The way this works is that at some point in the future
	 * some task will call _switch to switch to the new task.
	 * That will pop off the stack frame created below and start
	 * the new task running at ret_from_fork.  The new task will
	 * do some house keeping and then return from the fork or clone
	 * system call, using the stack frame created above.
	 */
1514
	((unsigned long *)sp)[0] = 0;
1515 1516 1517 1518
	sp -= sizeof(struct pt_regs);
	kregs = (struct pt_regs *) sp;
	sp -= STACK_FRAME_OVERHEAD;
	p->thread.ksp = sp;
1519
#ifdef CONFIG_PPC32
1520 1521
	p->thread.ksp_limit = (unsigned long)task_stack_page(p) +
				_ALIGN_UP(sizeof(struct thread_info), 16);
1522
#endif
1523 1524 1525 1526
#ifdef CONFIG_HAVE_HW_BREAKPOINT
	p->thread.ptrace_bps[0] = NULL;
#endif

1527 1528 1529 1530 1531
	p->thread.fp_save_area = NULL;
#ifdef CONFIG_ALTIVEC
	p->thread.vr_save_area = NULL;
#endif

1532 1533
	setup_ksp_vsid(p, sp);

1534 1535
#ifdef CONFIG_PPC64 
	if (cpu_has_feature(CPU_FTR_DSCR)) {
1536
		p->thread.dscr_inherit = current->thread.dscr_inherit;
1537
		p->thread.dscr = mfspr(SPRN_DSCR);
1538
	}
1539 1540
	if (cpu_has_feature(CPU_FTR_HAS_PPR))
		p->thread.ppr = INIT_PPR;
1541
#endif
1542
	kregs->nip = ppc_function_entry(f);
1543 1544 1545 1546 1547 1548
	return 0;
}

/*
 * Set up a thread for executing a new program
 */
1549
void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
1550
{
1551 1552 1553 1554
#ifdef CONFIG_PPC64
	unsigned long load_addr = regs->gpr[2];	/* saved by ELF_PLAT_INIT */
#endif

1555 1556 1557 1558 1559
	/*
	 * If we exec out of a kernel thread then thread.regs will not be
	 * set.  Do it now.
	 */
	if (!current->thread.regs) {
A
Al Viro 已提交
1560 1561
		struct pt_regs *regs = task_stack_page(current) + THREAD_SIZE;
		current->thread.regs = regs - 1;
1562 1563
	}

1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
	/*
	 * Clear any transactional state, we're exec()ing. The cause is
	 * not important as there will never be a recheckpoint so it's not
	 * user visible.
	 */
	if (MSR_TM_SUSPENDED(mfmsr()))
		tm_reclaim_current(0);
#endif

1574 1575 1576 1577 1578 1579
	memset(regs->gpr, 0, sizeof(regs->gpr));
	regs->ctr = 0;
	regs->link = 0;
	regs->xer = 0;
	regs->ccr = 0;
	regs->gpr[1] = sp;
1580

1581 1582 1583 1584 1585 1586 1587
	/*
	 * We have just cleared all the nonvolatile GPRs, so make
	 * FULL_REGS(regs) return true.  This is necessary to allow
	 * ptrace to examine the thread immediately after exec.
	 */
	regs->trap &= ~1UL;

1588 1589 1590
#ifdef CONFIG_PPC32
	regs->mq = 0;
	regs->nip = start;
1591
	regs->msr = MSR_USER;
1592
#else
1593
	if (!is_32bit_task()) {
1594
		unsigned long entry;
1595

1596 1597 1598
		if (is_elf2_task()) {
			/* Look ma, no function descriptors! */
			entry = start;
1599

1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
			/*
			 * Ulrich says:
			 *   The latest iteration of the ABI requires that when
			 *   calling a function (at its global entry point),
			 *   the caller must ensure r12 holds the entry point
			 *   address (so that the function can quickly
			 *   establish addressability).
			 */
			regs->gpr[12] = start;
			/* Make sure that's restored on entry to userspace. */
			set_thread_flag(TIF_RESTOREALL);
		} else {
			unsigned long toc;

			/* start is a relocated pointer to the function
			 * descriptor for the elf _start routine.  The first
			 * entry in the function descriptor is the entry
			 * address of _start and the second entry is the TOC
			 * value we need to use.
			 */
			__get_user(entry, (unsigned long __user *)start);
			__get_user(toc, (unsigned long __user *)start+1);

			/* Check whether the e_entry function descriptor entries
			 * need to be relocated before we can use them.
			 */
			if (load_addr != 0) {
				entry += load_addr;
				toc   += load_addr;
			}
			regs->gpr[2] = toc;
1631 1632 1633
		}
		regs->nip = entry;
		regs->msr = MSR_USER64;
S
Stephen Rothwell 已提交
1634 1635 1636 1637
	} else {
		regs->nip = start;
		regs->gpr[2] = 0;
		regs->msr = MSR_USER32;
1638 1639
	}
#endif
1640 1641 1642
#ifdef CONFIG_VSX
	current->thread.used_vsr = 0;
#endif
1643
	memset(&current->thread.fp_state, 0, sizeof(current->thread.fp_state));
1644
	current->thread.fp_save_area = NULL;
1645
#ifdef CONFIG_ALTIVEC
1646 1647
	memset(&current->thread.vr_state, 0, sizeof(current->thread.vr_state));
	current->thread.vr_state.vscr.u[3] = 0x00010000; /* Java mode disabled */
1648
	current->thread.vr_save_area = NULL;
1649 1650 1651 1652 1653 1654 1655 1656 1657
	current->thread.vrsave = 0;
	current->thread.used_vr = 0;
#endif /* CONFIG_ALTIVEC */
#ifdef CONFIG_SPE
	memset(current->thread.evr, 0, sizeof(current->thread.evr));
	current->thread.acc = 0;
	current->thread.spefscr = 0;
	current->thread.used_spe = 0;
#endif /* CONFIG_SPE */
1658 1659 1660 1661 1662
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
	current->thread.tm_tfhar = 0;
	current->thread.tm_texasr = 0;
	current->thread.tm_tfiar = 0;
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
1663
}
1664
EXPORT_SYMBOL(start_thread);
1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678

#define PR_FP_ALL_EXCEPT (PR_FP_EXC_DIV | PR_FP_EXC_OVF | PR_FP_EXC_UND \
		| PR_FP_EXC_RES | PR_FP_EXC_INV)

int set_fpexc_mode(struct task_struct *tsk, unsigned int val)
{
	struct pt_regs *regs = tsk->thread.regs;

	/* This is a bit hairy.  If we are an SPE enabled  processor
	 * (have embedded fp) we store the IEEE exception enable flags in
	 * fpexc_mode.  fpexc_mode is also used for setting FP exception
	 * mode (asyn, precise, disabled) for 'Classic' FP. */
	if (val & PR_FP_EXC_SW_ENABLE) {
#ifdef CONFIG_SPE
1679
		if (cpu_has_feature(CPU_FTR_SPE)) {
1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
			/*
			 * When the sticky exception bits are set
			 * directly by userspace, it must call prctl
			 * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
			 * in the existing prctl settings) or
			 * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
			 * the bits being set).  <fenv.h> functions
			 * saving and restoring the whole
			 * floating-point environment need to do so
			 * anyway to restore the prctl settings from
			 * the saved environment.
			 */
			tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
1693 1694 1695 1696 1697 1698
			tsk->thread.fpexc_mode = val &
				(PR_FP_EXC_SW_ENABLE | PR_FP_ALL_EXCEPT);
			return 0;
		} else {
			return -EINVAL;
		}
1699 1700 1701 1702
#else
		return -EINVAL;
#endif
	}
1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714

	/* on a CONFIG_SPE this does not hurt us.  The bits that
	 * __pack_fe01 use do not overlap with bits used for
	 * PR_FP_EXC_SW_ENABLE.  Additionally, the MSR[FE0,FE1] bits
	 * on CONFIG_SPE implementations are reserved so writing to
	 * them does not change anything */
	if (val > PR_FP_EXC_PRECISE)
		return -EINVAL;
	tsk->thread.fpexc_mode = __pack_fe01(val);
	if (regs != NULL && (regs->msr & MSR_FP) != 0)
		regs->msr = (regs->msr & ~(MSR_FE0|MSR_FE1))
			| tsk->thread.fpexc_mode;
1715 1716 1717 1718 1719 1720 1721 1722 1723
	return 0;
}

int get_fpexc_mode(struct task_struct *tsk, unsigned long adr)
{
	unsigned int val;

	if (tsk->thread.fpexc_mode & PR_FP_EXC_SW_ENABLE)
#ifdef CONFIG_SPE
1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
		if (cpu_has_feature(CPU_FTR_SPE)) {
			/*
			 * When the sticky exception bits are set
			 * directly by userspace, it must call prctl
			 * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
			 * in the existing prctl settings) or
			 * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
			 * the bits being set).  <fenv.h> functions
			 * saving and restoring the whole
			 * floating-point environment need to do so
			 * anyway to restore the prctl settings from
			 * the saved environment.
			 */
			tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
1738
			val = tsk->thread.fpexc_mode;
1739
		} else
1740
			return -EINVAL;
1741 1742 1743 1744 1745 1746 1747 1748
#else
		return -EINVAL;
#endif
	else
		val = __unpack_fe01(tsk->thread.fpexc_mode);
	return put_user(val, (unsigned int __user *) adr);
}

1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
int set_endian(struct task_struct *tsk, unsigned int val)
{
	struct pt_regs *regs = tsk->thread.regs;

	if ((val == PR_ENDIAN_LITTLE && !cpu_has_feature(CPU_FTR_REAL_LE)) ||
	    (val == PR_ENDIAN_PPC_LITTLE && !cpu_has_feature(CPU_FTR_PPC_LE)))
		return -EINVAL;

	if (regs == NULL)
		return -EINVAL;

	if (val == PR_ENDIAN_BIG)
		regs->msr &= ~MSR_LE;
	else if (val == PR_ENDIAN_LITTLE || val == PR_ENDIAN_PPC_LITTLE)
		regs->msr |= MSR_LE;
	else
		return -EINVAL;

	return 0;
}

int get_endian(struct task_struct *tsk, unsigned long adr)
{
	struct pt_regs *regs = tsk->thread.regs;
	unsigned int val;

	if (!cpu_has_feature(CPU_FTR_PPC_LE) &&
	    !cpu_has_feature(CPU_FTR_REAL_LE))
		return -EINVAL;

	if (regs == NULL)
		return -EINVAL;

	if (regs->msr & MSR_LE) {
		if (cpu_has_feature(CPU_FTR_REAL_LE))
			val = PR_ENDIAN_LITTLE;
		else
			val = PR_ENDIAN_PPC_LITTLE;
	} else
		val = PR_ENDIAN_BIG;

	return put_user(val, (unsigned int __user *)adr);
}

1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
{
	tsk->thread.align_ctl = val;
	return 0;
}

int get_unalign_ctl(struct task_struct *tsk, unsigned long adr)
{
	return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr);
}

1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827
static inline int valid_irq_stack(unsigned long sp, struct task_struct *p,
				  unsigned long nbytes)
{
	unsigned long stack_page;
	unsigned long cpu = task_cpu(p);

	/*
	 * Avoid crashing if the stack has overflowed and corrupted
	 * task_cpu(p), which is in the thread_info struct.
	 */
	if (cpu < NR_CPUS && cpu_possible(cpu)) {
		stack_page = (unsigned long) hardirq_ctx[cpu];
		if (sp >= stack_page + sizeof(struct thread_struct)
		    && sp <= stack_page + THREAD_SIZE - nbytes)
			return 1;

		stack_page = (unsigned long) softirq_ctx[cpu];
		if (sp >= stack_page + sizeof(struct thread_struct)
		    && sp <= stack_page + THREAD_SIZE - nbytes)
			return 1;
	}
	return 0;
}

1828
int validate_sp(unsigned long sp, struct task_struct *p,
1829 1830
		       unsigned long nbytes)
{
A
Al Viro 已提交
1831
	unsigned long stack_page = (unsigned long)task_stack_page(p);
1832 1833 1834 1835 1836

	if (sp >= stack_page + sizeof(struct thread_struct)
	    && sp <= stack_page + THREAD_SIZE - nbytes)
		return 1;

1837
	return valid_irq_stack(sp, p, nbytes);
1838 1839
}

1840 1841
EXPORT_SYMBOL(validate_sp);

1842 1843 1844 1845 1846 1847 1848 1849 1850
unsigned long get_wchan(struct task_struct *p)
{
	unsigned long ip, sp;
	int count = 0;

	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;

	sp = p->thread.ksp;
1851
	if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD))
1852 1853 1854 1855
		return 0;

	do {
		sp = *(unsigned long *)sp;
1856
		if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD))
1857 1858
			return 0;
		if (count > 0) {
1859
			ip = ((unsigned long *)sp)[STACK_FRAME_LR_SAVE];
1860 1861 1862 1863 1864 1865
			if (!in_sched_functions(ip))
				return ip;
		}
	} while (count++ < 16);
	return 0;
}
1866

1867
static int kstack_depth_to_print = CONFIG_PRINT_STACK_DEPTH;
1868 1869 1870 1871 1872 1873

void show_stack(struct task_struct *tsk, unsigned long *stack)
{
	unsigned long sp, ip, lr, newsp;
	int count = 0;
	int firstframe = 1;
1874 1875 1876
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	int curr_frame = current->curr_ret_stack;
	extern void return_to_handler(void);
1877
	unsigned long rth = (unsigned long)return_to_handler;
1878
#endif
1879 1880 1881 1882 1883 1884

	sp = (unsigned long) stack;
	if (tsk == NULL)
		tsk = current;
	if (sp == 0) {
		if (tsk == current)
1885
			sp = current_stack_pointer();
1886 1887 1888 1889 1890 1891 1892
		else
			sp = tsk->thread.ksp;
	}

	lr = 0;
	printk("Call Trace:\n");
	do {
1893
		if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD))
1894 1895 1896 1897
			return;

		stack = (unsigned long *) sp;
		newsp = stack[0];
1898
		ip = stack[STACK_FRAME_LR_SAVE];
1899
		if (!firstframe || ip != lr) {
1900
			printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip);
1901
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1902
			if ((ip == rth) && curr_frame >= 0) {
1903
				pr_cont(" (%pS)",
1904 1905 1906 1907
				       (void *)current->ret_stack[curr_frame].ret);
				curr_frame--;
			}
#endif
1908
			if (firstframe)
1909 1910
				pr_cont(" (unreliable)");
			pr_cont("\n");
1911 1912 1913 1914 1915 1916 1917
		}
		firstframe = 0;

		/*
		 * See if this is an exception frame.
		 * We look for the "regshere" marker in the current frame.
		 */
1918 1919
		if (validate_sp(sp, tsk, STACK_INT_FRAME_SIZE)
		    && stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
1920 1921 1922
			struct pt_regs *regs = (struct pt_regs *)
				(sp + STACK_FRAME_OVERHEAD);
			lr = regs->link;
1923
			printk("--- interrupt: %lx at %pS\n    LR = %pS\n",
1924
			       regs->trap, (void *)regs->nip, (void *)lr);
1925 1926 1927 1928 1929 1930 1931
			firstframe = 1;
		}

		sp = newsp;
	} while (count++ < kstack_depth_to_print);
}

1932
#ifdef CONFIG_PPC64
1933
/* Called with hard IRQs off */
1934
void notrace __ppc64_runlatch_on(void)
1935
{
1936
	struct thread_info *ti = current_thread_info();
1937 1938
	unsigned long ctrl;

1939 1940 1941
	ctrl = mfspr(SPRN_CTRLF);
	ctrl |= CTRL_RUNLATCH;
	mtspr(SPRN_CTRLT, ctrl);
1942

1943
	ti->local_flags |= _TLF_RUNLATCH;
1944 1945
}

1946
/* Called with hard IRQs off */
1947
void notrace __ppc64_runlatch_off(void)
1948
{
1949
	struct thread_info *ti = current_thread_info();
1950 1951
	unsigned long ctrl;

1952
	ti->local_flags &= ~_TLF_RUNLATCH;
1953

1954 1955 1956
	ctrl = mfspr(SPRN_CTRLF);
	ctrl &= ~CTRL_RUNLATCH;
	mtspr(SPRN_CTRLT, ctrl);
1957
}
1958
#endif /* CONFIG_PPC64 */
1959

1960 1961 1962 1963 1964 1965
unsigned long arch_align_stack(unsigned long sp)
{
	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
		sp -= get_random_int() & ~PAGE_MASK;
	return sp & ~0xf;
}
1966 1967 1968 1969 1970 1971 1972

static inline unsigned long brk_rnd(void)
{
        unsigned long rnd = 0;

	/* 8MB for 32bit, 1GB for 64bit */
	if (is_32bit_task())
D
Daniel Cashman 已提交
1973
		rnd = (get_random_long() % (1UL<<(23-PAGE_SHIFT)));
1974
	else
D
Daniel Cashman 已提交
1975
		rnd = (get_random_long() % (1UL<<(30-PAGE_SHIFT)));
1976 1977 1978 1979 1980 1981

	return rnd << PAGE_SHIFT;
}

unsigned long arch_randomize_brk(struct mm_struct *mm)
{
1982 1983 1984
	unsigned long base = mm->brk;
	unsigned long ret;

1985
#ifdef CONFIG_PPC_STD_MMU_64
1986 1987 1988 1989 1990
	/*
	 * If we are using 1TB segments and we are allowed to randomise
	 * the heap, we can put it above 1TB so it is backed by a 1TB
	 * segment. Otherwise the heap will be in the bottom 1TB
	 * which always uses 256MB segments and this may result in a
1991 1992
	 * performance penalty. We don't need to worry about radix. For
	 * radix, mmu_highuser_ssize remains unchanged from 256MB.
1993 1994 1995 1996 1997 1998
	 */
	if (!is_32bit_task() && (mmu_highuser_ssize == MMU_SEGSIZE_1T))
		base = max_t(unsigned long, mm->brk, 1UL << SID_SHIFT_1T);
#endif

	ret = PAGE_ALIGN(base + brk_rnd());
1999 2000 2001 2002 2003 2004

	if (ret < mm->brk)
		return mm->brk;

	return ret;
}
A
Anton Blanchard 已提交
2005