tlbex.c 47.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Synthesize TLB refill handlers at runtime.
 *
8
 * Copyright (C) 2004,2005,2006 by Thiemo Seufer
9
 * Copyright (C) 2005, 2007  Maciej W. Rozycki
10 11 12 13 14 15 16 17 18 19
 * Copyright (C) 2006  Ralf Baechle (ralf@linux-mips.org)
 *
 * ... and the days got worse and worse and now you see
 * I've gone completly out of my mind.
 *
 * They're coming to take me a away haha
 * they're coming to take me a away hoho hihi haha
 * to the funny farm where code is beautiful all the time ...
 *
 * (Condolences to Napoleon XIV)
L
Linus Torvalds 已提交
20 21 22 23 24 25 26
 */

#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/init.h>

27
#include <asm/bugs.h>
L
Linus Torvalds 已提交
28 29 30 31 32
#include <asm/mmu_context.h>
#include <asm/inst.h>
#include <asm/elf.h>
#include <asm/war.h>

33
static inline int r45k_bvahwbug(void)
L
Linus Torvalds 已提交
34 35 36 37 38
{
	/* XXX: We should probe for the presence of this bug, but we don't. */
	return 0;
}

39
static inline int r4k_250MHZhwbug(void)
L
Linus Torvalds 已提交
40 41 42 43 44
{
	/* XXX: We should probe for the presence of this bug, but we don't. */
	return 0;
}

45
static inline int __maybe_unused bcm1250_m3_war(void)
L
Linus Torvalds 已提交
46 47 48 49
{
	return BCM1250_M3_WAR;
}

50
static inline int __maybe_unused r10000_llsc_war(void)
L
Linus Torvalds 已提交
51 52 53 54
{
	return R10000_LLSC_WAR;
}

55 56 57 58 59 60 61 62 63
/*
 * Found by experiment: At least some revisions of the 4kc throw under
 * some circumstances a machine check exception, triggered by invalid
 * values in the index register.  Delaying the tlbp instruction until
 * after the next branch,  plus adding an additional nop in front of
 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
 * why; it's not an issue caused by the core RTL.
 *
 */
64
static int __init m4kc_tlbp_war(void)
65 66 67 68 69
{
	return (current_cpu_data.processor_id & 0xffff00) ==
	       (PRID_COMP_MIPS | PRID_IMP_4KC);
}

L
Linus Torvalds 已提交
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
/*
 * A little micro-assembler, intended for TLB refill handler
 * synthesizing. It is intentionally kept simple, does only support
 * a subset of instructions, and does not try to hide pipeline effects
 * like branch delay slots.
 */

enum fields
{
	RS = 0x001,
	RT = 0x002,
	RD = 0x004,
	RE = 0x008,
	SIMM = 0x010,
	UIMM = 0x020,
	BIMM = 0x040,
	JIMM = 0x080,
	FUNC = 0x100,
88
	SET = 0x200
L
Linus Torvalds 已提交
89 90
};

91
#define OP_MASK		0x3f
L
Linus Torvalds 已提交
92 93 94 95 96 97 98 99 100 101 102 103 104
#define OP_SH		26
#define RS_MASK		0x1f
#define RS_SH		21
#define RT_MASK		0x1f
#define RT_SH		16
#define RD_MASK		0x1f
#define RD_SH		11
#define RE_MASK		0x1f
#define RE_SH		6
#define IMM_MASK	0xffff
#define IMM_SH		0
#define JIMM_MASK	0x3ffffff
#define JIMM_SH		0
105
#define FUNC_MASK	0x3f
L
Linus Torvalds 已提交
106
#define FUNC_SH		0
107 108
#define SET_MASK	0x7
#define SET_SH		0
L
Linus Torvalds 已提交
109 110 111 112 113 114

enum opcode {
	insn_invalid,
	insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
	insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
	insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
R
Ralf Baechle 已提交
115
	insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32,
L
Linus Torvalds 已提交
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
	insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
	insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
	insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
	insn_sra, insn_srl, insn_subu, insn_sw, insn_tlbp, insn_tlbwi,
	insn_tlbwr, insn_xor, insn_xori
};

struct insn {
	enum opcode opcode;
	u32 match;
	enum fields fields;
};

/* This macro sets the non-variable bits of an instruction. */
#define M(a, b, c, d, e, f)					\
	((a) << OP_SH						\
	 | (b) << RS_SH						\
	 | (c) << RT_SH						\
	 | (d) << RD_SH						\
	 | (e) << RE_SH						\
	 | (f) << FUNC_SH)

138
static struct insn insn_table[] __initdata = {
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
	{ insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
	{ insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD },
	{ insn_and, M(spec_op, 0, 0, 0, 0, and_op), RS | RT | RD },
	{ insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
	{ insn_beq, M(beq_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
	{ insn_beql, M(beql_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
	{ insn_bgez, M(bcond_op, 0, bgez_op, 0, 0, 0), RS | BIMM },
	{ insn_bgezl, M(bcond_op, 0, bgezl_op, 0, 0, 0), RS | BIMM },
	{ insn_bltz, M(bcond_op, 0, bltz_op, 0, 0, 0), RS | BIMM },
	{ insn_bltzl, M(bcond_op, 0, bltzl_op, 0, 0, 0), RS | BIMM },
	{ insn_bne, M(bne_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
	{ insn_daddiu, M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
	{ insn_daddu, M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD },
	{ insn_dmfc0, M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
	{ insn_dmtc0, M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
	{ insn_dsll, M(spec_op, 0, 0, 0, 0, dsll_op), RT | RD | RE },
	{ insn_dsll32, M(spec_op, 0, 0, 0, 0, dsll32_op), RT | RD | RE },
	{ insn_dsra, M(spec_op, 0, 0, 0, 0, dsra_op), RT | RD | RE },
	{ insn_dsrl, M(spec_op, 0, 0, 0, 0, dsrl_op), RT | RD | RE },
	{ insn_dsrl32, M(spec_op, 0, 0, 0, 0, dsrl32_op), RT | RD | RE },
	{ insn_dsubu, M(spec_op, 0, 0, 0, 0, dsubu_op), RS | RT | RD },
	{ insn_eret,  M(cop0_op, cop_op, 0, 0, 0, eret_op),  0 },
	{ insn_j,  M(j_op, 0, 0, 0, 0, 0),  JIMM },
	{ insn_jal,  M(jal_op, 0, 0, 0, 0, 0),  JIMM },
	{ insn_jr,  M(spec_op, 0, 0, 0, 0, jr_op),  RS },
	{ insn_ld,  M(ld_op, 0, 0, 0, 0, 0),  RS | RT | SIMM },
	{ insn_ll,  M(ll_op, 0, 0, 0, 0, 0),  RS | RT | SIMM },
	{ insn_lld,  M(lld_op, 0, 0, 0, 0, 0),  RS | RT | SIMM },
	{ insn_lui,  M(lui_op, 0, 0, 0, 0, 0),  RT | SIMM },
	{ insn_lw,  M(lw_op, 0, 0, 0, 0, 0),  RS | RT | SIMM },
	{ insn_mfc0,  M(cop0_op, mfc_op, 0, 0, 0, 0),  RT | RD | SET},
	{ insn_mtc0,  M(cop0_op, mtc_op, 0, 0, 0, 0),  RT | RD | SET},
	{ insn_ori,  M(ori_op, 0, 0, 0, 0, 0),  RS | RT | UIMM },
	{ insn_rfe,  M(cop0_op, cop_op, 0, 0, 0, rfe_op),  0 },
	{ insn_sc,  M(sc_op, 0, 0, 0, 0, 0),  RS | RT | SIMM },
	{ insn_scd,  M(scd_op, 0, 0, 0, 0, 0),  RS | RT | SIMM },
	{ insn_sd,  M(sd_op, 0, 0, 0, 0, 0),  RS | RT | SIMM },
	{ insn_sll,  M(spec_op, 0, 0, 0, 0, sll_op),  RT | RD | RE },
	{ insn_sra,  M(spec_op, 0, 0, 0, 0, sra_op),  RT | RD | RE },
	{ insn_srl,  M(spec_op, 0, 0, 0, 0, srl_op),  RT | RD | RE },
	{ insn_subu,  M(spec_op, 0, 0, 0, 0, subu_op),  RS | RT | RD },
	{ insn_sw,  M(sw_op, 0, 0, 0, 0, 0),  RS | RT | SIMM },
	{ insn_tlbp,  M(cop0_op, cop_op, 0, 0, 0, tlbp_op),  0 },
	{ insn_tlbwi,  M(cop0_op, cop_op, 0, 0, 0, tlbwi_op),  0 },
	{ insn_tlbwr,  M(cop0_op, cop_op, 0, 0, 0, tlbwr_op),  0 },
	{ insn_xor,  M(spec_op, 0, 0, 0, 0, xor_op),  RS | RT | RD },
	{ insn_xori,  M(xori_op, 0, 0, 0, 0, 0),  RS | RT | UIMM },
L
Linus Torvalds 已提交
186 187 188 189 190
	{ insn_invalid, 0, 0 }
};

#undef M

191
static u32 __init build_rs(u32 arg)
L
Linus Torvalds 已提交
192 193 194 195 196 197 198
{
	if (arg & ~RS_MASK)
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	return (arg & RS_MASK) << RS_SH;
}

199
static u32 __init build_rt(u32 arg)
L
Linus Torvalds 已提交
200 201 202 203 204 205 206
{
	if (arg & ~RT_MASK)
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	return (arg & RT_MASK) << RT_SH;
}

207
static u32 __init build_rd(u32 arg)
L
Linus Torvalds 已提交
208 209 210 211 212 213 214
{
	if (arg & ~RD_MASK)
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	return (arg & RD_MASK) << RD_SH;
}

215
static u32 __init build_re(u32 arg)
L
Linus Torvalds 已提交
216 217 218 219 220 221 222
{
	if (arg & ~RE_MASK)
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	return (arg & RE_MASK) << RE_SH;
}

223
static u32 __init build_simm(s32 arg)
L
Linus Torvalds 已提交
224 225 226 227 228 229 230
{
	if (arg > 0x7fff || arg < -0x8000)
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	return arg & 0xffff;
}

231
static u32 __init build_uimm(u32 arg)
L
Linus Torvalds 已提交
232 233 234 235 236 237 238
{
	if (arg & ~IMM_MASK)
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	return arg & IMM_MASK;
}

239
static u32 __init build_bimm(s32 arg)
L
Linus Torvalds 已提交
240 241 242 243 244 245 246 247 248 249
{
	if (arg > 0x1ffff || arg < -0x20000)
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	if (arg & 0x3)
		printk(KERN_WARNING "Invalid TLB synthesizer branch target\n");

	return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
}

250
static u32 __init build_jimm(u32 arg)
L
Linus Torvalds 已提交
251 252 253 254 255 256 257
{
	if (arg & ~((JIMM_MASK) << 2))
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	return (arg >> 2) & JIMM_MASK;
}

258
static u32 __init build_func(u32 arg)
L
Linus Torvalds 已提交
259 260 261 262 263 264 265
{
	if (arg & ~FUNC_MASK)
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	return arg & FUNC_MASK;
}

266
static u32 __init build_set(u32 arg)
267 268 269 270 271 272 273
{
	if (arg & ~SET_MASK)
		printk(KERN_WARNING "TLB synthesizer field overflow\n");

	return arg & SET_MASK;
}

L
Linus Torvalds 已提交
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
/*
 * The order of opcode arguments is implicitly left to right,
 * starting with RS and ending with FUNC or IMM.
 */
static void __init build_insn(u32 **buf, enum opcode opc, ...)
{
	struct insn *ip = NULL;
	unsigned int i;
	va_list ap;
	u32 op;

	for (i = 0; insn_table[i].opcode != insn_invalid; i++)
		if (insn_table[i].opcode == opc) {
			ip = &insn_table[i];
			break;
		}

291
	if (!ip || (opc == insn_daddiu && r4k_daddiu_bug()))
L
Linus Torvalds 已提交
292 293 294 295 296 297 298 299 300 301 302 303 304
		panic("Unsupported TLB synthesizer instruction %d", opc);

	op = ip->match;
	va_start(ap, opc);
	if (ip->fields & RS) op |= build_rs(va_arg(ap, u32));
	if (ip->fields & RT) op |= build_rt(va_arg(ap, u32));
	if (ip->fields & RD) op |= build_rd(va_arg(ap, u32));
	if (ip->fields & RE) op |= build_re(va_arg(ap, u32));
	if (ip->fields & SIMM) op |= build_simm(va_arg(ap, s32));
	if (ip->fields & UIMM) op |= build_uimm(va_arg(ap, u32));
	if (ip->fields & BIMM) op |= build_bimm(va_arg(ap, s32));
	if (ip->fields & JIMM) op |= build_jimm(va_arg(ap, u32));
	if (ip->fields & FUNC) op |= build_func(va_arg(ap, u32));
305
	if (ip->fields & SET) op |= build_set(va_arg(ap, u32));
L
Linus Torvalds 已提交
306 307 308 309 310 311 312
	va_end(ap);

	**buf = op;
	(*buf)++;
}

#define I_u1u2u3(op)						\
313
	static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
L
Linus Torvalds 已提交
314 315 316 317 318 319
	 	unsigned int b, unsigned int c)			\
	{							\
		build_insn(buf, insn##op, a, b, c);		\
	}

#define I_u2u1u3(op)						\
320
	static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
L
Linus Torvalds 已提交
321 322 323 324 325 326
	 	unsigned int b, unsigned int c)			\
	{							\
		build_insn(buf, insn##op, b, a, c);		\
	}

#define I_u3u1u2(op)						\
327
	static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
L
Linus Torvalds 已提交
328 329 330 331 332 333
	 	unsigned int b, unsigned int c)			\
	{							\
		build_insn(buf, insn##op, b, c, a);		\
	}

#define I_u1u2s3(op)						\
334
	static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
L
Linus Torvalds 已提交
335 336 337 338 339 340
	 	unsigned int b, signed int c)			\
	{							\
		build_insn(buf, insn##op, a, b, c);		\
	}

#define I_u2s3u1(op)						\
341
	static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
L
Linus Torvalds 已提交
342 343 344 345 346 347
	 	signed int b, unsigned int c)			\
	{							\
		build_insn(buf, insn##op, c, a, b);		\
	}

#define I_u2u1s3(op)						\
348
	static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
L
Linus Torvalds 已提交
349 350 351 352 353 354
	 	unsigned int b, signed int c)			\
	{							\
		build_insn(buf, insn##op, b, a, c);		\
	}

#define I_u1u2(op)						\
355
	static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
L
Linus Torvalds 已提交
356 357 358 359 360 361
	 	unsigned int b)					\
	{							\
		build_insn(buf, insn##op, a, b);		\
	}

#define I_u1s2(op)						\
362
	static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
L
Linus Torvalds 已提交
363 364 365 366 367 368
	 	signed int b)					\
	{							\
		build_insn(buf, insn##op, a, b);		\
	}

#define I_u1(op)						\
369
	static void __init __maybe_unused i##op(u32 **buf, unsigned int a) \
L
Linus Torvalds 已提交
370 371 372 373 374
	{							\
		build_insn(buf, insn##op, a);			\
	}

#define I_0(op)							\
375
	static void __init __maybe_unused i##op(u32 **buf)	\
L
Linus Torvalds 已提交
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
	{							\
		build_insn(buf, insn##op);			\
	}

I_u2u1s3(_addiu);
I_u3u1u2(_addu);
I_u2u1u3(_andi);
I_u3u1u2(_and);
I_u1u2s3(_beq);
I_u1u2s3(_beql);
I_u1s2(_bgez);
I_u1s2(_bgezl);
I_u1s2(_bltz);
I_u1s2(_bltzl);
I_u1u2s3(_bne);
391 392
I_u1u2u3(_dmfc0);
I_u1u2u3(_dmtc0);
L
Linus Torvalds 已提交
393 394 395 396 397 398
I_u2u1s3(_daddiu);
I_u3u1u2(_daddu);
I_u2u1u3(_dsll);
I_u2u1u3(_dsll32);
I_u2u1u3(_dsra);
I_u2u1u3(_dsrl);
R
Ralf Baechle 已提交
399
I_u2u1u3(_dsrl32);
L
Linus Torvalds 已提交
400 401 402 403 404 405 406 407 408 409
I_u3u1u2(_dsubu);
I_0(_eret);
I_u1(_j);
I_u1(_jal);
I_u1(_jr);
I_u2s3u1(_ld);
I_u2s3u1(_ll);
I_u2s3u1(_lld);
I_u1s2(_lui);
I_u2s3u1(_lw);
410 411
I_u1u2u3(_mfc0);
I_u1u2u3(_mtc0);
L
Linus Torvalds 已提交
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
I_u2u1u3(_ori);
I_0(_rfe);
I_u2s3u1(_sc);
I_u2s3u1(_scd);
I_u2s3u1(_sd);
I_u2u1u3(_sll);
I_u2u1u3(_sra);
I_u2u1u3(_srl);
I_u3u1u2(_subu);
I_u2s3u1(_sw);
I_0(_tlbp);
I_0(_tlbwi);
I_0(_tlbwr);
I_u3u1u2(_xor)
I_u2u1u3(_xori);

/*
 * handling labels
 */

enum label_id {
	label_invalid,
	label_second_part,
	label_leave,
436 437 438
#ifdef MODULE_START
	label_module_alloc,
#endif
L
Linus Torvalds 已提交
439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
	label_vmalloc,
	label_vmalloc_done,
	label_tlbw_hazard,
	label_split,
	label_nopage_tlbl,
	label_nopage_tlbs,
	label_nopage_tlbm,
	label_smp_pgtable_change,
	label_r3000_write_probe_fail,
};

struct label {
	u32 *addr;
	enum label_id lab;
};

455
static void __init build_label(struct label **lab, u32 *addr,
L
Linus Torvalds 已提交
456 457 458 459 460 461 462 463
			       enum label_id l)
{
	(*lab)->addr = addr;
	(*lab)->lab = l;
	(*lab)++;
}

#define L_LA(lb)						\
464
	static inline void __init l##lb(struct label **lab, u32 *addr) \
L
Linus Torvalds 已提交
465 466 467 468 469 470
	{							\
		build_label(lab, addr, label##lb);		\
	}

L_LA(_second_part)
L_LA(_leave)
471 472 473
#ifdef MODULE_START
L_LA(_module_alloc)
#endif
L
Linus Torvalds 已提交
474 475 476 477 478 479 480 481 482 483 484
L_LA(_vmalloc)
L_LA(_vmalloc_done)
L_LA(_tlbw_hazard)
L_LA(_split)
L_LA(_nopage_tlbl)
L_LA(_nopage_tlbs)
L_LA(_nopage_tlbm)
L_LA(_smp_pgtable_change)
L_LA(_r3000_write_probe_fail)

/* convenience macros for instructions */
485
#ifdef CONFIG_64BIT
L
Linus Torvalds 已提交
486 487 488 489 490
# define i_LW(buf, rs, rt, off) i_ld(buf, rs, rt, off)
# define i_SW(buf, rs, rt, off) i_sd(buf, rs, rt, off)
# define i_SLL(buf, rs, rt, sh) i_dsll(buf, rs, rt, sh)
# define i_SRA(buf, rs, rt, sh) i_dsra(buf, rs, rt, sh)
# define i_SRL(buf, rs, rt, sh) i_dsrl(buf, rs, rt, sh)
491 492
# define i_MFC0(buf, rt, rd...) i_dmfc0(buf, rt, rd)
# define i_MTC0(buf, rt, rd...) i_dmtc0(buf, rt, rd)
L
Linus Torvalds 已提交
493 494 495 496 497 498 499 500 501 502 503
# define i_ADDIU(buf, rs, rt, val) i_daddiu(buf, rs, rt, val)
# define i_ADDU(buf, rs, rt, rd) i_daddu(buf, rs, rt, rd)
# define i_SUBU(buf, rs, rt, rd) i_dsubu(buf, rs, rt, rd)
# define i_LL(buf, rs, rt, off) i_lld(buf, rs, rt, off)
# define i_SC(buf, rs, rt, off) i_scd(buf, rs, rt, off)
#else
# define i_LW(buf, rs, rt, off) i_lw(buf, rs, rt, off)
# define i_SW(buf, rs, rt, off) i_sw(buf, rs, rt, off)
# define i_SLL(buf, rs, rt, sh) i_sll(buf, rs, rt, sh)
# define i_SRA(buf, rs, rt, sh) i_sra(buf, rs, rt, sh)
# define i_SRL(buf, rs, rt, sh) i_srl(buf, rs, rt, sh)
504 505
# define i_MFC0(buf, rt, rd...) i_mfc0(buf, rt, rd)
# define i_MTC0(buf, rt, rd...) i_mtc0(buf, rt, rd)
L
Linus Torvalds 已提交
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
# define i_ADDIU(buf, rs, rt, val) i_addiu(buf, rs, rt, val)
# define i_ADDU(buf, rs, rt, rd) i_addu(buf, rs, rt, rd)
# define i_SUBU(buf, rs, rt, rd) i_subu(buf, rs, rt, rd)
# define i_LL(buf, rs, rt, off) i_ll(buf, rs, rt, off)
# define i_SC(buf, rs, rt, off) i_sc(buf, rs, rt, off)
#endif

#define i_b(buf, off) i_beq(buf, 0, 0, off)
#define i_beqz(buf, rs, off) i_beq(buf, rs, 0, off)
#define i_beqzl(buf, rs, off) i_beql(buf, rs, 0, off)
#define i_bnez(buf, rs, off) i_bne(buf, rs, 0, off)
#define i_bnezl(buf, rs, off) i_bnel(buf, rs, 0, off)
#define i_move(buf, a, b) i_ADDU(buf, a, 0, b)
#define i_nop(buf) i_sll(buf, 0, 0, 0)
#define i_ssnop(buf) i_sll(buf, 0, 0, 1)
#define i_ehb(buf) i_sll(buf, 0, 0, 3)

523
static int __init __maybe_unused in_compat_space_p(long addr)
L
Linus Torvalds 已提交
524 525
{
	/* Is this address in 32bit compat space? */
526
#ifdef CONFIG_64BIT
R
Ralf Baechle 已提交
527
	return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L);
528 529 530
#else
	return 1;
#endif
L
Linus Torvalds 已提交
531 532
}

533
static int __init __maybe_unused rel_highest(long val)
L
Linus Torvalds 已提交
534
{
535
#ifdef CONFIG_64BIT
L
Linus Torvalds 已提交
536
	return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
537 538 539
#else
	return 0;
#endif
L
Linus Torvalds 已提交
540 541
}

542
static int __init __maybe_unused rel_higher(long val)
L
Linus Torvalds 已提交
543
{
544
#ifdef CONFIG_64BIT
L
Linus Torvalds 已提交
545
	return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
546 547
#else
	return 0;
L
Linus Torvalds 已提交
548
#endif
549
}
L
Linus Torvalds 已提交
550

551
static int __init rel_hi(long val)
L
Linus Torvalds 已提交
552 553 554 555
{
	return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
}

556
static int __init rel_lo(long val)
L
Linus Torvalds 已提交
557 558 559 560
{
	return ((val & 0xffff) ^ 0x8000) - 0x8000;
}

561
static void __init i_LA_mostly(u32 **buf, unsigned int rs, long addr)
L
Linus Torvalds 已提交
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
{
	if (!in_compat_space_p(addr)) {
		i_lui(buf, rs, rel_highest(addr));
		if (rel_higher(addr))
			i_daddiu(buf, rs, rs, rel_higher(addr));
		if (rel_hi(addr)) {
			i_dsll(buf, rs, rs, 16);
			i_daddiu(buf, rs, rs, rel_hi(addr));
			i_dsll(buf, rs, rs, 16);
		} else
			i_dsll32(buf, rs, rs, 0);
	} else
		i_lui(buf, rs, rel_hi(addr));
}

577
static void __init __maybe_unused i_LA(u32 **buf, unsigned int rs, long addr)
L
Linus Torvalds 已提交
578 579
{
	i_LA_mostly(buf, rs, addr);
580 581 582 583 584 585
	if (rel_lo(addr)) {
		if (!in_compat_space_p(addr))
			i_daddiu(buf, rs, rs, rel_lo(addr));
		else
			i_addiu(buf, rs, rs, rel_lo(addr));
	}
L
Linus Torvalds 已提交
586 587 588 589 590 591 592 593 594 595 596 597
}

/*
 * handle relocations
 */

struct reloc {
	u32 *addr;
	unsigned int type;
	enum label_id lab;
};

598
static void __init r_mips_pc16(struct reloc **rel, u32 *addr,
L
Linus Torvalds 已提交
599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622
			       enum label_id l)
{
	(*rel)->addr = addr;
	(*rel)->type = R_MIPS_PC16;
	(*rel)->lab = l;
	(*rel)++;
}

static inline void __resolve_relocs(struct reloc *rel, struct label *lab)
{
	long laddr = (long)lab->addr;
	long raddr = (long)rel->addr;

	switch (rel->type) {
	case R_MIPS_PC16:
		*rel->addr |= build_bimm(laddr - (raddr + 4));
		break;

	default:
		panic("Unsupported TLB synthesizer relocation %d",
		      rel->type);
	}
}

623
static void __init resolve_relocs(struct reloc *rel, struct label *lab)
L
Linus Torvalds 已提交
624 625 626 627 628 629 630 631 632
{
	struct label *l;

	for (; rel->lab != label_invalid; rel++)
		for (l = lab; l->lab != label_invalid; l++)
			if (rel->lab == l->lab)
				__resolve_relocs(rel, l);
}

633
static void __init move_relocs(struct reloc *rel, u32 *first, u32 *end,
L
Linus Torvalds 已提交
634 635 636 637 638 639 640
			       long off)
{
	for (; rel->lab != label_invalid; rel++)
		if (rel->addr >= first && rel->addr < end)
			rel->addr += off;
}

641
static void __init move_labels(struct label *lab, u32 *first, u32 *end,
L
Linus Torvalds 已提交
642 643 644 645 646 647 648
			       long off)
{
	for (; lab->lab != label_invalid; lab++)
		if (lab->addr >= first && lab->addr < end)
			lab->addr += off;
}

649
static void __init copy_handler(struct reloc *rel, struct label *lab,
L
Linus Torvalds 已提交
650 651 652 653 654 655 656 657 658 659
				u32 *first, u32 *end, u32 *target)
{
	long off = (long)(target - first);

	memcpy(target, first, (end - first) * sizeof(u32));

	move_relocs(rel, first, end, off);
	move_labels(lab, first, end, off);
}

660
static int __init __maybe_unused insn_has_bdelay(struct reloc *rel,
661
						       u32 *addr)
L
Linus Torvalds 已提交
662 663 664 665 666 667 668 669 670 671 672 673
{
	for (; rel->lab != label_invalid; rel++) {
		if (rel->addr == addr
		    && (rel->type == R_MIPS_PC16
			|| rel->type == R_MIPS_26))
			return 1;
	}

	return 0;
}

/* convenience functions for labeled branches */
674
static void __init __maybe_unused
675
	il_bltz(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
L
Linus Torvalds 已提交
676 677 678 679 680
{
	r_mips_pc16(r, *p, l);
	i_bltz(p, reg, 0);
}

681 682
static void __init __maybe_unused il_b(u32 **p, struct reloc **r,
					     enum label_id l)
L
Linus Torvalds 已提交
683 684 685 686 687
{
	r_mips_pc16(r, *p, l);
	i_b(p, 0);
}

688
static void __init il_beqz(u32 **p, struct reloc **r, unsigned int reg,
L
Linus Torvalds 已提交
689 690 691 692 693 694
		    enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_beqz(p, reg, 0);
}

695
static void __init __maybe_unused
L
Linus Torvalds 已提交
696 697 698 699 700 701
il_beqzl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_beqzl(p, reg, 0);
}

702
static void __init il_bnez(u32 **p, struct reloc **r, unsigned int reg,
L
Linus Torvalds 已提交
703 704 705 706 707 708
		    enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_bnez(p, reg, 0);
}

709
static void __init il_bgezl(u32 **p, struct reloc **r, unsigned int reg,
L
Linus Torvalds 已提交
710 711 712 713 714 715
		     enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_bgezl(p, reg, 0);
}

716
static void __init __maybe_unused
717 718 719 720 721 722
il_bgez(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_bgez(p, reg, 0);
}

723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
/*
 * For debug purposes.
 */
static inline void dump_handler(const u32 *handler, int count)
{
	int i;

	pr_debug("\t.set push\n");
	pr_debug("\t.set noreorder\n");

	for (i = 0; i < count; i++)
		pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]);

	pr_debug("\t.set pop\n");
}

L
Linus Torvalds 已提交
739 740 741 742 743
/* The only general purpose registers allowed in TLB handlers. */
#define K0		26
#define K1		27

/* Some CP0 registers */
744 745 746 747 748 749 750 751 752
#define C0_INDEX	0, 0
#define C0_ENTRYLO0	2, 0
#define C0_TCBIND	2, 2
#define C0_ENTRYLO1	3, 0
#define C0_CONTEXT	4, 0
#define C0_BADVADDR	8, 0
#define C0_ENTRYHI	10, 0
#define C0_EPC		14, 0
#define C0_XCONTEXT	20, 0
L
Linus Torvalds 已提交
753

754
#ifdef CONFIG_64BIT
L
Linus Torvalds 已提交
755 756 757 758 759 760 761 762 763 764 765 766 767
# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_XCONTEXT)
#else
# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_CONTEXT)
#endif

/* The worst case length of the handler is around 18 instructions for
 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
 * Maximum space available is 32 instructions for R3000 and 64
 * instructions for R4000.
 *
 * We deliberately chose a buffer size of 128, so we won't scribble
 * over anything important on overflow before we panic.
 */
768
static u32 tlb_handler[128] __initdata;
L
Linus Torvalds 已提交
769 770

/* simply assume worst case size for labels and relocs */
771 772
static struct label labels[128] __initdata;
static struct reloc relocs[128] __initdata;
L
Linus Torvalds 已提交
773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805

/*
 * The R3000 TLB handler is simple.
 */
static void __init build_r3000_tlb_refill_handler(void)
{
	long pgdc = (long)pgd_current;
	u32 *p;

	memset(tlb_handler, 0, sizeof(tlb_handler));
	p = tlb_handler;

	i_mfc0(&p, K0, C0_BADVADDR);
	i_lui(&p, K1, rel_hi(pgdc)); /* cp0 delay */
	i_lw(&p, K1, rel_lo(pgdc), K1);
	i_srl(&p, K0, K0, 22); /* load delay */
	i_sll(&p, K0, K0, 2);
	i_addu(&p, K1, K1, K0);
	i_mfc0(&p, K0, C0_CONTEXT);
	i_lw(&p, K1, 0, K1); /* cp0 delay */
	i_andi(&p, K0, K0, 0xffc); /* load delay */
	i_addu(&p, K1, K1, K0);
	i_lw(&p, K0, 0, K1);
	i_nop(&p); /* load delay */
	i_mtc0(&p, K0, C0_ENTRYLO0);
	i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
	i_tlbwr(&p); /* cp0 delay */
	i_jr(&p, K1);
	i_rfe(&p); /* branch delay */

	if (p > tlb_handler + 32)
		panic("TLB refill handler space exceeded");

806 807
	pr_info("Synthesized TLB refill handler (%u instructions).\n",
		(unsigned int)(p - tlb_handler));
L
Linus Torvalds 已提交
808

809
	memcpy((void *)ebase, tlb_handler, 0x80);
810 811

	dump_handler((u32 *)ebase, 32);
L
Linus Torvalds 已提交
812 813 814 815 816 817 818 819 820
}

/*
 * The R4000 TLB handler is much more complicated. We have two
 * consecutive handler areas with 32 instructions space each.
 * Since they aren't used at the same time, we can overflow in the
 * other one.To keep things simple, we first assume linear space,
 * then we relocate it to the final handler layout as needed.
 */
821
static u32 final_handler[64] __initdata;
L
Linus Torvalds 已提交
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844

/*
 * Hazards
 *
 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
 * 2. A timing hazard exists for the TLBP instruction.
 *
 *      stalling_instruction
 *      TLBP
 *
 * The JTLB is being read for the TLBP throughout the stall generated by the
 * previous instruction. This is not really correct as the stalling instruction
 * can modify the address used to access the JTLB.  The failure symptom is that
 * the TLBP instruction will use an address created for the stalling instruction
 * and not the address held in C0_ENHI and thus report the wrong results.
 *
 * The software work-around is to not allow the instruction preceding the TLBP
 * to stall - make it an NOP or some other instruction guaranteed not to stall.
 *
 * Errata 2 will not be fixed.  This errata is also on the R5000.
 *
 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
 */
845
static void __init __maybe_unused build_tlb_probe_entry(u32 **p)
L
Linus Torvalds 已提交
846
{
847
	switch (current_cpu_type()) {
848 849
	/* Found by experiment: R4600 v2.0 needs this, too.  */
	case CPU_R4600:
L
Linus Torvalds 已提交
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868
	case CPU_R5000:
	case CPU_R5000A:
	case CPU_NEVADA:
		i_nop(p);
		i_tlbp(p);
		break;

	default:
		i_tlbp(p);
		break;
	}
}

/*
 * Write random or indexed TLB entry, and care about the hazards from
 * the preceeding mtc0 and for the following eret.
 */
enum tlb_write_entry { tlb_random, tlb_indexed };

869
static void __init build_tlb_write_entry(u32 **p, struct label **l,
L
Linus Torvalds 已提交
870 871 872 873 874 875 876 877 878 879
					 struct reloc **r,
					 enum tlb_write_entry wmode)
{
	void(*tlbw)(u32 **) = NULL;

	switch (wmode) {
	case tlb_random: tlbw = i_tlbwr; break;
	case tlb_indexed: tlbw = i_tlbwi; break;
	}

880 881 882 883 884 885
	if (cpu_has_mips_r2) {
		i_ehb(p);
		tlbw(p);
		return;
	}

886
	switch (current_cpu_type()) {
L
Linus Torvalds 已提交
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
	case CPU_R4000PC:
	case CPU_R4000SC:
	case CPU_R4000MC:
	case CPU_R4400PC:
	case CPU_R4400SC:
	case CPU_R4400MC:
		/*
		 * This branch uses up a mtc0 hazard nop slot and saves
		 * two nops after the tlbw instruction.
		 */
		il_bgezl(p, r, 0, label_tlbw_hazard);
		tlbw(p);
		l_tlbw_hazard(l, *p);
		i_nop(p);
		break;

	case CPU_R4600:
	case CPU_R4700:
	case CPU_R5000:
	case CPU_R5000A:
907 908 909 910 911 912
		i_nop(p);
		tlbw(p);
		i_nop(p);
		break;

	case CPU_R4300:
L
Linus Torvalds 已提交
913 914 915 916 917 918
	case CPU_5KC:
	case CPU_TX49XX:
	case CPU_AU1000:
	case CPU_AU1100:
	case CPU_AU1500:
	case CPU_AU1550:
P
Pete Popov 已提交
919
	case CPU_AU1200:
920
	case CPU_PR4450:
L
Linus Torvalds 已提交
921 922 923 924 925 926
		i_nop(p);
		tlbw(p);
		break;

	case CPU_R10000:
	case CPU_R12000:
K
Kumba 已提交
927
	case CPU_R14000:
L
Linus Torvalds 已提交
928 929
	case CPU_4KC:
	case CPU_SB1:
A
Andrew Isaacson 已提交
930
	case CPU_SB1A:
L
Linus Torvalds 已提交
931 932 933
	case CPU_4KSC:
	case CPU_20KC:
	case CPU_25KF:
934 935
	case CPU_BCM3302:
	case CPU_BCM4710:
936
	case CPU_LOONGSON2:
937 938
		if (m4kc_tlbp_war())
			i_nop(p);
L
Linus Torvalds 已提交
939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
		tlbw(p);
		break;

	case CPU_NEVADA:
		i_nop(p); /* QED specifies 2 nops hazard */
		/*
		 * This branch uses up a mtc0 hazard nop slot and saves
		 * a nop after the tlbw instruction.
		 */
		il_bgezl(p, r, 0, label_tlbw_hazard);
		tlbw(p);
		l_tlbw_hazard(l, *p);
		break;

	case CPU_RM7000:
		i_nop(p);
		i_nop(p);
		i_nop(p);
		i_nop(p);
		tlbw(p);
		break;

	case CPU_RM9000:
		/*
		 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
		 * use of the JTLB for instructions should not occur for 4
		 * cpu cycles and use for data translations should not occur
		 * for 3 cpu cycles.
		 */
		i_ssnop(p);
		i_ssnop(p);
		i_ssnop(p);
		i_ssnop(p);
		tlbw(p);
		i_ssnop(p);
		i_ssnop(p);
		i_ssnop(p);
		i_ssnop(p);
		break;

	case CPU_VR4111:
	case CPU_VR4121:
	case CPU_VR4122:
	case CPU_VR4181:
	case CPU_VR4181A:
		i_nop(p);
		i_nop(p);
		tlbw(p);
		i_nop(p);
		i_nop(p);
		break;

	case CPU_VR4131:
	case CPU_VR4133:
993
	case CPU_R5432:
L
Linus Torvalds 已提交
994 995 996 997 998 999 1000 1001 1002 1003 1004 1005
		i_nop(p);
		i_nop(p);
		tlbw(p);
		break;

	default:
		panic("No TLB refill handler yet (CPU type: %d)",
		      current_cpu_data.cputype);
		break;
	}
}

1006
#ifdef CONFIG_64BIT
L
Linus Torvalds 已提交
1007 1008 1009 1010
/*
 * TMP and PTR are scratch.
 * TMP will be clobbered, PTR will hold the pmd entry.
 */
1011
static void __init
L
Linus Torvalds 已提交
1012 1013 1014 1015 1016 1017 1018 1019 1020
build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
		 unsigned int tmp, unsigned int ptr)
{
	long pgdc = (long)pgd_current;

	/*
	 * The vmalloc handling is not in the hotpath.
	 */
	i_dmfc0(p, tmp, C0_BADVADDR);
1021 1022 1023
#ifdef MODULE_START
	il_bltz(p, r, tmp, label_module_alloc);
#else
L
Linus Torvalds 已提交
1024
	il_bltz(p, r, tmp, label_vmalloc);
1025
#endif
L
Linus Torvalds 已提交
1026 1027 1028
	/* No i_nop needed here, since the next insn doesn't touch TMP. */

#ifdef CONFIG_SMP
1029 1030 1031 1032 1033 1034 1035
# ifdef  CONFIG_MIPS_MT_SMTC
	/*
	 * SMTC uses TCBind value as "CPU" index
	 */
	i_mfc0(p, ptr, C0_TCBIND);
	i_dsrl(p, ptr, ptr, 19);
# else
L
Linus Torvalds 已提交
1036
	/*
1037
	 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
L
Linus Torvalds 已提交
1038 1039
	 * stored in CONTEXT.
	 */
1040 1041
	i_dmfc0(p, ptr, C0_CONTEXT);
	i_dsrl(p, ptr, ptr, 23);
1042
#endif
1043 1044 1045 1046
	i_LA_mostly(p, tmp, pgdc);
	i_daddu(p, ptr, ptr, tmp);
	i_dmfc0(p, tmp, C0_BADVADDR);
	i_ld(p, ptr, rel_lo(pgdc), ptr);
L
Linus Torvalds 已提交
1047 1048 1049 1050 1051 1052
#else
	i_LA_mostly(p, ptr, pgdc);
	i_ld(p, ptr, rel_lo(pgdc), ptr);
#endif

	l_vmalloc_done(l, *p);
R
Ralf Baechle 已提交
1053 1054 1055 1056 1057 1058

	if (PGDIR_SHIFT - 3 < 32)		/* get pgd offset in bytes */
		i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
	else
		i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);

L
Linus Torvalds 已提交
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
	i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
	i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
	i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
	i_ld(p, ptr, 0, ptr); /* get pmd pointer */
	i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
	i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
	i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
}

/*
 * BVADDR is the faulting address, PTR is scratch.
 * PTR will hold the pgd for vmalloc.
 */
1072
static void __init
L
Linus Torvalds 已提交
1073 1074 1075 1076 1077
build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
			unsigned int bvaddr, unsigned int ptr)
{
	long swpd = (long)swapper_pg_dir;

1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
#ifdef MODULE_START
	long modd = (long)module_pg_dir;

	l_module_alloc(l, *p);
	/*
	 * Assumption:
	 * VMALLOC_START >= 0xc000000000000000UL
	 * MODULE_START >= 0xe000000000000000UL
	 */
	i_SLL(p, ptr, bvaddr, 2);
	il_bgez(p, r, ptr, label_vmalloc);

	if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START)) {
		i_lui(p, ptr, rel_hi(MODULE_START)); /* delay slot */
	} else {
		/* unlikely configuration */
		i_nop(p); /* delay slot */
		i_LA(p, ptr, MODULE_START);
	}
	i_dsubu(p, bvaddr, bvaddr, ptr);

	if (in_compat_space_p(modd) && !rel_lo(modd)) {
		il_b(p, r, label_vmalloc_done);
		i_lui(p, ptr, rel_hi(modd));
	} else {
		i_LA_mostly(p, ptr, modd);
		il_b(p, r, label_vmalloc_done);
1105 1106 1107 1108
		if (in_compat_space_p(modd))
			i_addiu(p, ptr, ptr, rel_lo(modd));
		else
			i_daddiu(p, ptr, ptr, rel_lo(modd));
1109 1110 1111 1112 1113 1114 1115 1116 1117
	}

	l_vmalloc(l, *p);
	if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START) &&
	    MODULE_START << 32 == VMALLOC_START)
		i_dsll32(p, ptr, ptr, 0);	/* typical case */
	else
		i_LA(p, ptr, VMALLOC_START);
#else
L
Linus Torvalds 已提交
1118 1119
	l_vmalloc(l, *p);
	i_LA(p, ptr, VMALLOC_START);
1120
#endif
L
Linus Torvalds 已提交
1121 1122 1123 1124 1125 1126 1127 1128
	i_dsubu(p, bvaddr, bvaddr, ptr);

	if (in_compat_space_p(swpd) && !rel_lo(swpd)) {
		il_b(p, r, label_vmalloc_done);
		i_lui(p, ptr, rel_hi(swpd));
	} else {
		i_LA_mostly(p, ptr, swpd);
		il_b(p, r, label_vmalloc_done);
1129 1130 1131 1132
		if (in_compat_space_p(swpd))
			i_addiu(p, ptr, ptr, rel_lo(swpd));
		else
			i_daddiu(p, ptr, ptr, rel_lo(swpd));
L
Linus Torvalds 已提交
1133 1134 1135
	}
}

1136
#else /* !CONFIG_64BIT */
L
Linus Torvalds 已提交
1137 1138 1139 1140 1141

/*
 * TMP and PTR are scratch.
 * TMP will be clobbered, PTR will hold the pgd entry.
 */
1142
static void __init __maybe_unused
L
Linus Torvalds 已提交
1143 1144 1145 1146 1147 1148
build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
{
	long pgdc = (long)pgd_current;

	/* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
#ifdef CONFIG_SMP
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
#ifdef  CONFIG_MIPS_MT_SMTC
	/*
	 * SMTC uses TCBind value as "CPU" index
	 */
	i_mfc0(p, ptr, C0_TCBIND);
	i_LA_mostly(p, tmp, pgdc);
	i_srl(p, ptr, ptr, 19);
#else
	/*
	 * smp_processor_id() << 3 is stored in CONTEXT.
         */
L
Linus Torvalds 已提交
1160 1161 1162
	i_mfc0(p, ptr, C0_CONTEXT);
	i_LA_mostly(p, tmp, pgdc);
	i_srl(p, ptr, ptr, 23);
1163
#endif
L
Linus Torvalds 已提交
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
	i_addu(p, ptr, tmp, ptr);
#else
	i_LA_mostly(p, ptr, pgdc);
#endif
	i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
	i_lw(p, ptr, rel_lo(pgdc), ptr);
	i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
	i_sll(p, tmp, tmp, PGD_T_LOG2);
	i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
}

1175
#endif /* !CONFIG_64BIT */
L
Linus Torvalds 已提交
1176

1177
static void __init build_adjust_context(u32 **p, unsigned int ctx)
L
Linus Torvalds 已提交
1178
{
R
Ralf Baechle 已提交
1179
	unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
L
Linus Torvalds 已提交
1180 1181
	unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);

1182
	switch (current_cpu_type()) {
L
Linus Torvalds 已提交
1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202
	case CPU_VR41XX:
	case CPU_VR4111:
	case CPU_VR4121:
	case CPU_VR4122:
	case CPU_VR4131:
	case CPU_VR4181:
	case CPU_VR4181A:
	case CPU_VR4133:
		shift += 2;
		break;

	default:
		break;
	}

	if (shift)
		i_SRL(p, ctx, ctx, shift);
	i_andi(p, ctx, ctx, mask);
}

1203
static void __init build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
L
Linus Torvalds 已提交
1204 1205 1206 1207 1208 1209 1210 1211
{
	/*
	 * Bug workaround for the Nevada. It seems as if under certain
	 * circumstances the move from cp0_context might produce a
	 * bogus result when the mfc0 instruction and its consumer are
	 * in a different cacheline or a load instruction, probably any
	 * memory reference, is between them.
	 */
1212
	switch (current_cpu_type()) {
L
Linus Torvalds 已提交
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
	case CPU_NEVADA:
		i_LW(p, ptr, 0, ptr);
		GET_CONTEXT(p, tmp); /* get context reg */
		break;

	default:
		GET_CONTEXT(p, tmp); /* get context reg */
		i_LW(p, ptr, 0, ptr);
		break;
	}

	build_adjust_context(p, tmp);
	i_ADDU(p, ptr, ptr, tmp); /* add in offset */
}

1228
static void __init build_update_entries(u32 **p, unsigned int tmp,
L
Linus Torvalds 已提交
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
					unsigned int ptep)
{
	/*
	 * 64bit address support (36bit on a 32bit CPU) in a 32bit
	 * Kernel is a special case. Only a few CPUs use it.
	 */
#ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits) {
		i_ld(p, tmp, 0, ptep); /* get even pte */
		i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
		i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
		i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
		i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
		i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
	} else {
		int pte_off_even = sizeof(pte_t) / 2;
		int pte_off_odd = pte_off_even + sizeof(pte_t);

		/* The pte entries are pre-shifted */
		i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
		i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
		i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
		i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
	}
#else
	i_LW(p, tmp, 0, ptep); /* get even pte */
	i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
	if (r45k_bvahwbug())
		build_tlb_probe_entry(p);
	i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
	if (r4k_250MHZhwbug())
		i_mtc0(p, 0, C0_ENTRYLO0);
	i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
	i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
	if (r45k_bvahwbug())
		i_mfc0(p, tmp, C0_INDEX);
	if (r4k_250MHZhwbug())
		i_mtc0(p, 0, C0_ENTRYLO1);
	i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
#endif
}

static void __init build_r4000_tlb_refill_handler(void)
{
	u32 *p = tlb_handler;
	struct label *l = labels;
	struct reloc *r = relocs;
	u32 *f;
	unsigned int final_len;

	memset(tlb_handler, 0, sizeof(tlb_handler));
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));
	memset(final_handler, 0, sizeof(final_handler));

	/*
	 * create the plain linear handler
	 */
	if (bcm1250_m3_war()) {
		i_MFC0(&p, K0, C0_BADVADDR);
		i_MFC0(&p, K1, C0_ENTRYHI);
		i_xor(&p, K0, K0, K1);
		i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
		il_bnez(&p, &r, K0, label_leave);
		/* No need for i_nop */
	}

1296
#ifdef CONFIG_64BIT
L
Linus Torvalds 已提交
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
	build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
#else
	build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
#endif

	build_get_ptep(&p, K0, K1);
	build_update_entries(&p, K0, K1);
	build_tlb_write_entry(&p, &l, &r, tlb_random);
	l_leave(&l, p);
	i_eret(&p); /* return from trap */

1308
#ifdef CONFIG_64BIT
L
Linus Torvalds 已提交
1309 1310 1311 1312 1313 1314 1315
	build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
#endif

	/*
	 * Overflow check: For the 64bit handler, we need at least one
	 * free instruction slot for the wrap-around branch. In worst
	 * case, if the intended insertion point is a delay slot, we
M
Matt LaPlante 已提交
1316
	 * need three, with the second nop'ed and the third being
L
Linus Torvalds 已提交
1317 1318
	 * unused.
	 */
1319 1320
	/* Loongson2 ebase is different than r4k, we have more space */
#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
L
Linus Torvalds 已提交
1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
	if ((p - tlb_handler) > 64)
		panic("TLB refill handler space exceeded");
#else
	if (((p - tlb_handler) > 63)
	    || (((p - tlb_handler) > 61)
		&& insn_has_bdelay(relocs, tlb_handler + 29)))
		panic("TLB refill handler space exceeded");
#endif

	/*
	 * Now fold the handler in the TLB refill handler space.
	 */
1333
#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
L
Linus Torvalds 已提交
1334 1335 1336 1337
	f = final_handler;
	/* Simplest case, just copy the handler. */
	copy_handler(relocs, labels, tlb_handler, p, f);
	final_len = p - tlb_handler;
1338
#else /* CONFIG_64BIT */
L
Linus Torvalds 已提交
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
	f = final_handler + 32;
	if ((p - tlb_handler) <= 32) {
		/* Just copy the handler. */
		copy_handler(relocs, labels, tlb_handler, p, f);
		final_len = p - tlb_handler;
	} else {
		u32 *split = tlb_handler + 30;

		/*
		 * Find the split point.
		 */
		if (insn_has_bdelay(relocs, split - 1))
			split--;

		/* Copy first part of the handler. */
		copy_handler(relocs, labels, tlb_handler, split, f);
		f += split - tlb_handler;

		/* Insert branch. */
		l_split(&l, final_handler);
		il_b(&f, &r, label_split);
		if (insn_has_bdelay(relocs, split))
			i_nop(&f);
		else {
			copy_handler(relocs, labels, split, split + 1, f);
			move_labels(labels, f, f + 1, -1);
			f++;
			split++;
		}

		/* Copy the rest of the handler. */
		copy_handler(relocs, labels, split, p, final_handler);
		final_len = (f - (final_handler + 32)) + (p - split);
	}
1373
#endif /* CONFIG_64BIT */
L
Linus Torvalds 已提交
1374 1375

	resolve_relocs(relocs, labels);
1376 1377
	pr_info("Synthesized TLB refill handler (%u instructions).\n",
		final_len);
L
Linus Torvalds 已提交
1378

1379
	memcpy((void *)ebase, final_handler, 0x100);
1380 1381

	dump_handler((u32 *)ebase, 64);
L
Linus Torvalds 已提交
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
}

/*
 * TLB load/store/modify handlers.
 *
 * Only the fastpath gets synthesized at runtime, the slowpath for
 * do_page_fault remains normal asm.
 */
extern void tlb_do_page_fault_0(void);
extern void tlb_do_page_fault_1(void);

/*
 * 128 instructions for the fastpath handler is generous and should
 * never be exceeded.
 */
#define FASTPATH_SIZE 128

1399 1400 1401
u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
L
Linus Torvalds 已提交
1402 1403

static void __init
1404
iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr)
L
Linus Torvalds 已提交
1405 1406 1407 1408
{
#ifdef CONFIG_SMP
# ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits)
1409
		i_lld(p, pte, 0, ptr);
L
Linus Torvalds 已提交
1410 1411
	else
# endif
1412
		i_LL(p, pte, 0, ptr);
L
Linus Torvalds 已提交
1413 1414 1415
#else
# ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits)
1416
		i_ld(p, pte, 0, ptr);
L
Linus Torvalds 已提交
1417 1418
	else
# endif
1419
		i_LW(p, pte, 0, ptr);
L
Linus Torvalds 已提交
1420 1421 1422 1423
#endif
}

static void __init
1424 1425
iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr,
	unsigned int mode)
L
Linus Torvalds 已提交
1426
{
1427 1428 1429 1430 1431
#ifdef CONFIG_64BIT_PHYS_ADDR
	unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
#endif

	i_ori(p, pte, pte, mode);
L
Linus Torvalds 已提交
1432 1433 1434
#ifdef CONFIG_SMP
# ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits)
1435
		i_scd(p, pte, 0, ptr);
L
Linus Torvalds 已提交
1436 1437
	else
# endif
1438
		i_SC(p, pte, 0, ptr);
L
Linus Torvalds 已提交
1439 1440 1441 1442 1443 1444 1445 1446 1447 1448

	if (r10000_llsc_war())
		il_beqzl(p, r, pte, label_smp_pgtable_change);
	else
		il_beqz(p, r, pte, label_smp_pgtable_change);

# ifdef CONFIG_64BIT_PHYS_ADDR
	if (!cpu_has_64bits) {
		/* no i_nop needed */
		i_ll(p, pte, sizeof(pte_t) / 2, ptr);
1449
		i_ori(p, pte, pte, hwmode);
L
Linus Torvalds 已提交
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461
		i_sc(p, pte, sizeof(pte_t) / 2, ptr);
		il_beqz(p, r, pte, label_smp_pgtable_change);
		/* no i_nop needed */
		i_lw(p, pte, 0, ptr);
	} else
		i_nop(p);
# else
	i_nop(p);
# endif
#else
# ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits)
1462
		i_sd(p, pte, 0, ptr);
L
Linus Torvalds 已提交
1463 1464
	else
# endif
1465
		i_SW(p, pte, 0, ptr);
L
Linus Torvalds 已提交
1466 1467 1468 1469

# ifdef CONFIG_64BIT_PHYS_ADDR
	if (!cpu_has_64bits) {
		i_lw(p, pte, sizeof(pte_t) / 2, ptr);
1470
		i_ori(p, pte, pte, hwmode);
L
Linus Torvalds 已提交
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489
		i_sw(p, pte, sizeof(pte_t) / 2, ptr);
		i_lw(p, pte, 0, ptr);
	}
# endif
#endif
}

/*
 * Check if PTE is present, if not then jump to LABEL. PTR points to
 * the page table where this PTE is located, PTE will be re-loaded
 * with it's original value.
 */
static void __init
build_pte_present(u32 **p, struct label **l, struct reloc **r,
		  unsigned int pte, unsigned int ptr, enum label_id lid)
{
	i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
	i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
	il_bnez(p, r, pte, lid);
1490
	iPTE_LW(p, l, pte, ptr);
L
Linus Torvalds 已提交
1491 1492 1493 1494 1495 1496 1497
}

/* Make PTE valid, store result in PTR. */
static void __init
build_make_valid(u32 **p, struct reloc **r, unsigned int pte,
		 unsigned int ptr)
{
1498 1499 1500
	unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;

	iPTE_SW(p, r, pte, ptr, mode);
L
Linus Torvalds 已提交
1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
}

/*
 * Check if PTE can be written to, if not branch to LABEL. Regardless
 * restore PTE with value from PTR when done.
 */
static void __init
build_pte_writable(u32 **p, struct label **l, struct reloc **r,
		   unsigned int pte, unsigned int ptr, enum label_id lid)
{
	i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
	i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
	il_bnez(p, r, pte, lid);
1514
	iPTE_LW(p, l, pte, ptr);
L
Linus Torvalds 已提交
1515 1516 1517 1518 1519 1520 1521 1522 1523
}

/* Make PTE writable, update software status bits as well, then store
 * at PTR.
 */
static void __init
build_make_write(u32 **p, struct reloc **r, unsigned int pte,
		 unsigned int ptr)
{
1524 1525 1526 1527
	unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
			     | _PAGE_DIRTY);

	iPTE_SW(p, r, pte, ptr, mode);
L
Linus Torvalds 已提交
1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
}

/*
 * Check if PTE can be modified, if not branch to LABEL. Regardless
 * restore PTE with value from PTR when done.
 */
static void __init
build_pte_modifiable(u32 **p, struct label **l, struct reloc **r,
		     unsigned int pte, unsigned int ptr, enum label_id lid)
{
	i_andi(p, pte, pte, _PAGE_WRITE);
	il_beqz(p, r, pte, lid);
1540
	iPTE_LW(p, l, pte, ptr);
L
Linus Torvalds 已提交
1541 1542 1543 1544 1545 1546
}

/*
 * R3000 style TLB load/store/modify handlers.
 */

1547 1548 1549 1550
/*
 * This places the pte into ENTRYLO0 and writes it with tlbwi.
 * Then it returns.
 */
L
Linus Torvalds 已提交
1551
static void __init
1552
build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
L
Linus Torvalds 已提交
1553
{
1554 1555 1556 1557 1558
	i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
	i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
	i_tlbwi(p);
	i_jr(p, tmp);
	i_rfe(p); /* branch delay */
L
Linus Torvalds 已提交
1559 1560 1561
}

/*
1562 1563 1564 1565
 * This places the pte into ENTRYLO0 and writes it with tlbwi
 * or tlbwr as appropriate.  This is because the index register
 * may have the probe fail bit set as a result of a trap on a
 * kseg2 access, i.e. without refill.  Then it returns.
L
Linus Torvalds 已提交
1566 1567
 */
static void __init
1568 1569
build_r3000_tlb_reload_write(u32 **p, struct label **l, struct reloc **r,
			     unsigned int pte, unsigned int tmp)
L
Linus Torvalds 已提交
1570 1571
{
	i_mfc0(p, tmp, C0_INDEX);
1572 1573 1574 1575 1576 1577
	i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
	il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
	i_mfc0(p, tmp, C0_EPC); /* branch delay */
	i_tlbwi(p); /* cp0 delay */
	i_jr(p, tmp);
	i_rfe(p); /* branch delay */
L
Linus Torvalds 已提交
1578
	l_r3000_write_probe_fail(l, *p);
1579 1580 1581
	i_tlbwr(p); /* cp0 delay */
	i_jr(p, tmp);
	i_rfe(p); /* branch delay */
L
Linus Torvalds 已提交
1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600
}

static void __init
build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
				   unsigned int ptr)
{
	long pgdc = (long)pgd_current;

	i_mfc0(p, pte, C0_BADVADDR);
	i_lui(p, ptr, rel_hi(pgdc)); /* cp0 delay */
	i_lw(p, ptr, rel_lo(pgdc), ptr);
	i_srl(p, pte, pte, 22); /* load delay */
	i_sll(p, pte, pte, 2);
	i_addu(p, ptr, ptr, pte);
	i_mfc0(p, pte, C0_CONTEXT);
	i_lw(p, ptr, 0, ptr); /* cp0 delay */
	i_andi(p, pte, pte, 0xffc); /* load delay */
	i_addu(p, ptr, ptr, pte);
	i_lw(p, pte, 0, ptr);
1601
	i_tlbp(p); /* load delay */
L
Linus Torvalds 已提交
1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
}

static void __init build_r3000_tlb_load_handler(void)
{
	u32 *p = handle_tlbl;
	struct label *l = labels;
	struct reloc *r = relocs;

	memset(handle_tlbl, 0, sizeof(handle_tlbl));
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));

	build_r3000_tlbchange_handler_head(&p, K0, K1);
	build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
1616
	i_nop(&p); /* load delay */
L
Linus Torvalds 已提交
1617
	build_make_valid(&p, &r, K0, K1);
1618
	build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
L
Linus Torvalds 已提交
1619 1620 1621 1622 1623 1624 1625 1626 1627

	l_nopage_tlbl(&l, p);
	i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
	i_nop(&p);

	if ((p - handle_tlbl) > FASTPATH_SIZE)
		panic("TLB load handler fastpath space exceeded");

	resolve_relocs(relocs, labels);
1628 1629
	pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
		(unsigned int)(p - handle_tlbl));
L
Linus Torvalds 已提交
1630

1631
	dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
L
Linus Torvalds 已提交
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
}

static void __init build_r3000_tlb_store_handler(void)
{
	u32 *p = handle_tlbs;
	struct label *l = labels;
	struct reloc *r = relocs;

	memset(handle_tlbs, 0, sizeof(handle_tlbs));
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));

	build_r3000_tlbchange_handler_head(&p, K0, K1);
	build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1646
	i_nop(&p); /* load delay */
L
Linus Torvalds 已提交
1647
	build_make_write(&p, &r, K0, K1);
1648
	build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
L
Linus Torvalds 已提交
1649 1650 1651 1652 1653 1654 1655 1656 1657

	l_nopage_tlbs(&l, p);
	i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
	i_nop(&p);

	if ((p - handle_tlbs) > FASTPATH_SIZE)
		panic("TLB store handler fastpath space exceeded");

	resolve_relocs(relocs, labels);
1658 1659
	pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
		(unsigned int)(p - handle_tlbs));
L
Linus Torvalds 已提交
1660

1661
	dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
L
Linus Torvalds 已提交
1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675
}

static void __init build_r3000_tlb_modify_handler(void)
{
	u32 *p = handle_tlbm;
	struct label *l = labels;
	struct reloc *r = relocs;

	memset(handle_tlbm, 0, sizeof(handle_tlbm));
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));

	build_r3000_tlbchange_handler_head(&p, K0, K1);
	build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1676
	i_nop(&p); /* load delay */
L
Linus Torvalds 已提交
1677
	build_make_write(&p, &r, K0, K1);
1678
	build_r3000_pte_reload_tlbwi(&p, K0, K1);
L
Linus Torvalds 已提交
1679 1680 1681 1682 1683 1684 1685 1686 1687

	l_nopage_tlbm(&l, p);
	i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
	i_nop(&p);

	if ((p - handle_tlbm) > FASTPATH_SIZE)
		panic("TLB modify handler fastpath space exceeded");

	resolve_relocs(relocs, labels);
1688 1689
	pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
		(unsigned int)(p - handle_tlbm));
L
Linus Torvalds 已提交
1690

1691
	dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
L
Linus Torvalds 已提交
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
}

/*
 * R4000 style TLB load/store/modify handlers.
 */
static void __init
build_r4000_tlbchange_handler_head(u32 **p, struct label **l,
				   struct reloc **r, unsigned int pte,
				   unsigned int ptr)
{
1702
#ifdef CONFIG_64BIT
L
Linus Torvalds 已提交
1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
	build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
#else
	build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
#endif

	i_MFC0(p, pte, C0_BADVADDR);
	i_LW(p, ptr, 0, ptr);
	i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
	i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
	i_ADDU(p, ptr, ptr, pte);

#ifdef CONFIG_SMP
	l_smp_pgtable_change(l, *p);
# endif
1717
	iPTE_LW(p, l, pte, ptr); /* get even pte */
1718 1719
	if (!m4kc_tlbp_war())
		build_tlb_probe_entry(p);
L
Linus Torvalds 已提交
1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
}

static void __init
build_r4000_tlbchange_handler_tail(u32 **p, struct label **l,
				   struct reloc **r, unsigned int tmp,
				   unsigned int ptr)
{
	i_ori(p, ptr, ptr, sizeof(pte_t));
	i_xori(p, ptr, ptr, sizeof(pte_t));
	build_update_entries(p, tmp, ptr);
	build_tlb_write_entry(p, l, r, tlb_indexed);
	l_leave(l, *p);
	i_eret(p); /* return from trap */

1734
#ifdef CONFIG_64BIT
L
Linus Torvalds 已提交
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759
	build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
#endif
}

static void __init build_r4000_tlb_load_handler(void)
{
	u32 *p = handle_tlbl;
	struct label *l = labels;
	struct reloc *r = relocs;

	memset(handle_tlbl, 0, sizeof(handle_tlbl));
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));

	if (bcm1250_m3_war()) {
		i_MFC0(&p, K0, C0_BADVADDR);
		i_MFC0(&p, K1, C0_ENTRYHI);
		i_xor(&p, K0, K0, K1);
		i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
		il_bnez(&p, &r, K0, label_leave);
		/* No need for i_nop */
	}

	build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
	build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
1760 1761
	if (m4kc_tlbp_war())
		build_tlb_probe_entry(&p);
L
Linus Torvalds 已提交
1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772
	build_make_valid(&p, &r, K0, K1);
	build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);

	l_nopage_tlbl(&l, p);
	i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
	i_nop(&p);

	if ((p - handle_tlbl) > FASTPATH_SIZE)
		panic("TLB load handler fastpath space exceeded");

	resolve_relocs(relocs, labels);
1773 1774
	pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
		(unsigned int)(p - handle_tlbl));
L
Linus Torvalds 已提交
1775

1776
	dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
L
Linus Torvalds 已提交
1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790
}

static void __init build_r4000_tlb_store_handler(void)
{
	u32 *p = handle_tlbs;
	struct label *l = labels;
	struct reloc *r = relocs;

	memset(handle_tlbs, 0, sizeof(handle_tlbs));
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));

	build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
	build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1791 1792
	if (m4kc_tlbp_war())
		build_tlb_probe_entry(&p);
L
Linus Torvalds 已提交
1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
	build_make_write(&p, &r, K0, K1);
	build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);

	l_nopage_tlbs(&l, p);
	i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
	i_nop(&p);

	if ((p - handle_tlbs) > FASTPATH_SIZE)
		panic("TLB store handler fastpath space exceeded");

	resolve_relocs(relocs, labels);
1804 1805
	pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
		(unsigned int)(p - handle_tlbs));
L
Linus Torvalds 已提交
1806

1807
	dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
L
Linus Torvalds 已提交
1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821
}

static void __init build_r4000_tlb_modify_handler(void)
{
	u32 *p = handle_tlbm;
	struct label *l = labels;
	struct reloc *r = relocs;

	memset(handle_tlbm, 0, sizeof(handle_tlbm));
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));

	build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
	build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1822 1823
	if (m4kc_tlbp_war())
		build_tlb_probe_entry(&p);
L
Linus Torvalds 已提交
1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835
	/* Present and writable bits set, set accessed and dirty bits. */
	build_make_write(&p, &r, K0, K1);
	build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);

	l_nopage_tlbm(&l, p);
	i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
	i_nop(&p);

	if ((p - handle_tlbm) > FASTPATH_SIZE)
		panic("TLB modify handler fastpath space exceeded");

	resolve_relocs(relocs, labels);
1836 1837 1838
	pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
		(unsigned int)(p - handle_tlbm));

1839
	dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
L
Linus Torvalds 已提交
1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850
}

void __init build_tlb_refill_handler(void)
{
	/*
	 * The refill handler is generated per-CPU, multi-node systems
	 * may have local storage for it. The other handlers are only
	 * needed once.
	 */
	static int run_once = 0;

1851
	switch (current_cpu_type()) {
L
Linus Torvalds 已提交
1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886
	case CPU_R2000:
	case CPU_R3000:
	case CPU_R3000A:
	case CPU_R3081E:
	case CPU_TX3912:
	case CPU_TX3922:
	case CPU_TX3927:
		build_r3000_tlb_refill_handler();
		if (!run_once) {
			build_r3000_tlb_load_handler();
			build_r3000_tlb_store_handler();
			build_r3000_tlb_modify_handler();
			run_once++;
		}
		break;

	case CPU_R6000:
	case CPU_R6000A:
		panic("No R6000 TLB refill handler yet");
		break;

	case CPU_R8000:
		panic("No R8000 TLB refill handler yet");
		break;

	default:
		build_r4000_tlb_refill_handler();
		if (!run_once) {
			build_r4000_tlb_load_handler();
			build_r4000_tlb_store_handler();
			build_r4000_tlb_modify_handler();
			run_once++;
		}
	}
}
1887 1888 1889 1890 1891 1892 1893 1894 1895 1896

void __init flush_tlb_handlers(void)
{
	flush_icache_range((unsigned long)handle_tlbl,
			   (unsigned long)handle_tlbl + sizeof(handle_tlbl));
	flush_icache_range((unsigned long)handle_tlbs,
			   (unsigned long)handle_tlbs + sizeof(handle_tlbs));
	flush_icache_range((unsigned long)handle_tlbm,
			   (unsigned long)handle_tlbm + sizeof(handle_tlbm));
}