alternative.c 11.6 KB
Newer Older
G
Gerd Hoffmann 已提交
1
#include <linux/module.h>
A
Al Viro 已提交
2
#include <linux/sched.h>
G
Gerd Hoffmann 已提交
3 4
#include <linux/spinlock.h>
#include <linux/list.h>
5 6 7
#include <linux/kprobes.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
G
Gerd Hoffmann 已提交
8 9
#include <asm/alternative.h>
#include <asm/sections.h>
10
#include <asm/pgtable.h>
11 12
#include <asm/mce.h>
#include <asm/nmi.h>
D
Dave Jones 已提交
13
#include <asm/vsyscall.h>
G
Gerd Hoffmann 已提交
14

15 16
#define MAX_PATCH_LEN (255-1)

17 18
#ifdef CONFIG_HOTPLUG_CPU
static int smp_alt_once;
G
Gerd Hoffmann 已提交
19

20 21 22 23 24
static int __init bootonly(char *str)
{
	smp_alt_once = 1;
	return 1;
}
25
__setup("smp-alt-boot", bootonly);
26 27 28 29 30
#else
#define smp_alt_once 1
#endif

static int debug_alternative;
31

32 33 34 35 36 37 38
static int __init debug_alt(char *str)
{
	debug_alternative = 1;
	return 1;
}
__setup("debug-alternative", debug_alt);

39 40
static int noreplace_smp;

41 42 43 44 45 46 47
static int __init setup_noreplace_smp(char *str)
{
	noreplace_smp = 1;
	return 1;
}
__setup("noreplace-smp", setup_noreplace_smp);

48 49 50 51 52 53 54 55 56 57
#ifdef CONFIG_PARAVIRT
static int noreplace_paravirt = 0;

static int __init setup_noreplace_paravirt(char *str)
{
	noreplace_paravirt = 1;
	return 1;
}
__setup("noreplace-paravirt", setup_noreplace_paravirt);
#endif
58

59 60 61 62
#define DPRINTK(fmt, args...) if (debug_alternative) \
	printk(KERN_DEBUG fmt, args)

#ifdef GENERIC_NOP1
G
Gerd Hoffmann 已提交
63 64 65
/* Use inline assembly to define this because the nops are defined
   as inline assembly strings in the include files and we cannot
   get them easily into strings. */
J
Jan Beulich 已提交
66
asm("\t.section .rodata, \"a\"\nintelnops: "
G
Gerd Hoffmann 已提交
67 68
	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	GENERIC_NOP7 GENERIC_NOP8);
J
Jan Beulich 已提交
69 70
extern const unsigned char intelnops[];
static const unsigned char *const intel_nops[ASM_NOP_MAX+1] = {
G
Gerd Hoffmann 已提交
71 72 73 74 75 76 77 78 79 80
	NULL,
	intelnops,
	intelnops + 1,
	intelnops + 1 + 2,
	intelnops + 1 + 2 + 3,
	intelnops + 1 + 2 + 3 + 4,
	intelnops + 1 + 2 + 3 + 4 + 5,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
81 82 83
#endif

#ifdef K8_NOP1
J
Jan Beulich 已提交
84
asm("\t.section .rodata, \"a\"\nk8nops: "
85 86
	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	K8_NOP7 K8_NOP8);
J
Jan Beulich 已提交
87 88
extern const unsigned char k8nops[];
static const unsigned char *const k8_nops[ASM_NOP_MAX+1] = {
G
Gerd Hoffmann 已提交
89 90 91 92 93 94 95 96 97 98
	NULL,
	k8nops,
	k8nops + 1,
	k8nops + 1 + 2,
	k8nops + 1 + 2 + 3,
	k8nops + 1 + 2 + 3 + 4,
	k8nops + 1 + 2 + 3 + 4 + 5,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
99 100 101
#endif

#ifdef K7_NOP1
J
Jan Beulich 已提交
102
asm("\t.section .rodata, \"a\"\nk7nops: "
103 104
	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	K7_NOP7 K7_NOP8);
J
Jan Beulich 已提交
105 106
extern const unsigned char k7nops[];
static const unsigned char *const k7_nops[ASM_NOP_MAX+1] = {
G
Gerd Hoffmann 已提交
107 108 109 110 111 112 113 114 115 116
	NULL,
	k7nops,
	k7nops + 1,
	k7nops + 1 + 2,
	k7nops + 1 + 2 + 3,
	k7nops + 1 + 2 + 3 + 4,
	k7nops + 1 + 2 + 3 + 4 + 5,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
117 118
#endif

119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
#ifdef P6_NOP1
asm("\t.section .rodata, \"a\"\np6nops: "
	P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6
	P6_NOP7 P6_NOP8);
extern const unsigned char p6nops[];
static const unsigned char *const p6_nops[ASM_NOP_MAX+1] = {
	NULL,
	p6nops,
	p6nops + 1,
	p6nops + 1 + 2,
	p6nops + 1 + 2 + 3,
	p6nops + 1 + 2 + 3 + 4,
	p6nops + 1 + 2 + 3 + 4 + 5,
	p6nops + 1 + 2 + 3 + 4 + 5 + 6,
	p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

137 138 139
#ifdef CONFIG_X86_64

extern char __vsyscall_0;
J
Jan Beulich 已提交
140
static inline const unsigned char*const * find_nop_table(void)
141
{
142 143
	return boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
	       boot_cpu_data.x86 < 6 ? k8_nops : p6_nops;
144 145 146 147
}

#else /* CONFIG_X86_64 */

J
Jan Beulich 已提交
148
static const struct nop {
G
Gerd Hoffmann 已提交
149
	int cpuid;
J
Jan Beulich 已提交
150
	const unsigned char *const *noptable;
G
Gerd Hoffmann 已提交
151 152 153
} noptypes[] = {
	{ X86_FEATURE_K8, k8_nops },
	{ X86_FEATURE_K7, k7_nops },
154 155
	{ X86_FEATURE_P4, p6_nops },
	{ X86_FEATURE_P3, p6_nops },
G
Gerd Hoffmann 已提交
156 157 158
	{ -1, NULL }
};

J
Jan Beulich 已提交
159
static const unsigned char*const * find_nop_table(void)
G
Gerd Hoffmann 已提交
160
{
J
Jan Beulich 已提交
161
	const unsigned char *const *noptable = intel_nops;
G
Gerd Hoffmann 已提交
162 163 164 165 166 167 168 169 170 171 172
	int i;

	for (i = 0; noptypes[i].cpuid >= 0; i++) {
		if (boot_cpu_has(noptypes[i].cpuid)) {
			noptable = noptypes[i].noptable;
			break;
		}
	}
	return noptable;
}

173 174
#endif /* CONFIG_X86_64 */

175 176
/* Use this to add nops to a buffer, then text_poke the whole buffer. */
static void add_nops(void *insns, unsigned int len)
177
{
J
Jan Beulich 已提交
178
	const unsigned char *const *noptable = find_nop_table();
179 180 181 182 183

	while (len > 0) {
		unsigned int noplen = len;
		if (noplen > ASM_NOP_MAX)
			noplen = ASM_NOP_MAX;
184
		memcpy(insns, noptable[noplen], noplen);
185 186 187 188 189
		insns += noplen;
		len -= noplen;
	}
}

190 191 192
extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern u8 *__smp_locks[], *__smp_locks_end[];

G
Gerd Hoffmann 已提交
193 194 195 196 197 198 199 200 201
/* Replace instructions with better alternatives for this CPU type.
   This runs before SMP is initialized to avoid SMP problems with
   self modifying code. This implies that assymetric systems where
   APs have less capabilities than the boot processor are not handled.
   Tough. Make sure you disable such features by hand. */

void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
{
	struct alt_instr *a;
202
	char insnbuf[MAX_PATCH_LEN];
G
Gerd Hoffmann 已提交
203 204 205

	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {
206
		u8 *instr = a->instr;
G
Gerd Hoffmann 已提交
207
		BUG_ON(a->replacementlen > a->instrlen);
208
		BUG_ON(a->instrlen > sizeof(insnbuf));
G
Gerd Hoffmann 已提交
209 210
		if (!boot_cpu_has(a->cpuid))
			continue;
211 212 213 214 215 216 217 218
#ifdef CONFIG_X86_64
		/* vsyscall code is not mapped yet. resolve it manually. */
		if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
			instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
			DPRINTK("%s: vsyscall fixup: %p => %p\n",
				__FUNCTION__, a->instr, instr);
		}
#endif
219 220 221 222
		memcpy(insnbuf, a->replacement, a->replacementlen);
		add_nops(insnbuf + a->replacementlen,
			 a->instrlen - a->replacementlen);
		text_poke(instr, insnbuf, a->instrlen);
G
Gerd Hoffmann 已提交
223 224 225
	}
}

226 227
#ifdef CONFIG_SMP

G
Gerd Hoffmann 已提交
228 229 230 231 232 233 234 235 236
static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
237
		text_poke(*ptr, ((unsigned char []){0xf0}), 1); /* add lock prefix */
G
Gerd Hoffmann 已提交
238 239 240 241 242 243
	};
}

static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;
244
	char insn[1];
G
Gerd Hoffmann 已提交
245

246 247 248
	if (noreplace_smp)
		return;

249
	add_nops(insn, 1);
G
Gerd Hoffmann 已提交
250 251 252 253 254
	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
255
		text_poke(*ptr, insn, 1);
G
Gerd Hoffmann 已提交
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
	};
}

struct smp_alt_module {
	/* what is this ??? */
	struct module	*mod;
	char		*name;

	/* ptrs to lock prefixes */
	u8		**locks;
	u8		**locks_end;

	/* .text segment, needed to avoid patching init code ;) */
	u8		*text;
	u8		*text_end;

	struct list_head next;
};
static LIST_HEAD(smp_alt_modules);
static DEFINE_SPINLOCK(smp_alt);

void alternatives_smp_module_add(struct module *mod, char *name,
				 void *locks, void *locks_end,
				 void *text,  void *text_end)
{
	struct smp_alt_module *smp;
	unsigned long flags;

284 285 286
	if (noreplace_smp)
		return;

G
Gerd Hoffmann 已提交
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
	if (smp_alt_once) {
		if (boot_cpu_has(X86_FEATURE_UP))
			alternatives_smp_unlock(locks, locks_end,
						text, text_end);
		return;
	}

	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	if (NULL == smp)
		return; /* we'll run the (safe but slow) SMP code then ... */

	smp->mod	= mod;
	smp->name	= name;
	smp->locks	= locks;
	smp->locks_end	= locks_end;
	smp->text	= text;
	smp->text_end	= text_end;
	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
		__FUNCTION__, smp->locks, smp->locks_end,
		smp->text, smp->text_end, smp->name);

	spin_lock_irqsave(&smp_alt, flags);
	list_add_tail(&smp->next, &smp_alt_modules);
	if (boot_cpu_has(X86_FEATURE_UP))
		alternatives_smp_unlock(smp->locks, smp->locks_end,
					smp->text, smp->text_end);
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_module_del(struct module *mod)
{
	struct smp_alt_module *item;
	unsigned long flags;

321
	if (smp_alt_once || noreplace_smp)
G
Gerd Hoffmann 已提交
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
		return;

	spin_lock_irqsave(&smp_alt, flags);
	list_for_each_entry(item, &smp_alt_modules, next) {
		if (mod != item->mod)
			continue;
		list_del(&item->next);
		spin_unlock_irqrestore(&smp_alt, flags);
		DPRINTK("%s: %s\n", __FUNCTION__, item->name);
		kfree(item);
		return;
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_switch(int smp)
{
	struct smp_alt_module *mod;
	unsigned long flags;

342 343 344 345 346 347 348 349 350 351
#ifdef CONFIG_LOCKDEP
	/*
	 * A not yet fixed binutils section handling bug prevents
	 * alternatives-replacement from working reliably, so turn
	 * it off:
	 */
	printk("lockdep: not fixing up alternatives.\n");
	return;
#endif

352
	if (noreplace_smp || smp_alt_once)
G
Gerd Hoffmann 已提交
353 354 355 356 357 358
		return;
	BUG_ON(!smp && (num_online_cpus() > 1));

	spin_lock_irqsave(&smp_alt, flags);
	if (smp) {
		printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
359 360
		clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
		clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
G
Gerd Hoffmann 已提交
361 362 363 364 365
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_lock(mod->locks, mod->locks_end,
					      mod->text, mod->text_end);
	} else {
		printk(KERN_INFO "SMP alternatives: switching to UP code\n");
366 367
		set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
		set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
G
Gerd Hoffmann 已提交
368 369 370 371 372 373 374
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_unlock(mod->locks, mod->locks_end,
						mod->text, mod->text_end);
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

375 376
#endif

377
#ifdef CONFIG_PARAVIRT
378 379
void apply_paravirt(struct paravirt_patch_site *start,
		    struct paravirt_patch_site *end)
380
{
381
	struct paravirt_patch_site *p;
382
	char insnbuf[MAX_PATCH_LEN];
383

384 385 386
	if (noreplace_paravirt)
		return;

387 388 389
	for (p = start; p < end; p++) {
		unsigned int used;

390
		BUG_ON(p->len > MAX_PATCH_LEN);
391 392
		/* prep the buffer with the original instructions */
		memcpy(insnbuf, p->instr, p->len);
393 394
		used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf,
					 (unsigned long)p->instr, p->len);
395

396 397
		BUG_ON(used > p->len);

398
		/* Pad the rest with nops */
399 400
		add_nops(insnbuf + used, p->len - used);
		text_poke(p->instr, insnbuf, p->len);
401 402
	}
}
403
extern struct paravirt_patch_site __start_parainstructions[],
404 405 406
	__stop_parainstructions[];
#endif	/* CONFIG_PARAVIRT */

G
Gerd Hoffmann 已提交
407 408
void __init alternative_instructions(void)
{
409 410
	unsigned long flags;

411 412 413 414
	/* The patching is not fully atomic, so try to avoid local interruptions
	   that might execute the to be patched code.
	   Other CPUs are not running. */
	stop_nmi();
415
#ifdef CONFIG_X86_MCE
416 417 418
	stop_mce();
#endif

419
	local_irq_save(flags);
G
Gerd Hoffmann 已提交
420 421 422 423 424 425 426 427 428 429
	apply_alternatives(__alt_instructions, __alt_instructions_end);

	/* switch to patch-once-at-boottime-only mode and free the
	 * tables in case we know the number of CPUs will never ever
	 * change */
#ifdef CONFIG_HOTPLUG_CPU
	if (num_possible_cpus() < 2)
		smp_alt_once = 1;
#endif

430
#ifdef CONFIG_SMP
G
Gerd Hoffmann 已提交
431 432 433
	if (smp_alt_once) {
		if (1 == num_possible_cpus()) {
			printk(KERN_INFO "SMP alternatives: switching to UP code\n");
434 435 436
			set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
			set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);

G
Gerd Hoffmann 已提交
437 438 439 440 441 442 443 444 445
			alternatives_smp_unlock(__smp_locks, __smp_locks_end,
						_text, _etext);
		}
	} else {
		alternatives_smp_module_add(NULL, "core kernel",
					    __smp_locks, __smp_locks_end,
					    _text, _etext);
		alternatives_smp_switch(0);
	}
446
#endif
447
 	apply_paravirt(__parainstructions, __parainstructions_end);
448
	local_irq_restore(flags);
449

450 451 452 453 454
	if (smp_alt_once)
		free_init_pages("SMP alternatives",
				(unsigned long)__smp_locks,
				(unsigned long)__smp_locks_end);

455
	restart_nmi();
456
#ifdef CONFIG_X86_MCE
457 458
	restart_mce();
#endif
G
Gerd Hoffmann 已提交
459
}
460 461 462 463 464 465 466 467 468

/*
 * Warning:
 * When you use this code to patch more than one byte of an instruction
 * you need to make sure that other CPUs cannot execute this code in parallel.
 * Also no thread must be currently preempted in the middle of these instructions.
 * And on the local CPU you need to be protected again NMI or MCE handlers
 * seeing an inconsistent instruction while you patch.
 */
469
void __kprobes text_poke(void *addr, unsigned char *opcode, int len)
470 471 472
{
	memcpy(addr, opcode, len);
	sync_core();
473 474
	/* Could also do a CLFLUSH here to speed up CPU recovery; but
	   that causes hangs on some VIA CPUs. */
475
}