enlighten.c 36.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * Core of Xen paravirt_ops implementation.
 *
 * This file contains the xen_paravirt_ops structure itself, and the
 * implementations for:
 * - privileged instructions
 * - interrupt flags
 * - segment operations
 * - booting and setup
 *
 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
 */

14
#include <linux/cpu.h>
15 16 17 18
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/preempt.h>
19
#include <linux/hardirq.h>
20 21 22 23
#include <linux/percpu.h>
#include <linux/delay.h>
#include <linux/start_kernel.h>
#include <linux/sched.h>
24
#include <linux/kprobes.h>
25 26
#include <linux/bootmem.h>
#include <linux/module.h>
27 28 29
#include <linux/mm.h>
#include <linux/page-flags.h>
#include <linux/highmem.h>
30
#include <linux/console.h>
C
Chris Wright 已提交
31
#include <linux/pci.h>
32
#include <linux/gfp.h>
33
#include <linux/memblock.h>
34

35
#include <xen/xen.h>
36
#include <xen/interface/xen.h>
37
#include <xen/interface/version.h>
38 39
#include <xen/interface/physdev.h>
#include <xen/interface/vcpu.h>
40
#include <xen/interface/memory.h>
41 42
#include <xen/features.h>
#include <xen/page.h>
43
#include <xen/hvm.h>
J
Jeremy Fitzhardinge 已提交
44
#include <xen/hvc-console.h>
45
#include <xen/acpi.h>
46 47

#include <asm/paravirt.h>
I
Ingo Molnar 已提交
48
#include <asm/apic.h>
49
#include <asm/page.h>
50
#include <asm/xen/pci.h>
51 52 53 54
#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>
#include <asm/fixmap.h>
#include <asm/processor.h>
55
#include <asm/proto.h>
56
#include <asm/msr-index.h>
57
#include <asm/traps.h>
58 59
#include <asm/setup.h>
#include <asm/desc.h>
60
#include <asm/pgalloc.h>
61
#include <asm/pgtable.h>
J
Jeremy Fitzhardinge 已提交
62
#include <asm/tlbflush.h>
J
Jeremy Fitzhardinge 已提交
63
#include <asm/reboot.h>
64
#include <asm/stackprotector.h>
65
#include <asm/hypervisor.h>
66
#include <asm/mwait.h>
67
#include <asm/pci_x86.h>
68 69 70 71 72 73 74 75

#ifdef CONFIG_ACPI
#include <linux/acpi.h>
#include <asm/acpi.h>
#include <acpi/pdc_intel.h>
#include <acpi/processor.h>
#include <xen/interface/platform.h>
#endif
76 77

#include "xen-ops.h"
J
Jeremy Fitzhardinge 已提交
78
#include "mmu.h"
B
Ben Guthro 已提交
79
#include "smp.h"
80 81 82 83 84 85
#include "multicalls.h"

EXPORT_SYMBOL_GPL(hypercall_page);

DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
86

87 88 89
enum xen_domain_type xen_domain_type = XEN_NATIVE;
EXPORT_SYMBOL_GPL(xen_domain_type);

90 91
unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START;
EXPORT_SYMBOL(machine_to_phys_mapping);
92 93
unsigned long  machine_to_phys_nr;
EXPORT_SYMBOL(machine_to_phys_nr);
94

95 96 97
struct start_info *xen_start_info;
EXPORT_SYMBOL_GPL(xen_start_info);

98
struct shared_info xen_dummy_shared_info;
99

100 101
void *xen_initial_gdt;

102
RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
103 104
__read_mostly int xen_have_vector_callback;
EXPORT_SYMBOL_GPL(xen_have_vector_callback);
105

106 107 108 109
/*
 * Point at some empty memory to start with. We map the real shared_info
 * page as soon as fixmap is up and running.
 */
110
struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info;
111 112 113 114 115 116 117 118 119 120 121 122 123 124

/*
 * Flag to determine whether vcpu info placement is available on all
 * VCPUs.  We assume it is to start with, and then set it to zero on
 * the first failure.  This is because it can succeed on some VCPUs
 * and not others, since it can involve hypervisor memory allocation,
 * or because the guest failed to guarantee all the appropriate
 * constraints on all VCPUs (ie buffer can't cross a page boundary).
 *
 * Note that any particular CPU may be using a placed vcpu structure,
 * but we can only optimise if the all are.
 *
 * 0: not available, 1: available
 */
125
static int have_vcpu_info_placement = 1;
126

127 128 129 130 131 132 133 134
static void clamp_max_cpus(void)
{
#ifdef CONFIG_SMP
	if (setup_max_cpus > MAX_VIRT_CPUS)
		setup_max_cpus = MAX_VIRT_CPUS;
#endif
}

135
static void xen_vcpu_setup(int cpu)
136
{
137 138 139 140
	struct vcpu_register_vcpu_info info;
	int err;
	struct vcpu_info *vcpup;

141
	BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
142

143 144
	if (cpu < MAX_VIRT_CPUS)
		per_cpu(xen_vcpu,cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
145

146 147 148 149 150
	if (!have_vcpu_info_placement) {
		if (cpu >= MAX_VIRT_CPUS)
			clamp_max_cpus();
		return;
	}
151

152
	vcpup = &per_cpu(xen_vcpu_info, cpu);
153
	info.mfn = arbitrary_virt_to_mfn(vcpup);
154 155 156 157 158 159 160 161 162 163
	info.offset = offset_in_page(vcpup);

	/* Check to see if the hypervisor will put the vcpu_info
	   structure where we want it, which allows direct access via
	   a percpu-variable. */
	err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);

	if (err) {
		printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
		have_vcpu_info_placement = 0;
164
		clamp_max_cpus();
165 166 167 168 169
	} else {
		/* This cpu is using the registered vcpu info, even if
		   later ones fail to. */
		per_cpu(xen_vcpu, cpu) = vcpup;
	}
170 171
}

172 173 174 175 176 177 178
/*
 * On restore, set the vcpu placement up again.
 * If it fails, then we're in a bad state, since
 * we can't back out from using it...
 */
void xen_vcpu_restore(void)
{
179
	int cpu;
180

181 182
	for_each_online_cpu(cpu) {
		bool other_cpu = (cpu != smp_processor_id());
183

184 185 186
		if (other_cpu &&
		    HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL))
			BUG();
187

188
		xen_setup_runstate_info(cpu);
189

190
		if (have_vcpu_info_placement)
191 192
			xen_vcpu_setup(cpu);

193 194 195
		if (other_cpu &&
		    HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
			BUG();
196 197 198
	}
}

199 200
static void __init xen_banner(void)
{
201 202 203 204
	unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL);
	struct xen_extraversion extra;
	HYPERVISOR_xen_version(XENVER_extraversion, &extra);

205
	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
206
	       pv_info.name);
207 208
	printk(KERN_INFO "Xen version: %d.%d%s%s\n",
	       version >> 16, version & 0xffff, extra.extraversion,
209
	       xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
210 211
}

J
Jeremy Fitzhardinge 已提交
212 213 214
static __read_mostly unsigned int cpuid_leaf1_edx_mask = ~0;
static __read_mostly unsigned int cpuid_leaf1_ecx_mask = ~0;

215 216 217 218
static __read_mostly unsigned int cpuid_leaf1_ecx_set_mask;
static __read_mostly unsigned int cpuid_leaf5_ecx_val;
static __read_mostly unsigned int cpuid_leaf5_edx_val;

219 220
static void xen_cpuid(unsigned int *ax, unsigned int *bx,
		      unsigned int *cx, unsigned int *dx)
221
{
222
	unsigned maskebx = ~0;
J
Jeremy Fitzhardinge 已提交
223
	unsigned maskecx = ~0;
224
	unsigned maskedx = ~0;
225
	unsigned setecx = 0;
226 227 228 229
	/*
	 * Mask out inconvenient features, to try and disable as many
	 * unsupported kernel subsystems as possible.
	 */
230 231
	switch (*ax) {
	case 1:
J
Jeremy Fitzhardinge 已提交
232
		maskecx = cpuid_leaf1_ecx_mask;
233
		setecx = cpuid_leaf1_ecx_set_mask;
J
Jeremy Fitzhardinge 已提交
234
		maskedx = cpuid_leaf1_edx_mask;
235 236
		break;

237 238 239 240 241 242 243 244
	case CPUID_MWAIT_LEAF:
		/* Synthesize the values.. */
		*ax = 0;
		*bx = 0;
		*cx = cpuid_leaf5_ecx_val;
		*dx = cpuid_leaf5_edx_val;
		return;

245 246 247 248
	case 0xb:
		/* Suppress extended topology stuff */
		maskebx = 0;
		break;
J
Jeremy Fitzhardinge 已提交
249
	}
250 251

	asm(XEN_EMULATE_PREFIX "cpuid"
252 253 254 255 256
		: "=a" (*ax),
		  "=b" (*bx),
		  "=c" (*cx),
		  "=d" (*dx)
		: "0" (*ax), "2" (*cx));
J
Jeremy Fitzhardinge 已提交
257

258
	*bx &= maskebx;
J
Jeremy Fitzhardinge 已提交
259
	*cx &= maskecx;
260
	*cx |= setecx;
261
	*dx &= maskedx;
262

263 264
}

265 266
static bool __init xen_check_mwait(void)
{
267 268
#if defined(CONFIG_ACPI) && !defined(CONFIG_ACPI_PROCESSOR_AGGREGATOR) && \
	!defined(CONFIG_ACPI_PROCESSOR_AGGREGATOR_MODULE)
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
	struct xen_platform_op op = {
		.cmd			= XENPF_set_processor_pminfo,
		.u.set_pminfo.id	= -1,
		.u.set_pminfo.type	= XEN_PM_PDC,
	};
	uint32_t buf[3];
	unsigned int ax, bx, cx, dx;
	unsigned int mwait_mask;

	/* We need to determine whether it is OK to expose the MWAIT
	 * capability to the kernel to harvest deeper than C3 states from ACPI
	 * _CST using the processor_harvest_xen.c module. For this to work, we
	 * need to gather the MWAIT_LEAF values (which the cstate.c code
	 * checks against). The hypervisor won't expose the MWAIT flag because
	 * it would break backwards compatibility; so we will find out directly
	 * from the hardware and hypercall.
	 */
	if (!xen_initial_domain())
		return false;

	ax = 1;
	cx = 0;

	native_cpuid(&ax, &bx, &cx, &dx);

	mwait_mask = (1 << (X86_FEATURE_EST % 32)) |
		     (1 << (X86_FEATURE_MWAIT % 32));

	if ((cx & mwait_mask) != mwait_mask)
		return false;

	/* We need to emulate the MWAIT_LEAF and for that we need both
	 * ecx and edx. The hypercall provides only partial information.
	 */

	ax = CPUID_MWAIT_LEAF;
	bx = 0;
	cx = 0;
	dx = 0;

	native_cpuid(&ax, &bx, &cx, &dx);

	/* Ask the Hypervisor whether to clear ACPI_PDC_C_C2C3_FFH. If so,
	 * don't expose MWAIT_LEAF and let ACPI pick the IOPORT version of C3.
	 */
	buf[0] = ACPI_PDC_REVISION_ID;
	buf[1] = 1;
	buf[2] = (ACPI_PDC_C_CAPABILITY_SMP | ACPI_PDC_EST_CAPABILITY_SWSMP);

	set_xen_guest_handle(op.u.set_pminfo.pdc, buf);

	if ((HYPERVISOR_dom0_op(&op) == 0) &&
	    (buf[2] & (ACPI_PDC_C_C1_FFH | ACPI_PDC_C_C2C3_FFH))) {
		cpuid_leaf5_ecx_val = cx;
		cpuid_leaf5_edx_val = dx;
	}
	return true;
#else
	return false;
#endif
}
330
static void __init xen_init_cpuid_mask(void)
J
Jeremy Fitzhardinge 已提交
331 332
{
	unsigned int ax, bx, cx, dx;
333
	unsigned int xsave_mask;
J
Jeremy Fitzhardinge 已提交
334 335 336 337

	cpuid_leaf1_edx_mask =
		~((1 << X86_FEATURE_MCE)  |  /* disable MCE */
		  (1 << X86_FEATURE_MCA)  |  /* disable MCA */
338
		  (1 << X86_FEATURE_MTRR) |  /* disable MTRR */
J
Jeremy Fitzhardinge 已提交
339 340 341 342 343 344
		  (1 << X86_FEATURE_ACC));   /* thermal monitoring */

	if (!xen_initial_domain())
		cpuid_leaf1_edx_mask &=
			~((1 << X86_FEATURE_APIC) |  /* disable local APIC */
			  (1 << X86_FEATURE_ACPI));  /* disable ACPI */
345
	ax = 1;
346
	cx = 0;
347
	xen_cpuid(&ax, &bx, &cx, &dx);
J
Jeremy Fitzhardinge 已提交
348

349 350 351 352 353 354 355
	xsave_mask =
		(1 << (X86_FEATURE_XSAVE % 32)) |
		(1 << (X86_FEATURE_OSXSAVE % 32));

	/* Xen will set CR4.OSXSAVE if supported and not disabled by force */
	if ((cx & xsave_mask) != xsave_mask)
		cpuid_leaf1_ecx_mask &= ~xsave_mask; /* disable XSAVE & OSXSAVE */
356 357
	if (xen_check_mwait())
		cpuid_leaf1_ecx_set_mask = (1 << (X86_FEATURE_MWAIT % 32));
J
Jeremy Fitzhardinge 已提交
358 359
}

360 361 362 363 364 365 366 367 368 369
static void xen_set_debugreg(int reg, unsigned long val)
{
	HYPERVISOR_set_debugreg(reg, val);
}

static unsigned long xen_get_debugreg(int reg)
{
	return HYPERVISOR_get_debugreg(reg);
}

370
static void xen_end_context_switch(struct task_struct *next)
371 372
{
	xen_mc_flush();
373
	paravirt_end_context_switch(next);
374 375 376 377 378 379 380
}

static unsigned long xen_store_tr(void)
{
	return 0;
}

381
/*
382 383 384 385
 * Set the page permissions for a particular virtual address.  If the
 * address is a vmalloc mapping (or other non-linear mapping), then
 * find the linear mapping of the page and also set its protections to
 * match.
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
 */
static void set_aliased_prot(void *v, pgprot_t prot)
{
	int level;
	pte_t *ptep;
	pte_t pte;
	unsigned long pfn;
	struct page *page;

	ptep = lookup_address((unsigned long)v, &level);
	BUG_ON(ptep == NULL);

	pfn = pte_pfn(*ptep);
	page = pfn_to_page(pfn);

	pte = pfn_pte(pfn, prot);

	if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0))
		BUG();

	if (!PageHighMem(page)) {
		void *av = __va(PFN_PHYS(pfn));

		if (av != v)
			if (HYPERVISOR_update_va_mapping((unsigned long)av, pte, 0))
				BUG();
	} else
		kmap_flush_unused();
}

416 417
static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries)
{
418
	const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
419 420
	int i;

421 422
	for(i = 0; i < entries; i += entries_per_page)
		set_aliased_prot(ldt + i, PAGE_KERNEL_RO);
423 424 425 426
}

static void xen_free_ldt(struct desc_struct *ldt, unsigned entries)
{
427
	const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
428 429
	int i;

430 431
	for(i = 0; i < entries; i += entries_per_page)
		set_aliased_prot(ldt + i, PAGE_KERNEL);
432 433
}

434 435 436 437 438
static void xen_set_ldt(const void *addr, unsigned entries)
{
	struct mmuext_op *op;
	struct multicall_space mcs = xen_mc_entry(sizeof(*op));

439 440
	trace_xen_cpu_set_ldt(addr, entries);

441 442
	op = mcs.args;
	op->cmd = MMUEXT_SET_LDT;
443
	op->arg1.linear_addr = (unsigned long)addr;
444 445 446 447 448 449 450
	op->arg2.nr_ents = entries;

	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

451
static void xen_load_gdt(const struct desc_ptr *dtr)
452 453 454 455
{
	unsigned long va = dtr->address;
	unsigned int size = dtr->size + 1;
	unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
456
	unsigned long frames[pages];
457 458
	int f;

459 460 461 462
	/*
	 * A GDT can be up to 64k in size, which corresponds to 8192
	 * 8-byte entries, or 16 4k pages..
	 */
463 464 465 466 467

	BUG_ON(size > 65536);
	BUG_ON(va & ~PAGE_MASK);

	for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
J
Jeremy Fitzhardinge 已提交
468
		int level;
469
		pte_t *ptep;
J
Jeremy Fitzhardinge 已提交
470 471 472
		unsigned long pfn, mfn;
		void *virt;

473 474 475 476 477 478 479 480
		/*
		 * The GDT is per-cpu and is in the percpu data area.
		 * That can be virtually mapped, so we need to do a
		 * page-walk to get the underlying MFN for the
		 * hypercall.  The page can also be in the kernel's
		 * linear range, so we need to RO that mapping too.
		 */
		ptep = lookup_address(va, &level);
J
Jeremy Fitzhardinge 已提交
481 482 483 484 485 486 487
		BUG_ON(ptep == NULL);

		pfn = pte_pfn(*ptep);
		mfn = pfn_to_mfn(pfn);
		virt = __va(PFN_PHYS(pfn));

		frames[f] = mfn;
488

489
		make_lowmem_page_readonly((void *)va);
J
Jeremy Fitzhardinge 已提交
490
		make_lowmem_page_readonly(virt);
491 492
	}

493 494
	if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
		BUG();
495 496
}

497 498 499
/*
 * load_gdt for early boot, when the gdt is only mapped once
 */
500
static void __init xen_load_gdt_boot(const struct desc_ptr *dtr)
501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
{
	unsigned long va = dtr->address;
	unsigned int size = dtr->size + 1;
	unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
	unsigned long frames[pages];
	int f;

	/*
	 * A GDT can be up to 64k in size, which corresponds to 8192
	 * 8-byte entries, or 16 4k pages..
	 */

	BUG_ON(size > 65536);
	BUG_ON(va & ~PAGE_MASK);

	for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
		pte_t pte;
		unsigned long pfn, mfn;

		pfn = virt_to_pfn(va);
		mfn = pfn_to_mfn(pfn);

		pte = pfn_pte(pfn, PAGE_KERNEL_RO);

		if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
			BUG();

		frames[f] = mfn;
	}

	if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
		BUG();
}

535 536 537 538
static void load_TLS_descriptor(struct thread_struct *t,
				unsigned int cpu, unsigned int i)
{
	struct desc_struct *gdt = get_cpu_gdt_table(cpu);
539
	xmaddr_t maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
540 541 542 543 544 545 546
	struct multicall_space mc = __xen_mc_entry(0);

	MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
}

static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
{
547
	/*
548 549 550 551 552 553 554 555
	 * XXX sleazy hack: If we're being called in a lazy-cpu zone
	 * and lazy gs handling is enabled, it means we're in a
	 * context switch, and %gs has just been saved.  This means we
	 * can zero it out to prevent faults on exit from the
	 * hypervisor if the next process has no %gs.  Either way, it
	 * has been saved, and the new value will get loaded properly.
	 * This will go away as soon as Xen has been modified to not
	 * save/restore %gs for normal hypercalls.
556 557 558 559 560 561 562 563
	 *
	 * On x86_64, this hack is not used for %gs, because gs points
	 * to KERNEL_GS_BASE (and uses it for PDA references), so we
	 * must not zero %gs on x86_64
	 *
	 * For x86_64, we need to zero %fs, otherwise we may get an
	 * exception between the new %fs descriptor being loaded and
	 * %fs being effectively cleared at __switch_to().
564
	 */
565 566
	if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
#ifdef CONFIG_X86_32
567
		lazy_load_gs(0);
568 569 570 571 572 573 574 575 576 577 578 579
#else
		loadsegment(fs, 0);
#endif
	}

	xen_mc_batch();

	load_TLS_descriptor(t, cpu, 0);
	load_TLS_descriptor(t, cpu, 1);
	load_TLS_descriptor(t, cpu, 2);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
580 581
}

582 583 584 585 586
#ifdef CONFIG_X86_64
static void xen_load_gs_index(unsigned int idx)
{
	if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx))
		BUG();
587
}
588
#endif
589 590

static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
591
				const void *ptr)
592
{
593
	xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]);
594
	u64 entry = *(u64 *)ptr;
595

596 597
	trace_xen_cpu_write_ldt_entry(dt, entrynum, entry);

598 599
	preempt_disable();

600 601 602
	xen_mc_flush();
	if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
		BUG();
603 604

	preempt_enable();
605 606
}

607
static int cvt_gate_to_trap(int vector, const gate_desc *val,
608 609
			    struct trap_info *info)
{
610 611
	unsigned long addr;

612
	if (val->type != GATE_TRAP && val->type != GATE_INTERRUPT)
613 614 615
		return 0;

	info->vector = vector;
616 617 618

	addr = gate_offset(*val);
#ifdef CONFIG_X86_64
619 620 621 622 623 624 625
	/*
	 * Look for known traps using IST, and substitute them
	 * appropriately.  The debugger ones are the only ones we care
	 * about.  Xen will handle faults like double_fault and
	 * machine_check, so we should never see them.  Warn if
	 * there's an unexpected IST-using fault handler.
	 */
626 627 628 629 630 631
	if (addr == (unsigned long)debug)
		addr = (unsigned long)xen_debug;
	else if (addr == (unsigned long)int3)
		addr = (unsigned long)xen_int3;
	else if (addr == (unsigned long)stack_segment)
		addr = (unsigned long)xen_stack_segment;
632 633 634 635 636 637 638 639 640 641 642 643 644
	else if (addr == (unsigned long)double_fault ||
		 addr == (unsigned long)nmi) {
		/* Don't need to handle these */
		return 0;
#ifdef CONFIG_X86_MCE
	} else if (addr == (unsigned long)machine_check) {
		return 0;
#endif
	} else {
		/* Some other trap using IST? */
		if (WARN_ON(val->ist != 0))
			return 0;
	}
645 646 647
#endif	/* CONFIG_X86_64 */
	info->address = addr;

648 649
	info->cs = gate_segment(*val);
	info->flags = val->dpl;
650
	/* interrupt gates clear IF */
651 652
	if (val->type == GATE_INTERRUPT)
		info->flags |= 1 << 2;
653 654 655 656 657

	return 1;
}

/* Locations of each CPU's IDT */
658
static DEFINE_PER_CPU(struct desc_ptr, idt_desc);
659 660 661

/* Set an IDT entry.  If the entry is part of the current IDT, then
   also update Xen. */
662
static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g)
663 664
{
	unsigned long p = (unsigned long)&dt[entrynum];
665 666
	unsigned long start, end;

667 668
	trace_xen_cpu_write_idt_entry(dt, entrynum, g);

669 670
	preempt_disable();

C
Christoph Lameter 已提交
671 672
	start = __this_cpu_read(idt_desc.address);
	end = start + __this_cpu_read(idt_desc.size) + 1;
673 674 675

	xen_mc_flush();

676
	native_write_idt_entry(dt, entrynum, g);
677 678 679 680 681 682

	if (p >= start && (p + 8) <= end) {
		struct trap_info info[2];

		info[1].address = 0;

683
		if (cvt_gate_to_trap(entrynum, g, &info[0]))
684 685 686
			if (HYPERVISOR_set_trap_table(info))
				BUG();
	}
687 688

	preempt_enable();
689 690
}

691
static void xen_convert_trap_info(const struct desc_ptr *desc,
J
Jeremy Fitzhardinge 已提交
692
				  struct trap_info *traps)
693 694 695
{
	unsigned in, out, count;

696
	count = (desc->size+1) / sizeof(gate_desc);
697 698 699
	BUG_ON(count > 256);

	for (in = out = 0; in < count; in++) {
700
		gate_desc *entry = (gate_desc*)(desc->address) + in;
701

702
		if (cvt_gate_to_trap(in, entry, &traps[out]))
703 704 705
			out++;
	}
	traps[out].address = 0;
J
Jeremy Fitzhardinge 已提交
706 707 708 709
}

void xen_copy_trap_info(struct trap_info *traps)
{
710
	const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
J
Jeremy Fitzhardinge 已提交
711 712 713 714 715 716 717

	xen_convert_trap_info(desc, traps);
}

/* Load a new IDT into Xen.  In principle this can be per-CPU, so we
   hold a spinlock to protect the static traps[] array (static because
   it avoids allocation, and saves stack space). */
718
static void xen_load_idt(const struct desc_ptr *desc)
J
Jeremy Fitzhardinge 已提交
719 720 721 722
{
	static DEFINE_SPINLOCK(lock);
	static struct trap_info traps[257];

723 724
	trace_xen_cpu_load_idt(desc);

J
Jeremy Fitzhardinge 已提交
725 726
	spin_lock(&lock);

727 728
	__get_cpu_var(idt_desc) = *desc;

J
Jeremy Fitzhardinge 已提交
729
	xen_convert_trap_info(desc, traps);
730 731 732 733 734 735 736 737 738 739 740

	xen_mc_flush();
	if (HYPERVISOR_set_trap_table(traps))
		BUG();

	spin_unlock(&lock);
}

/* Write a GDT descriptor entry.  Ignore LDT descriptors, since
   they're handled differently. */
static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
741
				const void *desc, int type)
742
{
743 744
	trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);

745 746
	preempt_disable();

747 748 749
	switch (type) {
	case DESC_LDT:
	case DESC_TSS:
750 751 752 753
		/* ignore */
		break;

	default: {
754
		xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]);
755 756

		xen_mc_flush();
757
		if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
758 759 760 761
			BUG();
	}

	}
762 763

	preempt_enable();
764 765
}

766 767 768 769
/*
 * Version of write_gdt_entry for use at early boot-time needed to
 * update an entry as simply as possible.
 */
770
static void __init xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
771 772
					    const void *desc, int type)
{
773 774
	trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);

775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
	switch (type) {
	case DESC_LDT:
	case DESC_TSS:
		/* ignore */
		break;

	default: {
		xmaddr_t maddr = virt_to_machine(&dt[entry]);

		if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
			dt[entry] = *(struct desc_struct *)desc;
	}

	}
}

791
static void xen_load_sp0(struct tss_struct *tss,
792
			 struct thread_struct *thread)
793
{
794 795 796
	struct multicall_space mcs;

	mcs = xen_mc_entry(0);
797
	MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

static void xen_set_iopl_mask(unsigned mask)
{
	struct physdev_set_iopl set_iopl;

	/* Force the change at ring 0. */
	set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
	HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
}

static void xen_io_delay(void)
{
}

#ifdef CONFIG_X86_LOCAL_APIC
815 816 817 818 819 820 821 822 823
static unsigned long xen_set_apic_id(unsigned int x)
{
	WARN_ON(1);
	return x;
}
static unsigned int xen_get_apic_id(unsigned long x)
{
	return ((x)>>24) & 0xFFu;
}
824
static u32 xen_apic_read(u32 reg)
825
{
826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
	struct xen_platform_op op = {
		.cmd = XENPF_get_cpuinfo,
		.interface_version = XENPF_INTERFACE_VERSION,
		.u.pcpu_info.xen_cpuid = 0,
	};
	int ret = 0;

	/* Shouldn't need this as APIC is turned off for PV, and we only
	 * get called on the bootup processor. But just in case. */
	if (!xen_initial_domain() || smp_processor_id())
		return 0;

	if (reg == APIC_LVR)
		return 0x10;

	if (reg != APIC_ID)
		return 0;

	ret = HYPERVISOR_dom0_op(&op);
	if (ret)
		return 0;

	return op.u.pcpu_info.apic_id << 24;
849
}
J
Jeremy Fitzhardinge 已提交
850

851
static void xen_apic_write(u32 reg, u32 val)
J
Jeremy Fitzhardinge 已提交
852 853 854 855
{
	/* Warn to see if there's any stray references */
	WARN_ON(1);
}
856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872

static u64 xen_apic_icr_read(void)
{
	return 0;
}

static void xen_apic_icr_write(u32 low, u32 id)
{
	/* Warn to see if there's any stray references */
	WARN_ON(1);
}

static void xen_apic_wait_icr_idle(void)
{
        return;
}

873 874 875 876 877
static u32 xen_safe_apic_wait_icr_idle(void)
{
        return 0;
}

Y
Yinghai Lu 已提交
878 879 880 881 882 883 884 885
static void set_xen_basic_apic_ops(void)
{
	apic->read = xen_apic_read;
	apic->write = xen_apic_write;
	apic->icr_read = xen_apic_icr_read;
	apic->icr_write = xen_apic_icr_write;
	apic->wait_icr_idle = xen_apic_wait_icr_idle;
	apic->safe_wait_icr_idle = xen_safe_apic_wait_icr_idle;
886 887
	apic->set_apic_id = xen_set_apic_id;
	apic->get_apic_id = xen_get_apic_id;
B
Ben Guthro 已提交
888 889 890 891 892 893 894 895

#ifdef CONFIG_SMP
	apic->send_IPI_allbutself = xen_send_IPI_allbutself;
	apic->send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself;
	apic->send_IPI_mask = xen_send_IPI_mask;
	apic->send_IPI_all = xen_send_IPI_all;
	apic->send_IPI_self = xen_send_IPI_self;
#endif
Y
Yinghai Lu 已提交
896
}
897

898 899
#endif

900 901 902 903 904 905 906 907 908 909 910
static void xen_clts(void)
{
	struct multicall_space mcs;

	mcs = xen_mc_entry(0);

	MULTI_fpu_taskswitch(mcs.mc, 0);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

911 912 913 914
static DEFINE_PER_CPU(unsigned long, xen_cr0_value);

static unsigned long xen_read_cr0(void)
{
915
	unsigned long cr0 = this_cpu_read(xen_cr0_value);
916 917 918

	if (unlikely(cr0 == 0)) {
		cr0 = native_read_cr0();
919
		this_cpu_write(xen_cr0_value, cr0);
920 921 922 923 924
	}

	return cr0;
}

925 926 927 928
static void xen_write_cr0(unsigned long cr0)
{
	struct multicall_space mcs;

929
	this_cpu_write(xen_cr0_value, cr0);
930

931 932 933 934 935 936 937 938 939
	/* Only pay attention to cr0.TS; everything else is
	   ignored. */
	mcs = xen_mc_entry(0);

	MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

940 941
static void xen_write_cr4(unsigned long cr4)
{
942 943 944 945
	cr4 &= ~X86_CR4_PGE;
	cr4 &= ~X86_CR4_PSE;

	native_write_cr4(cr4);
946 947
}

948 949 950 951 952 953
static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
{
	int ret;

	ret = 0;

T
Tej 已提交
954
	switch (msr) {
955 956 957 958 959 960 961 962 963 964 965
#ifdef CONFIG_X86_64
		unsigned which;
		u64 base;

	case MSR_FS_BASE:		which = SEGBASE_FS; goto set;
	case MSR_KERNEL_GS_BASE:	which = SEGBASE_GS_USER; goto set;
	case MSR_GS_BASE:		which = SEGBASE_GS_KERNEL; goto set;

	set:
		base = ((u64)high << 32) | low;
		if (HYPERVISOR_set_segment_base(which, base) != 0)
966
			ret = -EIO;
967 968
		break;
#endif
J
Jeremy Fitzhardinge 已提交
969 970 971 972 973 974 975 976 977 978 979 980 981

	case MSR_STAR:
	case MSR_CSTAR:
	case MSR_LSTAR:
	case MSR_SYSCALL_MASK:
	case MSR_IA32_SYSENTER_CS:
	case MSR_IA32_SYSENTER_ESP:
	case MSR_IA32_SYSENTER_EIP:
		/* Fast syscall setup is all done in hypercalls, so
		   these are all ignored.  Stub them out here to stop
		   Xen console noise. */
		break;

J
Jeremy Fitzhardinge 已提交
982 983 984 985 986
	case MSR_IA32_CR_PAT:
		if (smp_processor_id() == 0)
			xen_set_pat(((u64)high << 32) | low);
		break;

987 988 989 990 991 992 993
	default:
		ret = native_write_msr_safe(msr, low, high);
	}

	return ret;
}

994
void xen_setup_shared_info(void)
995 996
{
	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
997 998 999 1000 1001
		set_fixmap(FIX_PARAVIRT_BOOTMAP,
			   xen_start_info->shared_info);

		HYPERVISOR_shared_info =
			(struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
1002 1003 1004 1005
	} else
		HYPERVISOR_shared_info =
			(struct shared_info *)__va(xen_start_info->shared_info);

1006 1007 1008 1009
#ifndef CONFIG_SMP
	/* In UP this is as good a place as any to set up shared info */
	xen_setup_vcpu_info_placement();
#endif
J
Jeremy Fitzhardinge 已提交
1010 1011

	xen_setup_mfn_list_list();
1012 1013
}

1014
/* This is called once we have the cpu_possible_mask */
1015
void xen_setup_vcpu_info_placement(void)
1016 1017 1018 1019 1020 1021 1022 1023 1024
{
	int cpu;

	for_each_possible_cpu(cpu)
		xen_vcpu_setup(cpu);

	/* xen_vcpu_setup managed to place the vcpu_info within the
	   percpu area for all cpus, so make use of it */
	if (have_vcpu_info_placement) {
1025 1026 1027 1028
		pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
		pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
		pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
		pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
1029
		pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
1030
	}
1031 1032
}

1033 1034
static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
			  unsigned long addr, unsigned len)
1035 1036 1037 1038 1039 1040
{
	char *start, *end, *reloc;
	unsigned ret;

	start = end = reloc = NULL;

1041 1042
#define SITE(op, x)							\
	case PARAVIRT_PATCH(op.x):					\
1043 1044 1045 1046 1047 1048 1049 1050
	if (have_vcpu_info_placement) {					\
		start = (char *)xen_##x##_direct;			\
		end = xen_##x##_direct_end;				\
		reloc = xen_##x##_direct_reloc;				\
	}								\
	goto patch_site

	switch (type) {
1051 1052 1053 1054
		SITE(pv_irq_ops, irq_enable);
		SITE(pv_irq_ops, irq_disable);
		SITE(pv_irq_ops, save_fl);
		SITE(pv_irq_ops, restore_fl);
1055 1056 1057 1058 1059 1060
#undef SITE

	patch_site:
		if (start == NULL || (end-start) > len)
			goto default_patch;

1061
		ret = paravirt_patch_insns(insnbuf, len, start, end);
1062 1063 1064 1065 1066 1067 1068

		/* Note: because reloc is assigned from something that
		   appears to be an array, gcc assumes it's non-null,
		   but doesn't know its relationship with start and
		   end. */
		if (reloc > start && reloc < end) {
			int reloc_off = reloc - start;
1069 1070
			long *relocp = (long *)(insnbuf + reloc_off);
			long delta = start - (char *)addr;
1071 1072 1073 1074 1075 1076 1077

			*relocp += delta;
		}
		break;

	default_patch:
	default:
1078 1079
		ret = paravirt_patch_default(type, clobbers, insnbuf,
					     addr, len);
1080 1081 1082 1083 1084 1085
		break;
	}

	return ret;
}

1086
static const struct pv_info xen_info __initconst = {
1087 1088 1089
	.paravirt_enabled = 1,
	.shared_kernel_pmd = 0,

1090 1091 1092 1093
#ifdef CONFIG_X86_64
	.extra_user_64bit_cs = FLAT_USER_CS64,
#endif

1094
	.name = "Xen",
1095
};
1096

1097
static const struct pv_init_ops xen_init_ops __initconst = {
1098
	.patch = xen_patch,
1099
};
1100

1101
static const struct pv_cpu_ops xen_cpu_ops __initconst = {
1102 1103 1104 1105 1106
	.cpuid = xen_cpuid,

	.set_debugreg = xen_set_debugreg,
	.get_debugreg = xen_get_debugreg,

1107
	.clts = xen_clts,
1108

1109
	.read_cr0 = xen_read_cr0,
1110
	.write_cr0 = xen_write_cr0,
1111 1112 1113 1114 1115 1116 1117 1118

	.read_cr4 = native_read_cr4,
	.read_cr4_safe = native_read_cr4_safe,
	.write_cr4 = xen_write_cr4,

	.wbinvd = native_wbinvd,

	.read_msr = native_read_msr_safe,
1119
	.rdmsr_regs = native_rdmsr_safe_regs,
1120
	.write_msr = xen_write_msr_safe,
1121 1122
	.wrmsr_regs = native_wrmsr_safe_regs,

1123 1124 1125
	.read_tsc = native_read_tsc,
	.read_pmc = native_read_pmc,

1126
	.iret = xen_iret,
1127
	.irq_enable_sysexit = xen_sysexit,
1128 1129 1130 1131
#ifdef CONFIG_X86_64
	.usergs_sysret32 = xen_sysret32,
	.usergs_sysret64 = xen_sysret64,
#endif
1132 1133 1134 1135 1136 1137

	.load_tr_desc = paravirt_nop,
	.set_ldt = xen_set_ldt,
	.load_gdt = xen_load_gdt,
	.load_idt = xen_load_idt,
	.load_tls = xen_load_tls,
1138 1139 1140
#ifdef CONFIG_X86_64
	.load_gs_index = xen_load_gs_index,
#endif
1141

1142 1143 1144
	.alloc_ldt = xen_alloc_ldt,
	.free_ldt = xen_free_ldt,

1145 1146 1147 1148 1149 1150 1151
	.store_gdt = native_store_gdt,
	.store_idt = native_store_idt,
	.store_tr = xen_store_tr,

	.write_ldt_entry = xen_write_ldt_entry,
	.write_gdt_entry = xen_write_gdt_entry,
	.write_idt_entry = xen_write_idt_entry,
1152
	.load_sp0 = xen_load_sp0,
1153 1154 1155 1156

	.set_iopl_mask = xen_set_iopl_mask,
	.io_delay = xen_io_delay,

J
Jeremy Fitzhardinge 已提交
1157 1158 1159
	/* Xen takes care of %gs when switching to usermode for us */
	.swapgs = paravirt_nop,

1160 1161
	.start_context_switch = paravirt_start_context_switch,
	.end_context_switch = xen_end_context_switch,
1162 1163
};

1164
static const struct pv_apic_ops xen_apic_ops __initconst = {
1165 1166 1167
#ifdef CONFIG_X86_LOCAL_APIC
	.startup_ipi_hook = paravirt_nop,
#endif
1168 1169
};

J
Jeremy Fitzhardinge 已提交
1170 1171
static void xen_reboot(int reason)
{
J
Jeremy Fitzhardinge 已提交
1172 1173 1174
	struct sched_shutdown r = { .reason = reason };

	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
J
Jeremy Fitzhardinge 已提交
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
		BUG();
}

static void xen_restart(char *msg)
{
	xen_reboot(SHUTDOWN_reboot);
}

static void xen_emergency_restart(void)
{
	xen_reboot(SHUTDOWN_reboot);
}

static void xen_machine_halt(void)
{
	xen_reboot(SHUTDOWN_poweroff);
}

1193 1194 1195 1196 1197 1198 1199
static void xen_machine_power_off(void)
{
	if (pm_power_off)
		pm_power_off();
	xen_reboot(SHUTDOWN_poweroff);
}

J
Jeremy Fitzhardinge 已提交
1200 1201 1202 1203 1204
static void xen_crash_shutdown(struct pt_regs *regs)
{
	xen_reboot(SHUTDOWN_crash);
}

1205 1206 1207
static int
xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr)
{
1208
	xen_reboot(SHUTDOWN_crash);
1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
	return NOTIFY_DONE;
}

static struct notifier_block xen_panic_block = {
	.notifier_call= xen_panic_event,
};

int xen_panic_handler_init(void)
{
	atomic_notifier_chain_register(&panic_notifier_list, &xen_panic_block);
	return 0;
}

1222
static const struct machine_ops xen_machine_ops __initconst = {
J
Jeremy Fitzhardinge 已提交
1223 1224
	.restart = xen_restart,
	.halt = xen_machine_halt,
1225
	.power_off = xen_machine_power_off,
J
Jeremy Fitzhardinge 已提交
1226 1227 1228 1229 1230
	.shutdown = xen_machine_halt,
	.crash_shutdown = xen_crash_shutdown,
	.emergency_restart = xen_emergency_restart,
};

1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
/*
 * Set up the GDT and segment registers for -fstack-protector.  Until
 * we do this, we have to be careful not to call any stack-protected
 * function, which is most of the kernel.
 */
static void __init xen_setup_stackprotector(void)
{
	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
	pv_cpu_ops.load_gdt = xen_load_gdt_boot;

	setup_stack_canary_segment(0);
	switch_to_new_gdt(0);

	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
	pv_cpu_ops.load_gdt = xen_load_gdt;
}

1248 1249 1250
/* First C function to be called on Xen boot */
asmlinkage void __init xen_start_kernel(void)
{
1251 1252
	struct physdev_set_iopl set_iopl;
	int rc;
1253 1254 1255 1256 1257
	pgd_t *pgd;

	if (!xen_start_info)
		return;

1258 1259
	xen_domain_type = XEN_PV_DOMAIN;

1260 1261
	xen_setup_machphys_mapping();

1262
	/* Install Xen paravirt ops */
1263 1264 1265 1266 1267
	pv_info = xen_info;
	pv_init_ops = xen_init_ops;
	pv_cpu_ops = xen_cpu_ops;
	pv_apic_ops = xen_apic_ops;

1268
	x86_init.resources.memory_setup = xen_memory_setup;
1269
	x86_init.oem.arch_setup = xen_arch_setup;
1270
	x86_init.oem.banner = xen_banner;
1271

1272
	xen_init_time_ops();
1273

1274
	/*
1275
	 * Set up some pagetable state before starting to set any ptes.
1276
	 */
1277

1278 1279
	xen_init_mmu_ops();

1280 1281
	/* Prevent unwanted bits from being set in PTEs. */
	__supported_pte_mask &= ~_PAGE_GLOBAL;
1282
#if 0
1283
	if (!xen_initial_domain())
1284
#endif
1285 1286 1287 1288
		__supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);

	__supported_pte_mask |= _PAGE_IOMAP;

1289 1290 1291 1292 1293 1294
	/*
	 * Prevent page tables from being allocated in highmem, even
	 * if CONFIG_HIGHPTE is enabled.
	 */
	__userpte_alloc_gfp &= ~__GFP_HIGHMEM;

1295
	/* Work out if we support NX */
1296
	x86_configure_nx();
1297

1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
	xen_setup_features();

	/* Get mfn list */
	if (!xen_feature(XENFEAT_auto_translated_physmap))
		xen_build_dynamic_phys_to_machine();

	/*
	 * Set up kernel GDT and segment registers, mainly so that
	 * -fstack-protector code can be executed.
	 */
	xen_setup_stackprotector();
1309

1310
	xen_init_irq_ops();
J
Jeremy Fitzhardinge 已提交
1311 1312
	xen_init_cpuid_mask();

1313
#ifdef CONFIG_X86_LOCAL_APIC
1314
	/*
1315
	 * set up the basic apic ops.
1316
	 */
Y
Yinghai Lu 已提交
1317
	set_xen_basic_apic_ops();
1318
#endif
1319

1320 1321 1322 1323 1324
	if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
		pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
		pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
	}

J
Jeremy Fitzhardinge 已提交
1325 1326
	machine_ops = xen_machine_ops;

1327 1328 1329 1330 1331 1332
	/*
	 * The only reliable way to retain the initial address of the
	 * percpu gdt_page is to remember it here, so we can go and
	 * mark it RW later, when the initial percpu area is freed.
	 */
	xen_initial_gdt = &per_cpu(gdt_page, 0);
1333

1334
	xen_smp_init();
1335

1336 1337 1338 1339 1340 1341 1342 1343 1344
#ifdef CONFIG_ACPI_NUMA
	/*
	 * The pages we from Xen are not related to machine pages, so
	 * any NUMA information the kernel tries to get from ACPI will
	 * be meaningless.  Prevent it from trying.
	 */
	acpi_numa = -1;
#endif

1345 1346
	pgd = (pgd_t *)xen_start_info->pt_base;

1347
	/* Don't do the full vcpu_info placement stuff until we have a
1348
	   possible map and a non-dummy shared_info. */
1349
	per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
1350

1351
	local_irq_disable();
1352
	early_boot_irqs_disabled = true;
1353

J
Jeremy Fitzhardinge 已提交
1354
	xen_raw_console_write("mapping kernel into physical memory\n");
1355
	pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages);
1356

1357 1358 1359
	/* Allocate and initialize top and mid mfn levels for p2m structure */
	xen_build_mfn_list_list();

1360 1361
	/* keep using Xen gdt for now; no urgent need to change it */

1362
#ifdef CONFIG_X86_32
1363
	pv_info.kernel_rpl = 1;
1364
	if (xen_feature(XENFEAT_supervisor_mode_kernel))
1365
		pv_info.kernel_rpl = 0;
1366 1367 1368
#else
	pv_info.kernel_rpl = 0;
#endif
1369
	/* set the limit of our address space */
1370
	xen_reserve_top();
1371

1372 1373 1374 1375 1376 1377 1378 1379 1380
	/* We used to do this in xen_arch_setup, but that is too late on AMD
	 * were early_cpu_init (run before ->arch_setup()) calls early_amd_init
	 * which pokes 0xcf8 port.
	 */
	set_iopl.iopl = 1;
	rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
	if (rc != 0)
		xen_raw_printk("physdev_op failed %d\n", rc);

1381
#ifdef CONFIG_X86_32
1382 1383 1384
	/* set up basic CPUID stuff */
	cpu_detect(&new_cpu_data);
	new_cpu_data.hard_math = 1;
1385
	new_cpu_data.wp_works_ok = 1;
1386
	new_cpu_data.x86_capability[0] = cpuid_edx(1);
1387
#endif
1388 1389

	/* Poke various useful things into boot_params */
1390 1391 1392 1393
	boot_params.hdr.type_of_loader = (9 << 4) | 0;
	boot_params.hdr.ramdisk_image = xen_start_info->mod_start
		? __pa(xen_start_info->mod_start) : 0;
	boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
1394
	boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
1395

1396
	if (!xen_initial_domain()) {
1397
		add_preferred_console("xenboot", 0, NULL);
1398
		add_preferred_console("tty", 0, NULL);
1399
		add_preferred_console("hvc", 0, NULL);
1400 1401
		if (pci_xen)
			x86_init.pci.arch_init = pci_xen_init;
C
Chris Wright 已提交
1402
	} else {
1403 1404 1405 1406 1407 1408 1409 1410
		const struct dom0_vga_console_info *info =
			(void *)((char *)xen_start_info +
				 xen_start_info->console.dom0.info_off);

		xen_init_vga(info, xen_start_info->console.dom0.info_size);
		xen_start_info->console.domU.mfn = 0;
		xen_start_info->console.domU.evtchn = 0;

1411 1412
		xen_init_apic();

C
Chris Wright 已提交
1413 1414
		/* Make sure ACS will be enabled */
		pci_request_acs();
1415 1416

		xen_acpi_sleep_register();
1417
	}
1418 1419 1420 1421
#ifdef CONFIG_PCI
	/* PCI BIOS service won't work from a PV guest. */
	pci_probe &= ~PCI_PROBE_BIOS;
#endif
J
Jeremy Fitzhardinge 已提交
1422 1423
	xen_raw_console_write("about to get started...\n");

1424 1425
	xen_setup_runstate_info(0);

1426
	/* Start the world */
1427
#ifdef CONFIG_X86_32
1428
	i386_start_kernel();
1429
#else
J
Jeremy Fitzhardinge 已提交
1430
	x86_64_start_reservations((char *)__pa_symbol(&boot_params));
1431
#endif
1432
}
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452

static int init_hvm_pv_info(int *major, int *minor)
{
	uint32_t eax, ebx, ecx, edx, pages, msr, base;
	u64 pfn;

	base = xen_cpuid_base();
	cpuid(base + 1, &eax, &ebx, &ecx, &edx);

	*major = eax >> 16;
	*minor = eax & 0xffff;
	printk(KERN_INFO "Xen version %d.%d.\n", *major, *minor);

	cpuid(base + 2, &pages, &msr, &ecx, &edx);

	pfn = __pa(hypercall_page);
	wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));

	xen_setup_features();

1453
	pv_info.name = "Xen HVM";
1454 1455 1456 1457 1458 1459

	xen_domain_type = XEN_HVM_DOMAIN;

	return 0;
}

1460
void __ref xen_hvm_init_shared_info(void)
1461
{
1462
	int cpu;
1463
	struct xen_add_to_physmap xatp;
1464
	static struct shared_info *shared_info_page = 0;
1465

1466 1467 1468
	if (!shared_info_page)
		shared_info_page = (struct shared_info *)
			extend_brk(PAGE_SIZE, PAGE_SIZE);
1469 1470 1471 1472 1473 1474 1475 1476 1477
	xatp.domid = DOMID_SELF;
	xatp.idx = 0;
	xatp.space = XENMAPSPACE_shared_info;
	xatp.gpfn = __pa(shared_info_page) >> PAGE_SHIFT;
	if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
		BUG();

	HYPERVISOR_shared_info = (struct shared_info *)shared_info_page;

1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
	/* xen_vcpu is a pointer to the vcpu_info struct in the shared_info
	 * page, we use it in the event channel upcall and in some pvclock
	 * related functions. We don't need the vcpu_info placement
	 * optimizations because we don't use any pv_mmu or pv_irq op on
	 * HVM.
	 * When xen_hvm_init_shared_info is run at boot time only vcpu 0 is
	 * online but xen_hvm_init_shared_info is run at resume time too and
	 * in that case multiple vcpus might be online. */
	for_each_online_cpu(cpu) {
		per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
	}
1489 1490
}

1491
#ifdef CONFIG_XEN_PVHVM
1492 1493 1494 1495 1496 1497
static int __cpuinit xen_hvm_cpu_notify(struct notifier_block *self,
				    unsigned long action, void *hcpu)
{
	int cpu = (long)hcpu;
	switch (action) {
	case CPU_UP_PREPARE:
1498
		xen_vcpu_setup(cpu);
1499 1500
		if (xen_have_vector_callback)
			xen_init_lock_cpu(cpu);
1501 1502 1503 1504 1505 1506 1507
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

1508
static struct notifier_block xen_hvm_cpu_notifier __cpuinitdata = {
1509 1510 1511
	.notifier_call	= xen_hvm_cpu_notify,
};

1512 1513 1514 1515 1516 1517 1518 1519 1520
static void __init xen_hvm_guest_init(void)
{
	int r;
	int major, minor;

	r = init_hvm_pv_info(&major, &minor);
	if (r < 0)
		return;

1521
	xen_hvm_init_shared_info();
1522 1523 1524

	if (xen_feature(XENFEAT_hvm_callback_vector))
		xen_have_vector_callback = 1;
1525
	xen_hvm_smp_init();
1526
	register_cpu_notifier(&xen_hvm_cpu_notifier);
1527
	xen_unplug_emulated_devices();
1528
	x86_init.irqs.intr_init = xen_init_IRQ;
1529
	xen_hvm_init_time_ops();
1530
	xen_hvm_init_mmu_ops();
1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
}

static bool __init xen_hvm_platform(void)
{
	if (xen_pv_domain())
		return false;

	if (!xen_cpuid_base())
		return false;

	return true;
}

S
Sheng Yang 已提交
1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
bool xen_hvm_need_lapic(void)
{
	if (xen_pv_domain())
		return false;
	if (!xen_hvm_domain())
		return false;
	if (xen_feature(XENFEAT_hvm_pirqs) && xen_have_vector_callback)
		return false;
	return true;
}
EXPORT_SYMBOL_GPL(xen_hvm_need_lapic);

1556
const struct hypervisor_x86 x86_hyper_xen_hvm __refconst = {
1557 1558 1559 1560 1561
	.name			= "Xen HVM",
	.detect			= xen_hvm_platform,
	.init_platform		= xen_hvm_guest_init,
};
EXPORT_SYMBOL(x86_hyper_xen_hvm);
1562
#endif