enlighten.c 40.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * Core of Xen paravirt_ops implementation.
 *
 * This file contains the xen_paravirt_ops structure itself, and the
 * implementations for:
 * - privileged instructions
 * - interrupt flags
 * - segment operations
 * - booting and setup
 *
 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
 */

14
#include <linux/cpu.h>
15 16 17 18
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/preempt.h>
19
#include <linux/hardirq.h>
20 21 22 23
#include <linux/percpu.h>
#include <linux/delay.h>
#include <linux/start_kernel.h>
#include <linux/sched.h>
24
#include <linux/kprobes.h>
25 26
#include <linux/bootmem.h>
#include <linux/module.h>
27 28 29
#include <linux/mm.h>
#include <linux/page-flags.h>
#include <linux/highmem.h>
30
#include <linux/console.h>
C
Chris Wright 已提交
31
#include <linux/pci.h>
32
#include <linux/gfp.h>
33
#include <linux/memblock.h>
34
#include <linux/edd.h>
35

36
#include <xen/xen.h>
37
#include <xen/events.h>
38
#include <xen/interface/xen.h>
39
#include <xen/interface/version.h>
40 41
#include <xen/interface/physdev.h>
#include <xen/interface/vcpu.h>
42
#include <xen/interface/memory.h>
43
#include <xen/interface/xen-mca.h>
44 45
#include <xen/features.h>
#include <xen/page.h>
46
#include <xen/hvm.h>
J
Jeremy Fitzhardinge 已提交
47
#include <xen/hvc-console.h>
48
#include <xen/acpi.h>
49 50

#include <asm/paravirt.h>
I
Ingo Molnar 已提交
51
#include <asm/apic.h>
52
#include <asm/page.h>
53
#include <asm/xen/pci.h>
54 55 56 57
#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>
#include <asm/fixmap.h>
#include <asm/processor.h>
58
#include <asm/proto.h>
59
#include <asm/msr-index.h>
60
#include <asm/traps.h>
61 62
#include <asm/setup.h>
#include <asm/desc.h>
63
#include <asm/pgalloc.h>
64
#include <asm/pgtable.h>
J
Jeremy Fitzhardinge 已提交
65
#include <asm/tlbflush.h>
J
Jeremy Fitzhardinge 已提交
66
#include <asm/reboot.h>
67
#include <asm/stackprotector.h>
68
#include <asm/hypervisor.h>
69
#include <asm/mwait.h>
70
#include <asm/pci_x86.h>
71
#include <asm/pat.h>
72 73 74 75 76 77 78 79

#ifdef CONFIG_ACPI
#include <linux/acpi.h>
#include <asm/acpi.h>
#include <acpi/pdc_intel.h>
#include <acpi/processor.h>
#include <xen/interface/platform.h>
#endif
80 81

#include "xen-ops.h"
J
Jeremy Fitzhardinge 已提交
82
#include "mmu.h"
B
Ben Guthro 已提交
83
#include "smp.h"
84 85 86 87 88 89
#include "multicalls.h"

EXPORT_SYMBOL_GPL(hypercall_page);

DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
90

91 92 93
enum xen_domain_type xen_domain_type = XEN_NATIVE;
EXPORT_SYMBOL_GPL(xen_domain_type);

94 95
unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START;
EXPORT_SYMBOL(machine_to_phys_mapping);
96 97
unsigned long  machine_to_phys_nr;
EXPORT_SYMBOL(machine_to_phys_nr);
98

99 100 101
struct start_info *xen_start_info;
EXPORT_SYMBOL_GPL(xen_start_info);

102
struct shared_info xen_dummy_shared_info;
103

104 105
void *xen_initial_gdt;

106
RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
107 108
__read_mostly int xen_have_vector_callback;
EXPORT_SYMBOL_GPL(xen_have_vector_callback);
109

110 111 112 113
/*
 * Point at some empty memory to start with. We map the real shared_info
 * page as soon as fixmap is up and running.
 */
114
struct shared_info *HYPERVISOR_shared_info = &xen_dummy_shared_info;
115 116 117 118 119 120 121 122 123 124 125 126 127 128

/*
 * Flag to determine whether vcpu info placement is available on all
 * VCPUs.  We assume it is to start with, and then set it to zero on
 * the first failure.  This is because it can succeed on some VCPUs
 * and not others, since it can involve hypervisor memory allocation,
 * or because the guest failed to guarantee all the appropriate
 * constraints on all VCPUs (ie buffer can't cross a page boundary).
 *
 * Note that any particular CPU may be using a placed vcpu structure,
 * but we can only optimise if the all are.
 *
 * 0: not available, 1: available
 */
129
static int have_vcpu_info_placement = 1;
130

131 132 133 134 135 136 137 138 139 140 141 142 143
struct tls_descs {
	struct desc_struct desc[3];
};

/*
 * Updating the 3 TLS descriptors in the GDT on every task switch is
 * surprisingly expensive so we avoid updating them if they haven't
 * changed.  Since Xen writes different descriptors than the one
 * passed in the update_descriptor hypercall we keep shadow copies to
 * compare against.
 */
static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc);

144 145 146 147 148 149 150 151
static void clamp_max_cpus(void)
{
#ifdef CONFIG_SMP
	if (setup_max_cpus > MAX_VIRT_CPUS)
		setup_max_cpus = MAX_VIRT_CPUS;
#endif
}

152
static void xen_vcpu_setup(int cpu)
153
{
154 155 156 157
	struct vcpu_register_vcpu_info info;
	int err;
	struct vcpu_info *vcpup;

158
	BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
159

160 161
	if (cpu < MAX_VIRT_CPUS)
		per_cpu(xen_vcpu,cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
162

163 164 165 166 167
	if (!have_vcpu_info_placement) {
		if (cpu >= MAX_VIRT_CPUS)
			clamp_max_cpus();
		return;
	}
168

169
	vcpup = &per_cpu(xen_vcpu_info, cpu);
170
	info.mfn = arbitrary_virt_to_mfn(vcpup);
171 172 173 174 175 176 177 178 179 180
	info.offset = offset_in_page(vcpup);

	/* Check to see if the hypervisor will put the vcpu_info
	   structure where we want it, which allows direct access via
	   a percpu-variable. */
	err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);

	if (err) {
		printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
		have_vcpu_info_placement = 0;
181
		clamp_max_cpus();
182 183 184 185 186
	} else {
		/* This cpu is using the registered vcpu info, even if
		   later ones fail to. */
		per_cpu(xen_vcpu, cpu) = vcpup;
	}
187 188
}

189 190 191 192 193 194 195
/*
 * On restore, set the vcpu placement up again.
 * If it fails, then we're in a bad state, since
 * we can't back out from using it...
 */
void xen_vcpu_restore(void)
{
196
	int cpu;
197

W
Wei Liu 已提交
198
	for_each_possible_cpu(cpu) {
199
		bool other_cpu = (cpu != smp_processor_id());
W
Wei Liu 已提交
200
		bool is_up = HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL);
201

W
Wei Liu 已提交
202
		if (other_cpu && is_up &&
203 204
		    HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL))
			BUG();
205

206
		xen_setup_runstate_info(cpu);
207

208
		if (have_vcpu_info_placement)
209 210
			xen_vcpu_setup(cpu);

W
Wei Liu 已提交
211
		if (other_cpu && is_up &&
212 213
		    HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
			BUG();
214 215 216
	}
}

217 218
static void __init xen_banner(void)
{
219 220 221 222
	unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL);
	struct xen_extraversion extra;
	HYPERVISOR_xen_version(XENVER_extraversion, &extra);

223
	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
224
	       pv_info.name);
225 226
	printk(KERN_INFO "Xen version: %d.%d%s%s\n",
	       version >> 16, version & 0xffff, extra.extraversion,
227
	       xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
228
}
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
/* Check if running on Xen version (major, minor) or later */
bool
xen_running_on_version_or_later(unsigned int major, unsigned int minor)
{
	unsigned int version;

	if (!xen_domain())
		return false;

	version = HYPERVISOR_xen_version(XENVER_version, NULL);
	if ((((version >> 16) == major) && ((version & 0xffff) >= minor)) ||
		((version >> 16) > major))
		return true;
	return false;
}
244

245 246 247
#define CPUID_THERM_POWER_LEAF 6
#define APERFMPERF_PRESENT 0

J
Jeremy Fitzhardinge 已提交
248 249 250
static __read_mostly unsigned int cpuid_leaf1_edx_mask = ~0;
static __read_mostly unsigned int cpuid_leaf1_ecx_mask = ~0;

251 252 253 254
static __read_mostly unsigned int cpuid_leaf1_ecx_set_mask;
static __read_mostly unsigned int cpuid_leaf5_ecx_val;
static __read_mostly unsigned int cpuid_leaf5_edx_val;

255 256
static void xen_cpuid(unsigned int *ax, unsigned int *bx,
		      unsigned int *cx, unsigned int *dx)
257
{
258
	unsigned maskebx = ~0;
J
Jeremy Fitzhardinge 已提交
259
	unsigned maskecx = ~0;
260
	unsigned maskedx = ~0;
261
	unsigned setecx = 0;
262 263 264 265
	/*
	 * Mask out inconvenient features, to try and disable as many
	 * unsupported kernel subsystems as possible.
	 */
266 267
	switch (*ax) {
	case 1:
J
Jeremy Fitzhardinge 已提交
268
		maskecx = cpuid_leaf1_ecx_mask;
269
		setecx = cpuid_leaf1_ecx_set_mask;
J
Jeremy Fitzhardinge 已提交
270
		maskedx = cpuid_leaf1_edx_mask;
271 272
		break;

273 274 275 276 277 278 279 280
	case CPUID_MWAIT_LEAF:
		/* Synthesize the values.. */
		*ax = 0;
		*bx = 0;
		*cx = cpuid_leaf5_ecx_val;
		*dx = cpuid_leaf5_edx_val;
		return;

281 282 283 284 285
	case CPUID_THERM_POWER_LEAF:
		/* Disabling APERFMPERF for kernel usage */
		maskecx = ~(1 << APERFMPERF_PRESENT);
		break;

286 287 288 289
	case 0xb:
		/* Suppress extended topology stuff */
		maskebx = 0;
		break;
J
Jeremy Fitzhardinge 已提交
290
	}
291 292

	asm(XEN_EMULATE_PREFIX "cpuid"
293 294 295 296 297
		: "=a" (*ax),
		  "=b" (*bx),
		  "=c" (*cx),
		  "=d" (*dx)
		: "0" (*ax), "2" (*cx));
J
Jeremy Fitzhardinge 已提交
298

299
	*bx &= maskebx;
J
Jeremy Fitzhardinge 已提交
300
	*cx &= maskecx;
301
	*cx |= setecx;
302
	*dx &= maskedx;
303

304 305
}

306 307
static bool __init xen_check_mwait(void)
{
308
#ifdef CONFIG_ACPI
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
	struct xen_platform_op op = {
		.cmd			= XENPF_set_processor_pminfo,
		.u.set_pminfo.id	= -1,
		.u.set_pminfo.type	= XEN_PM_PDC,
	};
	uint32_t buf[3];
	unsigned int ax, bx, cx, dx;
	unsigned int mwait_mask;

	/* We need to determine whether it is OK to expose the MWAIT
	 * capability to the kernel to harvest deeper than C3 states from ACPI
	 * _CST using the processor_harvest_xen.c module. For this to work, we
	 * need to gather the MWAIT_LEAF values (which the cstate.c code
	 * checks against). The hypervisor won't expose the MWAIT flag because
	 * it would break backwards compatibility; so we will find out directly
	 * from the hardware and hypercall.
	 */
	if (!xen_initial_domain())
		return false;

329 330 331 332 333 334 335
	/*
	 * When running under platform earlier than Xen4.2, do not expose
	 * mwait, to avoid the risk of loading native acpi pad driver
	 */
	if (!xen_running_on_version_or_later(4, 2))
		return false;

336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
	ax = 1;
	cx = 0;

	native_cpuid(&ax, &bx, &cx, &dx);

	mwait_mask = (1 << (X86_FEATURE_EST % 32)) |
		     (1 << (X86_FEATURE_MWAIT % 32));

	if ((cx & mwait_mask) != mwait_mask)
		return false;

	/* We need to emulate the MWAIT_LEAF and for that we need both
	 * ecx and edx. The hypercall provides only partial information.
	 */

	ax = CPUID_MWAIT_LEAF;
	bx = 0;
	cx = 0;
	dx = 0;

	native_cpuid(&ax, &bx, &cx, &dx);

	/* Ask the Hypervisor whether to clear ACPI_PDC_C_C2C3_FFH. If so,
	 * don't expose MWAIT_LEAF and let ACPI pick the IOPORT version of C3.
	 */
	buf[0] = ACPI_PDC_REVISION_ID;
	buf[1] = 1;
	buf[2] = (ACPI_PDC_C_CAPABILITY_SMP | ACPI_PDC_EST_CAPABILITY_SWSMP);

	set_xen_guest_handle(op.u.set_pminfo.pdc, buf);

	if ((HYPERVISOR_dom0_op(&op) == 0) &&
	    (buf[2] & (ACPI_PDC_C_C1_FFH | ACPI_PDC_C_C2C3_FFH))) {
		cpuid_leaf5_ecx_val = cx;
		cpuid_leaf5_edx_val = dx;
	}
	return true;
#else
	return false;
#endif
}
377
static void __init xen_init_cpuid_mask(void)
J
Jeremy Fitzhardinge 已提交
378 379
{
	unsigned int ax, bx, cx, dx;
380
	unsigned int xsave_mask;
J
Jeremy Fitzhardinge 已提交
381 382

	cpuid_leaf1_edx_mask =
383
		~((1 << X86_FEATURE_MTRR) |  /* disable MTRR */
J
Jeremy Fitzhardinge 已提交
384 385 386 387 388 389
		  (1 << X86_FEATURE_ACC));   /* thermal monitoring */

	if (!xen_initial_domain())
		cpuid_leaf1_edx_mask &=
			~((1 << X86_FEATURE_APIC) |  /* disable local APIC */
			  (1 << X86_FEATURE_ACPI));  /* disable ACPI */
390
	ax = 1;
391
	cx = 0;
392
	xen_cpuid(&ax, &bx, &cx, &dx);
J
Jeremy Fitzhardinge 已提交
393

394 395 396 397 398 399 400
	xsave_mask =
		(1 << (X86_FEATURE_XSAVE % 32)) |
		(1 << (X86_FEATURE_OSXSAVE % 32));

	/* Xen will set CR4.OSXSAVE if supported and not disabled by force */
	if ((cx & xsave_mask) != xsave_mask)
		cpuid_leaf1_ecx_mask &= ~xsave_mask; /* disable XSAVE & OSXSAVE */
401 402
	if (xen_check_mwait())
		cpuid_leaf1_ecx_set_mask = (1 << (X86_FEATURE_MWAIT % 32));
J
Jeremy Fitzhardinge 已提交
403 404
}

405 406 407 408 409 410 411 412 413 414
static void xen_set_debugreg(int reg, unsigned long val)
{
	HYPERVISOR_set_debugreg(reg, val);
}

static unsigned long xen_get_debugreg(int reg)
{
	return HYPERVISOR_get_debugreg(reg);
}

415
static void xen_end_context_switch(struct task_struct *next)
416 417
{
	xen_mc_flush();
418
	paravirt_end_context_switch(next);
419 420 421 422 423 424 425
}

static unsigned long xen_store_tr(void)
{
	return 0;
}

426
/*
427 428 429 430
 * Set the page permissions for a particular virtual address.  If the
 * address is a vmalloc mapping (or other non-linear mapping), then
 * find the linear mapping of the page and also set its protections to
 * match.
431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
 */
static void set_aliased_prot(void *v, pgprot_t prot)
{
	int level;
	pte_t *ptep;
	pte_t pte;
	unsigned long pfn;
	struct page *page;

	ptep = lookup_address((unsigned long)v, &level);
	BUG_ON(ptep == NULL);

	pfn = pte_pfn(*ptep);
	page = pfn_to_page(pfn);

	pte = pfn_pte(pfn, prot);

	if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0))
		BUG();

	if (!PageHighMem(page)) {
		void *av = __va(PFN_PHYS(pfn));

		if (av != v)
			if (HYPERVISOR_update_va_mapping((unsigned long)av, pte, 0))
				BUG();
	} else
		kmap_flush_unused();
}

461 462
static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries)
{
463
	const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
464 465
	int i;

466 467
	for(i = 0; i < entries; i += entries_per_page)
		set_aliased_prot(ldt + i, PAGE_KERNEL_RO);
468 469 470 471
}

static void xen_free_ldt(struct desc_struct *ldt, unsigned entries)
{
472
	const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
473 474
	int i;

475 476
	for(i = 0; i < entries; i += entries_per_page)
		set_aliased_prot(ldt + i, PAGE_KERNEL);
477 478
}

479 480 481 482 483
static void xen_set_ldt(const void *addr, unsigned entries)
{
	struct mmuext_op *op;
	struct multicall_space mcs = xen_mc_entry(sizeof(*op));

484 485
	trace_xen_cpu_set_ldt(addr, entries);

486 487
	op = mcs.args;
	op->cmd = MMUEXT_SET_LDT;
488
	op->arg1.linear_addr = (unsigned long)addr;
489 490 491 492 493 494 495
	op->arg2.nr_ents = entries;

	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

496
static void xen_load_gdt(const struct desc_ptr *dtr)
497 498 499 500
{
	unsigned long va = dtr->address;
	unsigned int size = dtr->size + 1;
	unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
501
	unsigned long frames[pages];
502 503
	int f;

504 505 506 507
	/*
	 * A GDT can be up to 64k in size, which corresponds to 8192
	 * 8-byte entries, or 16 4k pages..
	 */
508 509 510 511 512

	BUG_ON(size > 65536);
	BUG_ON(va & ~PAGE_MASK);

	for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
J
Jeremy Fitzhardinge 已提交
513
		int level;
514
		pte_t *ptep;
J
Jeremy Fitzhardinge 已提交
515 516 517
		unsigned long pfn, mfn;
		void *virt;

518 519 520 521 522 523 524 525
		/*
		 * The GDT is per-cpu and is in the percpu data area.
		 * That can be virtually mapped, so we need to do a
		 * page-walk to get the underlying MFN for the
		 * hypercall.  The page can also be in the kernel's
		 * linear range, so we need to RO that mapping too.
		 */
		ptep = lookup_address(va, &level);
J
Jeremy Fitzhardinge 已提交
526 527 528 529 530 531 532
		BUG_ON(ptep == NULL);

		pfn = pte_pfn(*ptep);
		mfn = pfn_to_mfn(pfn);
		virt = __va(PFN_PHYS(pfn));

		frames[f] = mfn;
533

534
		make_lowmem_page_readonly((void *)va);
J
Jeremy Fitzhardinge 已提交
535
		make_lowmem_page_readonly(virt);
536 537
	}

538 539
	if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
		BUG();
540 541
}

542 543 544
/*
 * load_gdt for early boot, when the gdt is only mapped once
 */
545
static void __init xen_load_gdt_boot(const struct desc_ptr *dtr)
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
{
	unsigned long va = dtr->address;
	unsigned int size = dtr->size + 1;
	unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
	unsigned long frames[pages];
	int f;

	/*
	 * A GDT can be up to 64k in size, which corresponds to 8192
	 * 8-byte entries, or 16 4k pages..
	 */

	BUG_ON(size > 65536);
	BUG_ON(va & ~PAGE_MASK);

	for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
		pte_t pte;
		unsigned long pfn, mfn;

		pfn = virt_to_pfn(va);
		mfn = pfn_to_mfn(pfn);

		pte = pfn_pte(pfn, PAGE_KERNEL_RO);

		if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
			BUG();

		frames[f] = mfn;
	}

	if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
		BUG();
}

580 581 582 583 584 585
static inline bool desc_equal(const struct desc_struct *d1,
			      const struct desc_struct *d2)
{
	return d1->a == d2->a && d1->b == d2->b;
}

586 587 588
static void load_TLS_descriptor(struct thread_struct *t,
				unsigned int cpu, unsigned int i)
{
589 590 591 592 593 594 595 596 597 598 599 600 601
	struct desc_struct *shadow = &per_cpu(shadow_tls_desc, cpu).desc[i];
	struct desc_struct *gdt;
	xmaddr_t maddr;
	struct multicall_space mc;

	if (desc_equal(shadow, &t->tls_array[i]))
		return;

	*shadow = t->tls_array[i];

	gdt = get_cpu_gdt_table(cpu);
	maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
	mc = __xen_mc_entry(0);
602 603 604 605 606 607

	MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
}

static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
{
608
	/*
609 610 611 612 613 614 615 616
	 * XXX sleazy hack: If we're being called in a lazy-cpu zone
	 * and lazy gs handling is enabled, it means we're in a
	 * context switch, and %gs has just been saved.  This means we
	 * can zero it out to prevent faults on exit from the
	 * hypervisor if the next process has no %gs.  Either way, it
	 * has been saved, and the new value will get loaded properly.
	 * This will go away as soon as Xen has been modified to not
	 * save/restore %gs for normal hypercalls.
617 618 619 620 621 622 623 624
	 *
	 * On x86_64, this hack is not used for %gs, because gs points
	 * to KERNEL_GS_BASE (and uses it for PDA references), so we
	 * must not zero %gs on x86_64
	 *
	 * For x86_64, we need to zero %fs, otherwise we may get an
	 * exception between the new %fs descriptor being loaded and
	 * %fs being effectively cleared at __switch_to().
625
	 */
626 627
	if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
#ifdef CONFIG_X86_32
628
		lazy_load_gs(0);
629 630 631 632 633 634 635 636 637 638 639 640
#else
		loadsegment(fs, 0);
#endif
	}

	xen_mc_batch();

	load_TLS_descriptor(t, cpu, 0);
	load_TLS_descriptor(t, cpu, 1);
	load_TLS_descriptor(t, cpu, 2);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
641 642
}

643 644 645 646 647
#ifdef CONFIG_X86_64
static void xen_load_gs_index(unsigned int idx)
{
	if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx))
		BUG();
648
}
649
#endif
650 651

static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
652
				const void *ptr)
653
{
654
	xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]);
655
	u64 entry = *(u64 *)ptr;
656

657 658
	trace_xen_cpu_write_ldt_entry(dt, entrynum, entry);

659 660
	preempt_disable();

661 662 663
	xen_mc_flush();
	if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
		BUG();
664 665

	preempt_enable();
666 667
}

668
static int cvt_gate_to_trap(int vector, const gate_desc *val,
669 670
			    struct trap_info *info)
{
671 672
	unsigned long addr;

673
	if (val->type != GATE_TRAP && val->type != GATE_INTERRUPT)
674 675 676
		return 0;

	info->vector = vector;
677 678 679

	addr = gate_offset(*val);
#ifdef CONFIG_X86_64
680 681 682
	/*
	 * Look for known traps using IST, and substitute them
	 * appropriately.  The debugger ones are the only ones we care
683 684
	 * about.  Xen will handle faults like double_fault,
	 * so we should never see them.  Warn if
685 686
	 * there's an unexpected IST-using fault handler.
	 */
687 688 689 690 691 692
	if (addr == (unsigned long)debug)
		addr = (unsigned long)xen_debug;
	else if (addr == (unsigned long)int3)
		addr = (unsigned long)xen_int3;
	else if (addr == (unsigned long)stack_segment)
		addr = (unsigned long)xen_stack_segment;
693 694 695 696 697 698
	else if (addr == (unsigned long)double_fault ||
		 addr == (unsigned long)nmi) {
		/* Don't need to handle these */
		return 0;
#ifdef CONFIG_X86_MCE
	} else if (addr == (unsigned long)machine_check) {
699 700 701 702 703
		/*
		 * when xen hypervisor inject vMCE to guest,
		 * use native mce handler to handle it
		 */
		;
704 705 706 707 708 709
#endif
	} else {
		/* Some other trap using IST? */
		if (WARN_ON(val->ist != 0))
			return 0;
	}
710 711 712
#endif	/* CONFIG_X86_64 */
	info->address = addr;

713 714
	info->cs = gate_segment(*val);
	info->flags = val->dpl;
715
	/* interrupt gates clear IF */
716 717
	if (val->type == GATE_INTERRUPT)
		info->flags |= 1 << 2;
718 719 720 721 722

	return 1;
}

/* Locations of each CPU's IDT */
723
static DEFINE_PER_CPU(struct desc_ptr, idt_desc);
724 725 726

/* Set an IDT entry.  If the entry is part of the current IDT, then
   also update Xen. */
727
static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g)
728 729
{
	unsigned long p = (unsigned long)&dt[entrynum];
730 731
	unsigned long start, end;

732 733
	trace_xen_cpu_write_idt_entry(dt, entrynum, g);

734 735
	preempt_disable();

C
Christoph Lameter 已提交
736 737
	start = __this_cpu_read(idt_desc.address);
	end = start + __this_cpu_read(idt_desc.size) + 1;
738 739 740

	xen_mc_flush();

741
	native_write_idt_entry(dt, entrynum, g);
742 743 744 745 746 747

	if (p >= start && (p + 8) <= end) {
		struct trap_info info[2];

		info[1].address = 0;

748
		if (cvt_gate_to_trap(entrynum, g, &info[0]))
749 750 751
			if (HYPERVISOR_set_trap_table(info))
				BUG();
	}
752 753

	preempt_enable();
754 755
}

756
static void xen_convert_trap_info(const struct desc_ptr *desc,
J
Jeremy Fitzhardinge 已提交
757
				  struct trap_info *traps)
758 759 760
{
	unsigned in, out, count;

761
	count = (desc->size+1) / sizeof(gate_desc);
762 763 764
	BUG_ON(count > 256);

	for (in = out = 0; in < count; in++) {
765
		gate_desc *entry = (gate_desc*)(desc->address) + in;
766

767
		if (cvt_gate_to_trap(in, entry, &traps[out]))
768 769 770
			out++;
	}
	traps[out].address = 0;
J
Jeremy Fitzhardinge 已提交
771 772 773 774
}

void xen_copy_trap_info(struct trap_info *traps)
{
775
	const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
J
Jeremy Fitzhardinge 已提交
776 777 778 779 780 781 782

	xen_convert_trap_info(desc, traps);
}

/* Load a new IDT into Xen.  In principle this can be per-CPU, so we
   hold a spinlock to protect the static traps[] array (static because
   it avoids allocation, and saves stack space). */
783
static void xen_load_idt(const struct desc_ptr *desc)
J
Jeremy Fitzhardinge 已提交
784 785 786 787
{
	static DEFINE_SPINLOCK(lock);
	static struct trap_info traps[257];

788 789
	trace_xen_cpu_load_idt(desc);

J
Jeremy Fitzhardinge 已提交
790 791
	spin_lock(&lock);

792 793
	__get_cpu_var(idt_desc) = *desc;

J
Jeremy Fitzhardinge 已提交
794
	xen_convert_trap_info(desc, traps);
795 796 797 798 799 800 801 802 803 804 805

	xen_mc_flush();
	if (HYPERVISOR_set_trap_table(traps))
		BUG();

	spin_unlock(&lock);
}

/* Write a GDT descriptor entry.  Ignore LDT descriptors, since
   they're handled differently. */
static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
806
				const void *desc, int type)
807
{
808 809
	trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);

810 811
	preempt_disable();

812 813 814
	switch (type) {
	case DESC_LDT:
	case DESC_TSS:
815 816 817 818
		/* ignore */
		break;

	default: {
819
		xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]);
820 821

		xen_mc_flush();
822
		if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
823 824 825 826
			BUG();
	}

	}
827 828

	preempt_enable();
829 830
}

831 832 833 834
/*
 * Version of write_gdt_entry for use at early boot-time needed to
 * update an entry as simply as possible.
 */
835
static void __init xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
836 837
					    const void *desc, int type)
{
838 839
	trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);

840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
	switch (type) {
	case DESC_LDT:
	case DESC_TSS:
		/* ignore */
		break;

	default: {
		xmaddr_t maddr = virt_to_machine(&dt[entry]);

		if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
			dt[entry] = *(struct desc_struct *)desc;
	}

	}
}

856
static void xen_load_sp0(struct tss_struct *tss,
857
			 struct thread_struct *thread)
858
{
859 860 861
	struct multicall_space mcs;

	mcs = xen_mc_entry(0);
862
	MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879
	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

static void xen_set_iopl_mask(unsigned mask)
{
	struct physdev_set_iopl set_iopl;

	/* Force the change at ring 0. */
	set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
	HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
}

static void xen_io_delay(void)
{
}

#ifdef CONFIG_X86_LOCAL_APIC
880 881 882 883 884 885 886 887 888
static unsigned long xen_set_apic_id(unsigned int x)
{
	WARN_ON(1);
	return x;
}
static unsigned int xen_get_apic_id(unsigned long x)
{
	return ((x)>>24) & 0xFFu;
}
889
static u32 xen_apic_read(u32 reg)
890
{
891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913
	struct xen_platform_op op = {
		.cmd = XENPF_get_cpuinfo,
		.interface_version = XENPF_INTERFACE_VERSION,
		.u.pcpu_info.xen_cpuid = 0,
	};
	int ret = 0;

	/* Shouldn't need this as APIC is turned off for PV, and we only
	 * get called on the bootup processor. But just in case. */
	if (!xen_initial_domain() || smp_processor_id())
		return 0;

	if (reg == APIC_LVR)
		return 0x10;

	if (reg != APIC_ID)
		return 0;

	ret = HYPERVISOR_dom0_op(&op);
	if (ret)
		return 0;

	return op.u.pcpu_info.apic_id << 24;
914
}
J
Jeremy Fitzhardinge 已提交
915

916
static void xen_apic_write(u32 reg, u32 val)
J
Jeremy Fitzhardinge 已提交
917 918 919 920
{
	/* Warn to see if there's any stray references */
	WARN_ON(1);
}
921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937

static u64 xen_apic_icr_read(void)
{
	return 0;
}

static void xen_apic_icr_write(u32 low, u32 id)
{
	/* Warn to see if there's any stray references */
	WARN_ON(1);
}

static void xen_apic_wait_icr_idle(void)
{
        return;
}

938 939 940 941 942
static u32 xen_safe_apic_wait_icr_idle(void)
{
        return 0;
}

Y
Yinghai Lu 已提交
943 944 945 946 947 948 949 950
static void set_xen_basic_apic_ops(void)
{
	apic->read = xen_apic_read;
	apic->write = xen_apic_write;
	apic->icr_read = xen_apic_icr_read;
	apic->icr_write = xen_apic_icr_write;
	apic->wait_icr_idle = xen_apic_wait_icr_idle;
	apic->safe_wait_icr_idle = xen_safe_apic_wait_icr_idle;
951 952
	apic->set_apic_id = xen_set_apic_id;
	apic->get_apic_id = xen_get_apic_id;
B
Ben Guthro 已提交
953 954 955 956 957 958 959 960

#ifdef CONFIG_SMP
	apic->send_IPI_allbutself = xen_send_IPI_allbutself;
	apic->send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself;
	apic->send_IPI_mask = xen_send_IPI_mask;
	apic->send_IPI_all = xen_send_IPI_all;
	apic->send_IPI_self = xen_send_IPI_self;
#endif
Y
Yinghai Lu 已提交
961
}
962

963 964
#endif

965 966 967 968 969 970 971 972 973 974 975
static void xen_clts(void)
{
	struct multicall_space mcs;

	mcs = xen_mc_entry(0);

	MULTI_fpu_taskswitch(mcs.mc, 0);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

976 977 978 979
static DEFINE_PER_CPU(unsigned long, xen_cr0_value);

static unsigned long xen_read_cr0(void)
{
980
	unsigned long cr0 = this_cpu_read(xen_cr0_value);
981 982 983

	if (unlikely(cr0 == 0)) {
		cr0 = native_read_cr0();
984
		this_cpu_write(xen_cr0_value, cr0);
985 986 987 988 989
	}

	return cr0;
}

990 991 992 993
static void xen_write_cr0(unsigned long cr0)
{
	struct multicall_space mcs;

994
	this_cpu_write(xen_cr0_value, cr0);
995

996 997 998 999 1000 1001 1002 1003 1004
	/* Only pay attention to cr0.TS; everything else is
	   ignored. */
	mcs = xen_mc_entry(0);

	MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

1005 1006
static void xen_write_cr4(unsigned long cr4)
{
1007 1008 1009 1010
	cr4 &= ~X86_CR4_PGE;
	cr4 &= ~X86_CR4_PSE;

	native_write_cr4(cr4);
1011
}
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
#ifdef CONFIG_X86_64
static inline unsigned long xen_read_cr8(void)
{
	return 0;
}
static inline void xen_write_cr8(unsigned long val)
{
	BUG_ON(val);
}
#endif
1022 1023 1024 1025 1026 1027
static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
{
	int ret;

	ret = 0;

T
Tej 已提交
1028
	switch (msr) {
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
#ifdef CONFIG_X86_64
		unsigned which;
		u64 base;

	case MSR_FS_BASE:		which = SEGBASE_FS; goto set;
	case MSR_KERNEL_GS_BASE:	which = SEGBASE_GS_USER; goto set;
	case MSR_GS_BASE:		which = SEGBASE_GS_KERNEL; goto set;

	set:
		base = ((u64)high << 32) | low;
		if (HYPERVISOR_set_segment_base(which, base) != 0)
1040
			ret = -EIO;
1041 1042
		break;
#endif
J
Jeremy Fitzhardinge 已提交
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055

	case MSR_STAR:
	case MSR_CSTAR:
	case MSR_LSTAR:
	case MSR_SYSCALL_MASK:
	case MSR_IA32_SYSENTER_CS:
	case MSR_IA32_SYSENTER_ESP:
	case MSR_IA32_SYSENTER_EIP:
		/* Fast syscall setup is all done in hypercalls, so
		   these are all ignored.  Stub them out here to stop
		   Xen console noise. */
		break;

J
Jeremy Fitzhardinge 已提交
1056 1057 1058 1059 1060
	case MSR_IA32_CR_PAT:
		if (smp_processor_id() == 0)
			xen_set_pat(((u64)high << 32) | low);
		break;

1061 1062 1063 1064 1065 1066 1067
	default:
		ret = native_write_msr_safe(msr, low, high);
	}

	return ret;
}

1068
void xen_setup_shared_info(void)
1069 1070
{
	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1071 1072 1073 1074 1075
		set_fixmap(FIX_PARAVIRT_BOOTMAP,
			   xen_start_info->shared_info);

		HYPERVISOR_shared_info =
			(struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
1076 1077 1078 1079
	} else
		HYPERVISOR_shared_info =
			(struct shared_info *)__va(xen_start_info->shared_info);

1080 1081 1082 1083
#ifndef CONFIG_SMP
	/* In UP this is as good a place as any to set up shared info */
	xen_setup_vcpu_info_placement();
#endif
J
Jeremy Fitzhardinge 已提交
1084 1085

	xen_setup_mfn_list_list();
1086 1087
}

1088
/* This is called once we have the cpu_possible_mask */
1089
void xen_setup_vcpu_info_placement(void)
1090 1091 1092 1093 1094 1095 1096 1097 1098
{
	int cpu;

	for_each_possible_cpu(cpu)
		xen_vcpu_setup(cpu);

	/* xen_vcpu_setup managed to place the vcpu_info within the
	   percpu area for all cpus, so make use of it */
	if (have_vcpu_info_placement) {
1099 1100 1101 1102
		pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
		pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
		pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
		pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
1103
		pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
1104
	}
1105 1106
}

1107 1108
static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
			  unsigned long addr, unsigned len)
1109 1110 1111 1112 1113 1114
{
	char *start, *end, *reloc;
	unsigned ret;

	start = end = reloc = NULL;

1115 1116
#define SITE(op, x)							\
	case PARAVIRT_PATCH(op.x):					\
1117 1118 1119 1120 1121 1122 1123 1124
	if (have_vcpu_info_placement) {					\
		start = (char *)xen_##x##_direct;			\
		end = xen_##x##_direct_end;				\
		reloc = xen_##x##_direct_reloc;				\
	}								\
	goto patch_site

	switch (type) {
1125 1126 1127 1128
		SITE(pv_irq_ops, irq_enable);
		SITE(pv_irq_ops, irq_disable);
		SITE(pv_irq_ops, save_fl);
		SITE(pv_irq_ops, restore_fl);
1129 1130 1131 1132 1133 1134
#undef SITE

	patch_site:
		if (start == NULL || (end-start) > len)
			goto default_patch;

1135
		ret = paravirt_patch_insns(insnbuf, len, start, end);
1136 1137 1138 1139 1140 1141 1142

		/* Note: because reloc is assigned from something that
		   appears to be an array, gcc assumes it's non-null,
		   but doesn't know its relationship with start and
		   end. */
		if (reloc > start && reloc < end) {
			int reloc_off = reloc - start;
1143 1144
			long *relocp = (long *)(insnbuf + reloc_off);
			long delta = start - (char *)addr;
1145 1146 1147 1148 1149 1150 1151

			*relocp += delta;
		}
		break;

	default_patch:
	default:
1152 1153
		ret = paravirt_patch_default(type, clobbers, insnbuf,
					     addr, len);
1154 1155 1156 1157 1158 1159
		break;
	}

	return ret;
}

1160
static const struct pv_info xen_info __initconst = {
1161 1162 1163
	.paravirt_enabled = 1,
	.shared_kernel_pmd = 0,

1164 1165 1166 1167
#ifdef CONFIG_X86_64
	.extra_user_64bit_cs = FLAT_USER_CS64,
#endif

1168
	.name = "Xen",
1169
};
1170

1171
static const struct pv_init_ops xen_init_ops __initconst = {
1172
	.patch = xen_patch,
1173
};
1174

1175
static const struct pv_cpu_ops xen_cpu_ops __initconst = {
1176 1177 1178 1179 1180
	.cpuid = xen_cpuid,

	.set_debugreg = xen_set_debugreg,
	.get_debugreg = xen_get_debugreg,

1181
	.clts = xen_clts,
1182

1183
	.read_cr0 = xen_read_cr0,
1184
	.write_cr0 = xen_write_cr0,
1185 1186 1187 1188 1189

	.read_cr4 = native_read_cr4,
	.read_cr4_safe = native_read_cr4_safe,
	.write_cr4 = xen_write_cr4,

1190 1191 1192 1193 1194
#ifdef CONFIG_X86_64
	.read_cr8 = xen_read_cr8,
	.write_cr8 = xen_write_cr8,
#endif

1195 1196 1197
	.wbinvd = native_wbinvd,

	.read_msr = native_read_msr_safe,
1198
	.write_msr = xen_write_msr_safe,
1199

1200 1201 1202
	.read_tsc = native_read_tsc,
	.read_pmc = native_read_pmc,

1203 1204
	.read_tscp = native_read_tscp,

1205
	.iret = xen_iret,
1206
	.irq_enable_sysexit = xen_sysexit,
1207 1208 1209 1210
#ifdef CONFIG_X86_64
	.usergs_sysret32 = xen_sysret32,
	.usergs_sysret64 = xen_sysret64,
#endif
1211 1212 1213 1214 1215 1216

	.load_tr_desc = paravirt_nop,
	.set_ldt = xen_set_ldt,
	.load_gdt = xen_load_gdt,
	.load_idt = xen_load_idt,
	.load_tls = xen_load_tls,
1217 1218 1219
#ifdef CONFIG_X86_64
	.load_gs_index = xen_load_gs_index,
#endif
1220

1221 1222 1223
	.alloc_ldt = xen_alloc_ldt,
	.free_ldt = xen_free_ldt,

1224 1225 1226 1227 1228 1229 1230
	.store_gdt = native_store_gdt,
	.store_idt = native_store_idt,
	.store_tr = xen_store_tr,

	.write_ldt_entry = xen_write_ldt_entry,
	.write_gdt_entry = xen_write_gdt_entry,
	.write_idt_entry = xen_write_idt_entry,
1231
	.load_sp0 = xen_load_sp0,
1232 1233 1234 1235

	.set_iopl_mask = xen_set_iopl_mask,
	.io_delay = xen_io_delay,

J
Jeremy Fitzhardinge 已提交
1236 1237 1238
	/* Xen takes care of %gs when switching to usermode for us */
	.swapgs = paravirt_nop,

1239 1240
	.start_context_switch = paravirt_start_context_switch,
	.end_context_switch = xen_end_context_switch,
1241 1242
};

1243
static const struct pv_apic_ops xen_apic_ops __initconst = {
1244 1245 1246
#ifdef CONFIG_X86_LOCAL_APIC
	.startup_ipi_hook = paravirt_nop,
#endif
1247 1248
};

J
Jeremy Fitzhardinge 已提交
1249 1250
static void xen_reboot(int reason)
{
J
Jeremy Fitzhardinge 已提交
1251 1252 1253
	struct sched_shutdown r = { .reason = reason };

	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
J
Jeremy Fitzhardinge 已提交
1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
		BUG();
}

static void xen_restart(char *msg)
{
	xen_reboot(SHUTDOWN_reboot);
}

static void xen_emergency_restart(void)
{
	xen_reboot(SHUTDOWN_reboot);
}

static void xen_machine_halt(void)
{
	xen_reboot(SHUTDOWN_poweroff);
}

1272 1273 1274 1275 1276 1277 1278
static void xen_machine_power_off(void)
{
	if (pm_power_off)
		pm_power_off();
	xen_reboot(SHUTDOWN_poweroff);
}

J
Jeremy Fitzhardinge 已提交
1279 1280 1281 1282 1283
static void xen_crash_shutdown(struct pt_regs *regs)
{
	xen_reboot(SHUTDOWN_crash);
}

1284 1285 1286
static int
xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr)
{
1287
	xen_reboot(SHUTDOWN_crash);
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300
	return NOTIFY_DONE;
}

static struct notifier_block xen_panic_block = {
	.notifier_call= xen_panic_event,
};

int xen_panic_handler_init(void)
{
	atomic_notifier_chain_register(&panic_notifier_list, &xen_panic_block);
	return 0;
}

1301
static const struct machine_ops xen_machine_ops __initconst = {
J
Jeremy Fitzhardinge 已提交
1302 1303
	.restart = xen_restart,
	.halt = xen_machine_halt,
1304
	.power_off = xen_machine_power_off,
J
Jeremy Fitzhardinge 已提交
1305 1306 1307 1308 1309
	.shutdown = xen_machine_halt,
	.crash_shutdown = xen_crash_shutdown,
	.emergency_restart = xen_emergency_restart,
};

1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
static void __init xen_boot_params_init_edd(void)
{
#if IS_ENABLED(CONFIG_EDD)
	struct xen_platform_op op;
	struct edd_info *edd_info;
	u32 *mbr_signature;
	unsigned nr;
	int ret;

	edd_info = boot_params.eddbuf;
	mbr_signature = boot_params.edd_mbr_sig_buffer;

	op.cmd = XENPF_firmware_info;

	op.u.firmware_info.type = XEN_FW_DISK_INFO;
	for (nr = 0; nr < EDDMAXNR; nr++) {
		struct edd_info *info = edd_info + nr;

		op.u.firmware_info.index = nr;
		info->params.length = sizeof(info->params);
		set_xen_guest_handle(op.u.firmware_info.u.disk_info.edd_params,
				     &info->params);
		ret = HYPERVISOR_dom0_op(&op);
		if (ret)
			break;

#define C(x) info->x = op.u.firmware_info.u.disk_info.x
		C(device);
		C(version);
		C(interface_support);
		C(legacy_max_cylinder);
		C(legacy_max_head);
		C(legacy_sectors_per_track);
#undef C
	}
	boot_params.eddbuf_entries = nr;

	op.u.firmware_info.type = XEN_FW_DISK_MBR_SIGNATURE;
	for (nr = 0; nr < EDD_MBR_SIG_MAX; nr++) {
		op.u.firmware_info.index = nr;
		ret = HYPERVISOR_dom0_op(&op);
		if (ret)
			break;
		mbr_signature[nr] = op.u.firmware_info.u.disk_mbr_signature.mbr_signature;
	}
	boot_params.edd_mbr_sig_buf_entries = nr;
#endif
}

1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375
/*
 * Set up the GDT and segment registers for -fstack-protector.  Until
 * we do this, we have to be careful not to call any stack-protected
 * function, which is most of the kernel.
 */
static void __init xen_setup_stackprotector(void)
{
	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
	pv_cpu_ops.load_gdt = xen_load_gdt_boot;

	setup_stack_canary_segment(0);
	switch_to_new_gdt(0);

	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
	pv_cpu_ops.load_gdt = xen_load_gdt;
}

1376 1377 1378
/* First C function to be called on Xen boot */
asmlinkage void __init xen_start_kernel(void)
{
1379 1380
	struct physdev_set_iopl set_iopl;
	int rc;
1381 1382 1383 1384

	if (!xen_start_info)
		return;

1385 1386
	xen_domain_type = XEN_PV_DOMAIN;

1387 1388
	xen_setup_machphys_mapping();

1389
	/* Install Xen paravirt ops */
1390 1391 1392 1393 1394
	pv_info = xen_info;
	pv_init_ops = xen_init_ops;
	pv_cpu_ops = xen_cpu_ops;
	pv_apic_ops = xen_apic_ops;

1395
	x86_init.resources.memory_setup = xen_memory_setup;
1396
	x86_init.oem.arch_setup = xen_arch_setup;
1397
	x86_init.oem.banner = xen_banner;
1398

1399
	xen_init_time_ops();
1400

1401
	/*
1402
	 * Set up some pagetable state before starting to set any ptes.
1403
	 */
1404

1405 1406
	xen_init_mmu_ops();

1407 1408
	/* Prevent unwanted bits from being set in PTEs. */
	__supported_pte_mask &= ~_PAGE_GLOBAL;
1409
#if 0
1410
	if (!xen_initial_domain())
1411
#endif
1412 1413 1414 1415
		__supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);

	__supported_pte_mask |= _PAGE_IOMAP;

1416 1417 1418 1419 1420 1421
	/*
	 * Prevent page tables from being allocated in highmem, even
	 * if CONFIG_HIGHPTE is enabled.
	 */
	__userpte_alloc_gfp &= ~__GFP_HIGHMEM;

1422
	/* Work out if we support NX */
1423
	x86_configure_nx();
1424

1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435
	xen_setup_features();

	/* Get mfn list */
	if (!xen_feature(XENFEAT_auto_translated_physmap))
		xen_build_dynamic_phys_to_machine();

	/*
	 * Set up kernel GDT and segment registers, mainly so that
	 * -fstack-protector code can be executed.
	 */
	xen_setup_stackprotector();
1436

1437
	xen_init_irq_ops();
J
Jeremy Fitzhardinge 已提交
1438 1439
	xen_init_cpuid_mask();

1440
#ifdef CONFIG_X86_LOCAL_APIC
1441
	/*
1442
	 * set up the basic apic ops.
1443
	 */
Y
Yinghai Lu 已提交
1444
	set_xen_basic_apic_ops();
1445
#endif
1446

1447 1448 1449 1450 1451
	if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
		pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
		pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
	}

J
Jeremy Fitzhardinge 已提交
1452 1453
	machine_ops = xen_machine_ops;

1454 1455 1456 1457 1458 1459
	/*
	 * The only reliable way to retain the initial address of the
	 * percpu gdt_page is to remember it here, so we can go and
	 * mark it RW later, when the initial percpu area is freed.
	 */
	xen_initial_gdt = &per_cpu(gdt_page, 0);
1460

1461
	xen_smp_init();
1462

1463 1464 1465 1466 1467 1468 1469 1470
#ifdef CONFIG_ACPI_NUMA
	/*
	 * The pages we from Xen are not related to machine pages, so
	 * any NUMA information the kernel tries to get from ACPI will
	 * be meaningless.  Prevent it from trying.
	 */
	acpi_numa = -1;
#endif
1471 1472 1473 1474 1475 1476 1477 1478
#ifdef CONFIG_X86_PAT
	/*
	 * For right now disable the PAT. We should remove this once
	 * git commit 8eaffa67b43e99ae581622c5133e20b0f48bcef1
	 * (xen/pat: Disable PAT support for now) is reverted.
	 */
	pat_enabled = 0;
#endif
1479
	/* Don't do the full vcpu_info placement stuff until we have a
1480
	   possible map and a non-dummy shared_info. */
1481
	per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
1482

1483
	local_irq_disable();
1484
	early_boot_irqs_disabled = true;
1485

J
Jeremy Fitzhardinge 已提交
1486
	xen_raw_console_write("mapping kernel into physical memory\n");
1487
	xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base, xen_start_info->nr_pages);
1488

1489 1490 1491
	/* Allocate and initialize top and mid mfn levels for p2m structure */
	xen_build_mfn_list_list();

1492 1493
	/* keep using Xen gdt for now; no urgent need to change it */

1494
#ifdef CONFIG_X86_32
1495
	pv_info.kernel_rpl = 1;
1496
	if (xen_feature(XENFEAT_supervisor_mode_kernel))
1497
		pv_info.kernel_rpl = 0;
1498 1499 1500
#else
	pv_info.kernel_rpl = 0;
#endif
1501
	/* set the limit of our address space */
1502
	xen_reserve_top();
1503

1504 1505 1506 1507 1508 1509 1510 1511 1512
	/* We used to do this in xen_arch_setup, but that is too late on AMD
	 * were early_cpu_init (run before ->arch_setup()) calls early_amd_init
	 * which pokes 0xcf8 port.
	 */
	set_iopl.iopl = 1;
	rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
	if (rc != 0)
		xen_raw_printk("physdev_op failed %d\n", rc);

1513
#ifdef CONFIG_X86_32
1514 1515 1516
	/* set up basic CPUID stuff */
	cpu_detect(&new_cpu_data);
	new_cpu_data.hard_math = 1;
1517
	new_cpu_data.wp_works_ok = 1;
1518
	new_cpu_data.x86_capability[0] = cpuid_edx(1);
1519
#endif
1520 1521

	/* Poke various useful things into boot_params */
1522 1523 1524 1525
	boot_params.hdr.type_of_loader = (9 << 4) | 0;
	boot_params.hdr.ramdisk_image = xen_start_info->mod_start
		? __pa(xen_start_info->mod_start) : 0;
	boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
1526
	boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
1527

1528
	if (!xen_initial_domain()) {
1529
		add_preferred_console("xenboot", 0, NULL);
1530
		add_preferred_console("tty", 0, NULL);
1531
		add_preferred_console("hvc", 0, NULL);
1532 1533
		if (pci_xen)
			x86_init.pci.arch_init = pci_xen_init;
C
Chris Wright 已提交
1534
	} else {
1535 1536 1537
		const struct dom0_vga_console_info *info =
			(void *)((char *)xen_start_info +
				 xen_start_info->console.dom0.info_off);
1538 1539 1540 1541 1542
		struct xen_platform_op op = {
			.cmd = XENPF_firmware_info,
			.interface_version = XENPF_INTERFACE_VERSION,
			.u.firmware_info.type = XEN_FW_KBD_SHIFT_FLAGS,
		};
1543 1544 1545 1546 1547

		xen_init_vga(info, xen_start_info->console.dom0.info_size);
		xen_start_info->console.domU.mfn = 0;
		xen_start_info->console.domU.evtchn = 0;

1548 1549 1550
		if (HYPERVISOR_dom0_op(&op) == 0)
			boot_params.kbd_status = op.u.firmware_info.u.kbd_shift_flags;

1551 1552
		xen_init_apic();

C
Chris Wright 已提交
1553 1554
		/* Make sure ACS will be enabled */
		pci_request_acs();
1555 1556

		xen_acpi_sleep_register();
1557 1558 1559 1560

		/* Avoid searching for BIOS MP tables */
		x86_init.mpparse.find_smp_config = x86_init_noop;
		x86_init.mpparse.get_smp_config = x86_init_uint_noop;
1561 1562

		xen_boot_params_init_edd();
1563
	}
1564 1565 1566 1567
#ifdef CONFIG_PCI
	/* PCI BIOS service won't work from a PV guest. */
	pci_probe &= ~PCI_PROBE_BIOS;
#endif
J
Jeremy Fitzhardinge 已提交
1568 1569
	xen_raw_console_write("about to get started...\n");

1570 1571
	xen_setup_runstate_info(0);

1572
	/* Start the world */
1573
#ifdef CONFIG_X86_32
1574
	i386_start_kernel();
1575
#else
J
Jeremy Fitzhardinge 已提交
1576
	x86_64_start_reservations((char *)__pa_symbol(&boot_params));
1577
#endif
1578
}
1579

1580
void __ref xen_hvm_init_shared_info(void)
1581
{
1582
	int cpu;
1583
	struct xen_add_to_physmap xatp;
1584
	static struct shared_info *shared_info_page = 0;
1585

1586 1587 1588
	if (!shared_info_page)
		shared_info_page = (struct shared_info *)
			extend_brk(PAGE_SIZE, PAGE_SIZE);
1589 1590 1591
	xatp.domid = DOMID_SELF;
	xatp.idx = 0;
	xatp.space = XENMAPSPACE_shared_info;
1592
	xatp.gpfn = __pa(shared_info_page) >> PAGE_SHIFT;
1593 1594 1595
	if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
		BUG();

1596
	HYPERVISOR_shared_info = (struct shared_info *)shared_info_page;
1597

1598 1599 1600 1601
	/* xen_vcpu is a pointer to the vcpu_info struct in the shared_info
	 * page, we use it in the event channel upcall and in some pvclock
	 * related functions. We don't need the vcpu_info placement
	 * optimizations because we don't use any pv_mmu or pv_irq op on
1602 1603 1604 1605 1606
	 * HVM.
	 * When xen_hvm_init_shared_info is run at boot time only vcpu 0 is
	 * online but xen_hvm_init_shared_info is run at resume time too and
	 * in that case multiple vcpus might be online. */
	for_each_online_cpu(cpu) {
1607 1608
		per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
	}
1609 1610
}

1611
#ifdef CONFIG_XEN_PVHVM
O
Olaf Hering 已提交
1612 1613
static void __init init_hvm_pv_info(void)
{
1614
	int major, minor;
1615
	uint32_t eax, ebx, ecx, edx, pages, msr, base;
O
Olaf Hering 已提交
1616 1617 1618
	u64 pfn;

	base = xen_cpuid_base();
1619 1620 1621 1622 1623 1624
	cpuid(base + 1, &eax, &ebx, &ecx, &edx);

	major = eax >> 16;
	minor = eax & 0xffff;
	printk(KERN_INFO "Xen version %d.%d.\n", major, minor);

O
Olaf Hering 已提交
1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
	cpuid(base + 2, &pages, &msr, &ecx, &edx);

	pfn = __pa(hypercall_page);
	wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));

	xen_setup_features();

	pv_info.name = "Xen HVM";

	xen_domain_type = XEN_HVM_DOMAIN;
}

1637 1638 1639 1640 1641 1642
static int __cpuinit xen_hvm_cpu_notify(struct notifier_block *self,
				    unsigned long action, void *hcpu)
{
	int cpu = (long)hcpu;
	switch (action) {
	case CPU_UP_PREPARE:
1643
		xen_vcpu_setup(cpu);
1644 1645
		if (xen_have_vector_callback)
			xen_init_lock_cpu(cpu);
1646 1647 1648 1649 1650 1651 1652
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

1653
static struct notifier_block xen_hvm_cpu_notifier __cpuinitdata = {
1654 1655 1656
	.notifier_call	= xen_hvm_cpu_notify,
};

1657 1658
static void __init xen_hvm_guest_init(void)
{
O
Olaf Hering 已提交
1659
	init_hvm_pv_info();
1660

1661
	xen_hvm_init_shared_info();
1662 1663 1664

	if (xen_feature(XENFEAT_hvm_callback_vector))
		xen_have_vector_callback = 1;
1665
	xen_hvm_smp_init();
1666
	register_cpu_notifier(&xen_hvm_cpu_notifier);
1667
	xen_unplug_emulated_devices();
1668
	x86_init.irqs.intr_init = xen_init_IRQ;
1669
	xen_hvm_init_time_ops();
1670
	xen_hvm_init_mmu_ops();
1671 1672 1673 1674 1675 1676 1677
}

static bool __init xen_hvm_platform(void)
{
	if (xen_pv_domain())
		return false;

1678
	if (!xen_cpuid_base())
1679 1680 1681 1682 1683
		return false;

	return true;
}

S
Sheng Yang 已提交
1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
bool xen_hvm_need_lapic(void)
{
	if (xen_pv_domain())
		return false;
	if (!xen_hvm_domain())
		return false;
	if (xen_feature(XENFEAT_hvm_pirqs) && xen_have_vector_callback)
		return false;
	return true;
}
EXPORT_SYMBOL_GPL(xen_hvm_need_lapic);

1696
const struct hypervisor_x86 x86_hyper_xen_hvm __refconst = {
1697 1698 1699
	.name			= "Xen HVM",
	.detect			= xen_hvm_platform,
	.init_platform		= xen_hvm_guest_init,
1700
	.x2apic_available	= xen_x2apic_para_available,
1701 1702
};
EXPORT_SYMBOL(x86_hyper_xen_hvm);
1703
#endif