enlighten.c 43.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * Core of Xen paravirt_ops implementation.
 *
 * This file contains the xen_paravirt_ops structure itself, and the
 * implementations for:
 * - privileged instructions
 * - interrupt flags
 * - segment operations
 * - booting and setup
 *
 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
 */

14
#include <linux/cpu.h>
15 16 17 18
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/preempt.h>
19
#include <linux/hardirq.h>
20 21 22 23
#include <linux/percpu.h>
#include <linux/delay.h>
#include <linux/start_kernel.h>
#include <linux/sched.h>
24
#include <linux/kprobes.h>
25 26
#include <linux/bootmem.h>
#include <linux/module.h>
27 28 29
#include <linux/mm.h>
#include <linux/page-flags.h>
#include <linux/highmem.h>
30
#include <linux/console.h>
C
Chris Wright 已提交
31
#include <linux/pci.h>
32
#include <linux/gfp.h>
33
#include <linux/memblock.h>
34
#include <linux/edd.h>
35

36
#include <xen/xen.h>
37
#include <xen/events.h>
38
#include <xen/interface/xen.h>
39
#include <xen/interface/version.h>
40 41
#include <xen/interface/physdev.h>
#include <xen/interface/vcpu.h>
42
#include <xen/interface/memory.h>
43
#include <xen/interface/nmi.h>
44
#include <xen/interface/xen-mca.h>
45 46
#include <xen/features.h>
#include <xen/page.h>
47
#include <xen/hvm.h>
J
Jeremy Fitzhardinge 已提交
48
#include <xen/hvc-console.h>
49
#include <xen/acpi.h>
50 51

#include <asm/paravirt.h>
I
Ingo Molnar 已提交
52
#include <asm/apic.h>
53
#include <asm/page.h>
54
#include <asm/xen/pci.h>
55 56 57 58
#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>
#include <asm/fixmap.h>
#include <asm/processor.h>
59
#include <asm/proto.h>
60
#include <asm/msr-index.h>
61
#include <asm/traps.h>
62 63
#include <asm/setup.h>
#include <asm/desc.h>
64
#include <asm/pgalloc.h>
65
#include <asm/pgtable.h>
J
Jeremy Fitzhardinge 已提交
66
#include <asm/tlbflush.h>
J
Jeremy Fitzhardinge 已提交
67
#include <asm/reboot.h>
68
#include <asm/stackprotector.h>
69
#include <asm/hypervisor.h>
70
#include <asm/mach_traps.h>
71
#include <asm/mwait.h>
72
#include <asm/pci_x86.h>
73
#include <asm/pat.h>
74 75 76 77 78 79 80 81

#ifdef CONFIG_ACPI
#include <linux/acpi.h>
#include <asm/acpi.h>
#include <acpi/pdc_intel.h>
#include <acpi/processor.h>
#include <xen/interface/platform.h>
#endif
82 83

#include "xen-ops.h"
J
Jeremy Fitzhardinge 已提交
84
#include "mmu.h"
B
Ben Guthro 已提交
85
#include "smp.h"
86 87 88 89
#include "multicalls.h"

EXPORT_SYMBOL_GPL(hypercall_page);

90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
/*
 * Pointer to the xen_vcpu_info structure or
 * &HYPERVISOR_shared_info->vcpu_info[cpu]. See xen_hvm_init_shared_info
 * and xen_vcpu_setup for details. By default it points to share_info->vcpu_info
 * but if the hypervisor supports VCPUOP_register_vcpu_info then it can point
 * to xen_vcpu_info. The pointer is used in __xen_evtchn_do_upcall to
 * acknowledge pending events.
 * Also more subtly it is used by the patched version of irq enable/disable
 * e.g. xen_irq_enable_direct and xen_iret in PV mode.
 *
 * The desire to be able to do those mask/unmask operations as a single
 * instruction by using the per-cpu offset held in %gs is the real reason
 * vcpu info is in a per-cpu pointer and the original reason for this
 * hypercall.
 *
 */
106
DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
107 108 109 110 111 112

/*
 * Per CPU pages used if hypervisor supports VCPUOP_register_vcpu_info
 * hypercall. This can be used both in PV and PVHVM mode. The structure
 * overrides the default per_cpu(xen_vcpu, cpu) value.
 */
113
DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
114

115 116 117
enum xen_domain_type xen_domain_type = XEN_NATIVE;
EXPORT_SYMBOL_GPL(xen_domain_type);

118 119
unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START;
EXPORT_SYMBOL(machine_to_phys_mapping);
120 121
unsigned long  machine_to_phys_nr;
EXPORT_SYMBOL(machine_to_phys_nr);
122

123 124 125
struct start_info *xen_start_info;
EXPORT_SYMBOL_GPL(xen_start_info);

126
struct shared_info xen_dummy_shared_info;
127

128 129
void *xen_initial_gdt;

130
RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
131 132
__read_mostly int xen_have_vector_callback;
EXPORT_SYMBOL_GPL(xen_have_vector_callback);
133

134 135 136 137
/*
 * Point at some empty memory to start with. We map the real shared_info
 * page as soon as fixmap is up and running.
 */
138
struct shared_info *HYPERVISOR_shared_info = &xen_dummy_shared_info;
139 140 141 142 143 144 145 146 147 148 149 150 151 152

/*
 * Flag to determine whether vcpu info placement is available on all
 * VCPUs.  We assume it is to start with, and then set it to zero on
 * the first failure.  This is because it can succeed on some VCPUs
 * and not others, since it can involve hypervisor memory allocation,
 * or because the guest failed to guarantee all the appropriate
 * constraints on all VCPUs (ie buffer can't cross a page boundary).
 *
 * Note that any particular CPU may be using a placed vcpu structure,
 * but we can only optimise if the all are.
 *
 * 0: not available, 1: available
 */
153
static int have_vcpu_info_placement = 1;
154

155 156 157 158 159 160 161 162 163 164 165 166 167
struct tls_descs {
	struct desc_struct desc[3];
};

/*
 * Updating the 3 TLS descriptors in the GDT on every task switch is
 * surprisingly expensive so we avoid updating them if they haven't
 * changed.  Since Xen writes different descriptors than the one
 * passed in the update_descriptor hypercall we keep shadow copies to
 * compare against.
 */
static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc);

168 169 170 171 172 173 174 175
static void clamp_max_cpus(void)
{
#ifdef CONFIG_SMP
	if (setup_max_cpus > MAX_VIRT_CPUS)
		setup_max_cpus = MAX_VIRT_CPUS;
#endif
}

176
static void xen_vcpu_setup(int cpu)
177
{
178 179 180 181
	struct vcpu_register_vcpu_info info;
	int err;
	struct vcpu_info *vcpup;

182
	BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
183

184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
	/*
	 * This path is called twice on PVHVM - first during bootup via
	 * smp_init -> xen_hvm_cpu_notify, and then if the VCPU is being
	 * hotplugged: cpu_up -> xen_hvm_cpu_notify.
	 * As we can only do the VCPUOP_register_vcpu_info once lets
	 * not over-write its result.
	 *
	 * For PV it is called during restore (xen_vcpu_restore) and bootup
	 * (xen_setup_vcpu_info_placement). The hotplug mechanism does not
	 * use this function.
	 */
	if (xen_hvm_domain()) {
		if (per_cpu(xen_vcpu, cpu) == &per_cpu(xen_vcpu_info, cpu))
			return;
	}
199 200
	if (cpu < MAX_VIRT_CPUS)
		per_cpu(xen_vcpu,cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
201

202 203 204 205 206
	if (!have_vcpu_info_placement) {
		if (cpu >= MAX_VIRT_CPUS)
			clamp_max_cpus();
		return;
	}
207

208
	vcpup = &per_cpu(xen_vcpu_info, cpu);
209
	info.mfn = arbitrary_virt_to_mfn(vcpup);
210 211 212 213
	info.offset = offset_in_page(vcpup);

	/* Check to see if the hypervisor will put the vcpu_info
	   structure where we want it, which allows direct access via
214 215 216 217 218 219
	   a percpu-variable.
	   N.B. This hypercall can _only_ be called once per CPU. Subsequent
	   calls will error out with -EINVAL. This is due to the fact that
	   hypervisor has no unregister variant and this hypercall does not
	   allow to over-write info.mfn and info.offset.
	 */
220 221 222 223 224
	err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);

	if (err) {
		printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
		have_vcpu_info_placement = 0;
225
		clamp_max_cpus();
226 227 228 229 230
	} else {
		/* This cpu is using the registered vcpu info, even if
		   later ones fail to. */
		per_cpu(xen_vcpu, cpu) = vcpup;
	}
231 232
}

233 234 235 236 237 238 239
/*
 * On restore, set the vcpu placement up again.
 * If it fails, then we're in a bad state, since
 * we can't back out from using it...
 */
void xen_vcpu_restore(void)
{
240
	int cpu;
241

W
Wei Liu 已提交
242
	for_each_possible_cpu(cpu) {
243
		bool other_cpu = (cpu != smp_processor_id());
W
Wei Liu 已提交
244
		bool is_up = HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL);
245

W
Wei Liu 已提交
246
		if (other_cpu && is_up &&
247 248
		    HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL))
			BUG();
249

250
		xen_setup_runstate_info(cpu);
251

252
		if (have_vcpu_info_placement)
253 254
			xen_vcpu_setup(cpu);

W
Wei Liu 已提交
255
		if (other_cpu && is_up &&
256 257
		    HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
			BUG();
258 259 260
	}
}

261 262
static void __init xen_banner(void)
{
263 264 265 266
	unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL);
	struct xen_extraversion extra;
	HYPERVISOR_xen_version(XENVER_extraversion, &extra);

267 268 269
	pr_info("Booting paravirtualized kernel %son %s\n",
		xen_feature(XENFEAT_auto_translated_physmap) ?
			"with PVH extensions " : "", pv_info.name);
270 271
	printk(KERN_INFO "Xen version: %d.%d%s%s\n",
	       version >> 16, version & 0xffff, extra.extraversion,
272
	       xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
273
}
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
/* Check if running on Xen version (major, minor) or later */
bool
xen_running_on_version_or_later(unsigned int major, unsigned int minor)
{
	unsigned int version;

	if (!xen_domain())
		return false;

	version = HYPERVISOR_xen_version(XENVER_version, NULL);
	if ((((version >> 16) == major) && ((version & 0xffff) >= minor)) ||
		((version >> 16) > major))
		return true;
	return false;
}
289

290 291 292
#define CPUID_THERM_POWER_LEAF 6
#define APERFMPERF_PRESENT 0

J
Jeremy Fitzhardinge 已提交
293 294 295
static __read_mostly unsigned int cpuid_leaf1_edx_mask = ~0;
static __read_mostly unsigned int cpuid_leaf1_ecx_mask = ~0;

296 297 298 299
static __read_mostly unsigned int cpuid_leaf1_ecx_set_mask;
static __read_mostly unsigned int cpuid_leaf5_ecx_val;
static __read_mostly unsigned int cpuid_leaf5_edx_val;

300 301
static void xen_cpuid(unsigned int *ax, unsigned int *bx,
		      unsigned int *cx, unsigned int *dx)
302
{
303
	unsigned maskebx = ~0;
J
Jeremy Fitzhardinge 已提交
304
	unsigned maskecx = ~0;
305
	unsigned maskedx = ~0;
306
	unsigned setecx = 0;
307 308 309 310
	/*
	 * Mask out inconvenient features, to try and disable as many
	 * unsupported kernel subsystems as possible.
	 */
311 312
	switch (*ax) {
	case 1:
J
Jeremy Fitzhardinge 已提交
313
		maskecx = cpuid_leaf1_ecx_mask;
314
		setecx = cpuid_leaf1_ecx_set_mask;
J
Jeremy Fitzhardinge 已提交
315
		maskedx = cpuid_leaf1_edx_mask;
316 317
		break;

318 319 320 321 322 323 324 325
	case CPUID_MWAIT_LEAF:
		/* Synthesize the values.. */
		*ax = 0;
		*bx = 0;
		*cx = cpuid_leaf5_ecx_val;
		*dx = cpuid_leaf5_edx_val;
		return;

326 327 328 329 330
	case CPUID_THERM_POWER_LEAF:
		/* Disabling APERFMPERF for kernel usage */
		maskecx = ~(1 << APERFMPERF_PRESENT);
		break;

331 332 333 334
	case 0xb:
		/* Suppress extended topology stuff */
		maskebx = 0;
		break;
J
Jeremy Fitzhardinge 已提交
335
	}
336 337

	asm(XEN_EMULATE_PREFIX "cpuid"
338 339 340 341 342
		: "=a" (*ax),
		  "=b" (*bx),
		  "=c" (*cx),
		  "=d" (*dx)
		: "0" (*ax), "2" (*cx));
J
Jeremy Fitzhardinge 已提交
343

344
	*bx &= maskebx;
J
Jeremy Fitzhardinge 已提交
345
	*cx &= maskecx;
346
	*cx |= setecx;
347
	*dx &= maskedx;
348

349 350
}

351 352
static bool __init xen_check_mwait(void)
{
353
#ifdef CONFIG_ACPI
354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
	struct xen_platform_op op = {
		.cmd			= XENPF_set_processor_pminfo,
		.u.set_pminfo.id	= -1,
		.u.set_pminfo.type	= XEN_PM_PDC,
	};
	uint32_t buf[3];
	unsigned int ax, bx, cx, dx;
	unsigned int mwait_mask;

	/* We need to determine whether it is OK to expose the MWAIT
	 * capability to the kernel to harvest deeper than C3 states from ACPI
	 * _CST using the processor_harvest_xen.c module. For this to work, we
	 * need to gather the MWAIT_LEAF values (which the cstate.c code
	 * checks against). The hypervisor won't expose the MWAIT flag because
	 * it would break backwards compatibility; so we will find out directly
	 * from the hardware and hypercall.
	 */
	if (!xen_initial_domain())
		return false;

374 375 376 377 378 379 380
	/*
	 * When running under platform earlier than Xen4.2, do not expose
	 * mwait, to avoid the risk of loading native acpi pad driver
	 */
	if (!xen_running_on_version_or_later(4, 2))
		return false;

381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
	ax = 1;
	cx = 0;

	native_cpuid(&ax, &bx, &cx, &dx);

	mwait_mask = (1 << (X86_FEATURE_EST % 32)) |
		     (1 << (X86_FEATURE_MWAIT % 32));

	if ((cx & mwait_mask) != mwait_mask)
		return false;

	/* We need to emulate the MWAIT_LEAF and for that we need both
	 * ecx and edx. The hypercall provides only partial information.
	 */

	ax = CPUID_MWAIT_LEAF;
	bx = 0;
	cx = 0;
	dx = 0;

	native_cpuid(&ax, &bx, &cx, &dx);

	/* Ask the Hypervisor whether to clear ACPI_PDC_C_C2C3_FFH. If so,
	 * don't expose MWAIT_LEAF and let ACPI pick the IOPORT version of C3.
	 */
	buf[0] = ACPI_PDC_REVISION_ID;
	buf[1] = 1;
	buf[2] = (ACPI_PDC_C_CAPABILITY_SMP | ACPI_PDC_EST_CAPABILITY_SWSMP);

	set_xen_guest_handle(op.u.set_pminfo.pdc, buf);

	if ((HYPERVISOR_dom0_op(&op) == 0) &&
	    (buf[2] & (ACPI_PDC_C_C1_FFH | ACPI_PDC_C_C2C3_FFH))) {
		cpuid_leaf5_ecx_val = cx;
		cpuid_leaf5_edx_val = dx;
	}
	return true;
#else
	return false;
#endif
}
422
static void __init xen_init_cpuid_mask(void)
J
Jeremy Fitzhardinge 已提交
423 424
{
	unsigned int ax, bx, cx, dx;
425
	unsigned int xsave_mask;
J
Jeremy Fitzhardinge 已提交
426 427

	cpuid_leaf1_edx_mask =
428
		~((1 << X86_FEATURE_MTRR) |  /* disable MTRR */
J
Jeremy Fitzhardinge 已提交
429 430 431 432
		  (1 << X86_FEATURE_ACC));   /* thermal monitoring */

	if (!xen_initial_domain())
		cpuid_leaf1_edx_mask &=
433
			~((1 << X86_FEATURE_ACPI));  /* disable ACPI */
Z
Zhenzhong Duan 已提交
434 435 436

	cpuid_leaf1_ecx_mask &= ~(1 << (X86_FEATURE_X2APIC % 32));

437
	ax = 1;
438
	cx = 0;
439
	cpuid(1, &ax, &bx, &cx, &dx);
J
Jeremy Fitzhardinge 已提交
440

441 442 443 444 445 446 447
	xsave_mask =
		(1 << (X86_FEATURE_XSAVE % 32)) |
		(1 << (X86_FEATURE_OSXSAVE % 32));

	/* Xen will set CR4.OSXSAVE if supported and not disabled by force */
	if ((cx & xsave_mask) != xsave_mask)
		cpuid_leaf1_ecx_mask &= ~xsave_mask; /* disable XSAVE & OSXSAVE */
448 449
	if (xen_check_mwait())
		cpuid_leaf1_ecx_set_mask = (1 << (X86_FEATURE_MWAIT % 32));
J
Jeremy Fitzhardinge 已提交
450 451
}

452 453 454 455 456 457 458 459 460 461
static void xen_set_debugreg(int reg, unsigned long val)
{
	HYPERVISOR_set_debugreg(reg, val);
}

static unsigned long xen_get_debugreg(int reg)
{
	return HYPERVISOR_get_debugreg(reg);
}

462
static void xen_end_context_switch(struct task_struct *next)
463 464
{
	xen_mc_flush();
465
	paravirt_end_context_switch(next);
466 467 468 469 470 471 472
}

static unsigned long xen_store_tr(void)
{
	return 0;
}

473
/*
474 475 476 477
 * Set the page permissions for a particular virtual address.  If the
 * address is a vmalloc mapping (or other non-linear mapping), then
 * find the linear mapping of the page and also set its protections to
 * match.
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507
 */
static void set_aliased_prot(void *v, pgprot_t prot)
{
	int level;
	pte_t *ptep;
	pte_t pte;
	unsigned long pfn;
	struct page *page;

	ptep = lookup_address((unsigned long)v, &level);
	BUG_ON(ptep == NULL);

	pfn = pte_pfn(*ptep);
	page = pfn_to_page(pfn);

	pte = pfn_pte(pfn, prot);

	if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0))
		BUG();

	if (!PageHighMem(page)) {
		void *av = __va(PFN_PHYS(pfn));

		if (av != v)
			if (HYPERVISOR_update_va_mapping((unsigned long)av, pte, 0))
				BUG();
	} else
		kmap_flush_unused();
}

508 509
static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries)
{
510
	const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
511 512
	int i;

513 514
	for(i = 0; i < entries; i += entries_per_page)
		set_aliased_prot(ldt + i, PAGE_KERNEL_RO);
515 516 517 518
}

static void xen_free_ldt(struct desc_struct *ldt, unsigned entries)
{
519
	const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
520 521
	int i;

522 523
	for(i = 0; i < entries; i += entries_per_page)
		set_aliased_prot(ldt + i, PAGE_KERNEL);
524 525
}

526 527 528 529 530
static void xen_set_ldt(const void *addr, unsigned entries)
{
	struct mmuext_op *op;
	struct multicall_space mcs = xen_mc_entry(sizeof(*op));

531 532
	trace_xen_cpu_set_ldt(addr, entries);

533 534
	op = mcs.args;
	op->cmd = MMUEXT_SET_LDT;
535
	op->arg1.linear_addr = (unsigned long)addr;
536 537 538 539 540 541 542
	op->arg2.nr_ents = entries;

	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

543
static void xen_load_gdt(const struct desc_ptr *dtr)
544 545 546 547
{
	unsigned long va = dtr->address;
	unsigned int size = dtr->size + 1;
	unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
548
	unsigned long frames[pages];
549 550
	int f;

551 552 553 554
	/*
	 * A GDT can be up to 64k in size, which corresponds to 8192
	 * 8-byte entries, or 16 4k pages..
	 */
555 556 557 558 559

	BUG_ON(size > 65536);
	BUG_ON(va & ~PAGE_MASK);

	for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
J
Jeremy Fitzhardinge 已提交
560
		int level;
561
		pte_t *ptep;
J
Jeremy Fitzhardinge 已提交
562 563 564
		unsigned long pfn, mfn;
		void *virt;

565 566 567 568 569 570 571 572
		/*
		 * The GDT is per-cpu and is in the percpu data area.
		 * That can be virtually mapped, so we need to do a
		 * page-walk to get the underlying MFN for the
		 * hypercall.  The page can also be in the kernel's
		 * linear range, so we need to RO that mapping too.
		 */
		ptep = lookup_address(va, &level);
J
Jeremy Fitzhardinge 已提交
573 574 575 576 577 578 579
		BUG_ON(ptep == NULL);

		pfn = pte_pfn(*ptep);
		mfn = pfn_to_mfn(pfn);
		virt = __va(PFN_PHYS(pfn));

		frames[f] = mfn;
580

581
		make_lowmem_page_readonly((void *)va);
J
Jeremy Fitzhardinge 已提交
582
		make_lowmem_page_readonly(virt);
583 584
	}

585 586
	if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
		BUG();
587 588
}

589 590 591
/*
 * load_gdt for early boot, when the gdt is only mapped once
 */
592
static void __init xen_load_gdt_boot(const struct desc_ptr *dtr)
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
{
	unsigned long va = dtr->address;
	unsigned int size = dtr->size + 1;
	unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
	unsigned long frames[pages];
	int f;

	/*
	 * A GDT can be up to 64k in size, which corresponds to 8192
	 * 8-byte entries, or 16 4k pages..
	 */

	BUG_ON(size > 65536);
	BUG_ON(va & ~PAGE_MASK);

	for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
		pte_t pte;
		unsigned long pfn, mfn;

		pfn = virt_to_pfn(va);
		mfn = pfn_to_mfn(pfn);

		pte = pfn_pte(pfn, PAGE_KERNEL_RO);

		if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
			BUG();

		frames[f] = mfn;
	}

	if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
		BUG();
}

627 628 629 630 631 632
static inline bool desc_equal(const struct desc_struct *d1,
			      const struct desc_struct *d2)
{
	return d1->a == d2->a && d1->b == d2->b;
}

633 634 635
static void load_TLS_descriptor(struct thread_struct *t,
				unsigned int cpu, unsigned int i)
{
636 637 638 639 640 641 642 643 644 645 646 647 648
	struct desc_struct *shadow = &per_cpu(shadow_tls_desc, cpu).desc[i];
	struct desc_struct *gdt;
	xmaddr_t maddr;
	struct multicall_space mc;

	if (desc_equal(shadow, &t->tls_array[i]))
		return;

	*shadow = t->tls_array[i];

	gdt = get_cpu_gdt_table(cpu);
	maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
	mc = __xen_mc_entry(0);
649 650 651 652 653 654

	MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
}

static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
{
655
	/*
656 657 658 659 660 661 662 663
	 * XXX sleazy hack: If we're being called in a lazy-cpu zone
	 * and lazy gs handling is enabled, it means we're in a
	 * context switch, and %gs has just been saved.  This means we
	 * can zero it out to prevent faults on exit from the
	 * hypervisor if the next process has no %gs.  Either way, it
	 * has been saved, and the new value will get loaded properly.
	 * This will go away as soon as Xen has been modified to not
	 * save/restore %gs for normal hypercalls.
664 665 666 667 668 669 670 671
	 *
	 * On x86_64, this hack is not used for %gs, because gs points
	 * to KERNEL_GS_BASE (and uses it for PDA references), so we
	 * must not zero %gs on x86_64
	 *
	 * For x86_64, we need to zero %fs, otherwise we may get an
	 * exception between the new %fs descriptor being loaded and
	 * %fs being effectively cleared at __switch_to().
672
	 */
673 674
	if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
#ifdef CONFIG_X86_32
675
		lazy_load_gs(0);
676 677 678 679 680 681 682 683 684 685 686 687
#else
		loadsegment(fs, 0);
#endif
	}

	xen_mc_batch();

	load_TLS_descriptor(t, cpu, 0);
	load_TLS_descriptor(t, cpu, 1);
	load_TLS_descriptor(t, cpu, 2);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
688 689
}

690 691 692 693 694
#ifdef CONFIG_X86_64
static void xen_load_gs_index(unsigned int idx)
{
	if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx))
		BUG();
695
}
696
#endif
697 698

static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
699
				const void *ptr)
700
{
701
	xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]);
702
	u64 entry = *(u64 *)ptr;
703

704 705
	trace_xen_cpu_write_ldt_entry(dt, entrynum, entry);

706 707
	preempt_disable();

708 709 710
	xen_mc_flush();
	if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
		BUG();
711 712

	preempt_enable();
713 714
}

715
static int cvt_gate_to_trap(int vector, const gate_desc *val,
716 717
			    struct trap_info *info)
{
718 719
	unsigned long addr;

720
	if (val->type != GATE_TRAP && val->type != GATE_INTERRUPT)
721 722 723
		return 0;

	info->vector = vector;
724 725 726

	addr = gate_offset(*val);
#ifdef CONFIG_X86_64
727 728 729
	/*
	 * Look for known traps using IST, and substitute them
	 * appropriately.  The debugger ones are the only ones we care
730 731
	 * about.  Xen will handle faults like double_fault,
	 * so we should never see them.  Warn if
732 733
	 * there's an unexpected IST-using fault handler.
	 */
734 735 736 737 738 739
	if (addr == (unsigned long)debug)
		addr = (unsigned long)xen_debug;
	else if (addr == (unsigned long)int3)
		addr = (unsigned long)xen_int3;
	else if (addr == (unsigned long)stack_segment)
		addr = (unsigned long)xen_stack_segment;
740
	else if (addr == (unsigned long)double_fault) {
741 742 743 744
		/* Don't need to handle these */
		return 0;
#ifdef CONFIG_X86_MCE
	} else if (addr == (unsigned long)machine_check) {
745 746 747 748 749
		/*
		 * when xen hypervisor inject vMCE to guest,
		 * use native mce handler to handle it
		 */
		;
750
#endif
751 752 753 754 755 756
	} else if (addr == (unsigned long)nmi)
		/*
		 * Use the native version as well.
		 */
		;
	else {
757 758 759 760
		/* Some other trap using IST? */
		if (WARN_ON(val->ist != 0))
			return 0;
	}
761 762 763
#endif	/* CONFIG_X86_64 */
	info->address = addr;

764 765
	info->cs = gate_segment(*val);
	info->flags = val->dpl;
766
	/* interrupt gates clear IF */
767 768
	if (val->type == GATE_INTERRUPT)
		info->flags |= 1 << 2;
769 770 771 772 773

	return 1;
}

/* Locations of each CPU's IDT */
774
static DEFINE_PER_CPU(struct desc_ptr, idt_desc);
775 776 777

/* Set an IDT entry.  If the entry is part of the current IDT, then
   also update Xen. */
778
static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g)
779 780
{
	unsigned long p = (unsigned long)&dt[entrynum];
781 782
	unsigned long start, end;

783 784
	trace_xen_cpu_write_idt_entry(dt, entrynum, g);

785 786
	preempt_disable();

C
Christoph Lameter 已提交
787 788
	start = __this_cpu_read(idt_desc.address);
	end = start + __this_cpu_read(idt_desc.size) + 1;
789 790 791

	xen_mc_flush();

792
	native_write_idt_entry(dt, entrynum, g);
793 794 795 796 797 798

	if (p >= start && (p + 8) <= end) {
		struct trap_info info[2];

		info[1].address = 0;

799
		if (cvt_gate_to_trap(entrynum, g, &info[0]))
800 801 802
			if (HYPERVISOR_set_trap_table(info))
				BUG();
	}
803 804

	preempt_enable();
805 806
}

807
static void xen_convert_trap_info(const struct desc_ptr *desc,
J
Jeremy Fitzhardinge 已提交
808
				  struct trap_info *traps)
809 810 811
{
	unsigned in, out, count;

812
	count = (desc->size+1) / sizeof(gate_desc);
813 814 815
	BUG_ON(count > 256);

	for (in = out = 0; in < count; in++) {
816
		gate_desc *entry = (gate_desc*)(desc->address) + in;
817

818
		if (cvt_gate_to_trap(in, entry, &traps[out]))
819 820 821
			out++;
	}
	traps[out].address = 0;
J
Jeremy Fitzhardinge 已提交
822 823 824 825
}

void xen_copy_trap_info(struct trap_info *traps)
{
826
	const struct desc_ptr *desc = this_cpu_ptr(&idt_desc);
J
Jeremy Fitzhardinge 已提交
827 828 829 830 831 832 833

	xen_convert_trap_info(desc, traps);
}

/* Load a new IDT into Xen.  In principle this can be per-CPU, so we
   hold a spinlock to protect the static traps[] array (static because
   it avoids allocation, and saves stack space). */
834
static void xen_load_idt(const struct desc_ptr *desc)
J
Jeremy Fitzhardinge 已提交
835 836 837 838
{
	static DEFINE_SPINLOCK(lock);
	static struct trap_info traps[257];

839 840
	trace_xen_cpu_load_idt(desc);

J
Jeremy Fitzhardinge 已提交
841 842
	spin_lock(&lock);

843
	memcpy(this_cpu_ptr(&idt_desc), desc, sizeof(idt_desc));
844

J
Jeremy Fitzhardinge 已提交
845
	xen_convert_trap_info(desc, traps);
846 847 848 849 850 851 852 853 854 855 856

	xen_mc_flush();
	if (HYPERVISOR_set_trap_table(traps))
		BUG();

	spin_unlock(&lock);
}

/* Write a GDT descriptor entry.  Ignore LDT descriptors, since
   they're handled differently. */
static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
857
				const void *desc, int type)
858
{
859 860
	trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);

861 862
	preempt_disable();

863 864 865
	switch (type) {
	case DESC_LDT:
	case DESC_TSS:
866 867 868 869
		/* ignore */
		break;

	default: {
870
		xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]);
871 872

		xen_mc_flush();
873
		if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
874 875 876 877
			BUG();
	}

	}
878 879

	preempt_enable();
880 881
}

882 883 884 885
/*
 * Version of write_gdt_entry for use at early boot-time needed to
 * update an entry as simply as possible.
 */
886
static void __init xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
887 888
					    const void *desc, int type)
{
889 890
	trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);

891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
	switch (type) {
	case DESC_LDT:
	case DESC_TSS:
		/* ignore */
		break;

	default: {
		xmaddr_t maddr = virt_to_machine(&dt[entry]);

		if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
			dt[entry] = *(struct desc_struct *)desc;
	}

	}
}

907
static void xen_load_sp0(struct tss_struct *tss,
908
			 struct thread_struct *thread)
909
{
910 911 912
	struct multicall_space mcs;

	mcs = xen_mc_entry(0);
913
	MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
914
	xen_mc_issue(PARAVIRT_LAZY_CPU);
915
	tss->x86_tss.sp0 = thread->sp0;
916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
}

static void xen_set_iopl_mask(unsigned mask)
{
	struct physdev_set_iopl set_iopl;

	/* Force the change at ring 0. */
	set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
	HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
}

static void xen_io_delay(void)
{
}

931 932 933 934 935 936 937 938 939 940 941
static void xen_clts(void)
{
	struct multicall_space mcs;

	mcs = xen_mc_entry(0);

	MULTI_fpu_taskswitch(mcs.mc, 0);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

942 943 944 945
static DEFINE_PER_CPU(unsigned long, xen_cr0_value);

static unsigned long xen_read_cr0(void)
{
946
	unsigned long cr0 = this_cpu_read(xen_cr0_value);
947 948 949

	if (unlikely(cr0 == 0)) {
		cr0 = native_read_cr0();
950
		this_cpu_write(xen_cr0_value, cr0);
951 952 953 954 955
	}

	return cr0;
}

956 957 958 959
static void xen_write_cr0(unsigned long cr0)
{
	struct multicall_space mcs;

960
	this_cpu_write(xen_cr0_value, cr0);
961

962 963 964 965 966 967 968 969 970
	/* Only pay attention to cr0.TS; everything else is
	   ignored. */
	mcs = xen_mc_entry(0);

	MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

971 972
static void xen_write_cr4(unsigned long cr4)
{
973 974 975 976
	cr4 &= ~X86_CR4_PGE;
	cr4 &= ~X86_CR4_PSE;

	native_write_cr4(cr4);
977
}
978 979 980 981 982 983 984 985 986 987
#ifdef CONFIG_X86_64
static inline unsigned long xen_read_cr8(void)
{
	return 0;
}
static inline void xen_write_cr8(unsigned long val)
{
	BUG_ON(val);
}
#endif
988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004

static u64 xen_read_msr_safe(unsigned int msr, int *err)
{
	u64 val;

	val = native_read_msr_safe(msr, err);
	switch (msr) {
	case MSR_IA32_APICBASE:
#ifdef CONFIG_X86_X2APIC
		if (!(cpuid_ecx(1) & (1 << (X86_FEATURE_X2APIC & 31))))
#endif
			val &= ~X2APIC_ENABLE;
		break;
	}
	return val;
}

1005 1006 1007 1008 1009 1010
static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
{
	int ret;

	ret = 0;

T
Tej 已提交
1011
	switch (msr) {
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
#ifdef CONFIG_X86_64
		unsigned which;
		u64 base;

	case MSR_FS_BASE:		which = SEGBASE_FS; goto set;
	case MSR_KERNEL_GS_BASE:	which = SEGBASE_GS_USER; goto set;
	case MSR_GS_BASE:		which = SEGBASE_GS_KERNEL; goto set;

	set:
		base = ((u64)high << 32) | low;
		if (HYPERVISOR_set_segment_base(which, base) != 0)
1023
			ret = -EIO;
1024 1025
		break;
#endif
J
Jeremy Fitzhardinge 已提交
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036

	case MSR_STAR:
	case MSR_CSTAR:
	case MSR_LSTAR:
	case MSR_SYSCALL_MASK:
	case MSR_IA32_SYSENTER_CS:
	case MSR_IA32_SYSENTER_ESP:
	case MSR_IA32_SYSENTER_EIP:
		/* Fast syscall setup is all done in hypercalls, so
		   these are all ignored.  Stub them out here to stop
		   Xen console noise. */
J
Jeremy Fitzhardinge 已提交
1037

1038 1039 1040 1041 1042 1043 1044
	default:
		ret = native_write_msr_safe(msr, low, high);
	}

	return ret;
}

1045
void xen_setup_shared_info(void)
1046 1047
{
	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1048 1049 1050 1051 1052
		set_fixmap(FIX_PARAVIRT_BOOTMAP,
			   xen_start_info->shared_info);

		HYPERVISOR_shared_info =
			(struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
1053 1054 1055 1056
	} else
		HYPERVISOR_shared_info =
			(struct shared_info *)__va(xen_start_info->shared_info);

1057 1058 1059 1060
#ifndef CONFIG_SMP
	/* In UP this is as good a place as any to set up shared info */
	xen_setup_vcpu_info_placement();
#endif
J
Jeremy Fitzhardinge 已提交
1061 1062

	xen_setup_mfn_list_list();
1063 1064
}

1065
/* This is called once we have the cpu_possible_mask */
1066
void xen_setup_vcpu_info_placement(void)
1067 1068 1069 1070 1071 1072 1073
{
	int cpu;

	for_each_possible_cpu(cpu)
		xen_vcpu_setup(cpu);

	/* xen_vcpu_setup managed to place the vcpu_info within the
1074 1075 1076
	 * percpu area for all cpus, so make use of it. Note that for
	 * PVH we want to use native IRQ mechanism. */
	if (have_vcpu_info_placement && !xen_pvh_domain()) {
1077 1078 1079 1080
		pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
		pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
		pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
		pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
1081
		pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
1082
	}
1083 1084
}

1085 1086
static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
			  unsigned long addr, unsigned len)
1087 1088 1089 1090 1091 1092
{
	char *start, *end, *reloc;
	unsigned ret;

	start = end = reloc = NULL;

1093 1094
#define SITE(op, x)							\
	case PARAVIRT_PATCH(op.x):					\
1095 1096 1097 1098 1099 1100 1101 1102
	if (have_vcpu_info_placement) {					\
		start = (char *)xen_##x##_direct;			\
		end = xen_##x##_direct_end;				\
		reloc = xen_##x##_direct_reloc;				\
	}								\
	goto patch_site

	switch (type) {
1103 1104 1105 1106
		SITE(pv_irq_ops, irq_enable);
		SITE(pv_irq_ops, irq_disable);
		SITE(pv_irq_ops, save_fl);
		SITE(pv_irq_ops, restore_fl);
1107 1108 1109 1110 1111 1112
#undef SITE

	patch_site:
		if (start == NULL || (end-start) > len)
			goto default_patch;

1113
		ret = paravirt_patch_insns(insnbuf, len, start, end);
1114 1115 1116 1117 1118 1119 1120

		/* Note: because reloc is assigned from something that
		   appears to be an array, gcc assumes it's non-null,
		   but doesn't know its relationship with start and
		   end. */
		if (reloc > start && reloc < end) {
			int reloc_off = reloc - start;
1121 1122
			long *relocp = (long *)(insnbuf + reloc_off);
			long delta = start - (char *)addr;
1123 1124 1125 1126 1127 1128 1129

			*relocp += delta;
		}
		break;

	default_patch:
	default:
1130 1131
		ret = paravirt_patch_default(type, clobbers, insnbuf,
					     addr, len);
1132 1133 1134 1135 1136 1137
		break;
	}

	return ret;
}

1138
static const struct pv_info xen_info __initconst = {
1139 1140 1141
	.paravirt_enabled = 1,
	.shared_kernel_pmd = 0,

1142 1143 1144 1145
#ifdef CONFIG_X86_64
	.extra_user_64bit_cs = FLAT_USER_CS64,
#endif

1146
	.name = "Xen",
1147
};
1148

1149
static const struct pv_init_ops xen_init_ops __initconst = {
1150
	.patch = xen_patch,
1151
};
1152

1153
static const struct pv_cpu_ops xen_cpu_ops __initconst = {
1154 1155 1156 1157 1158
	.cpuid = xen_cpuid,

	.set_debugreg = xen_set_debugreg,
	.get_debugreg = xen_get_debugreg,

1159
	.clts = xen_clts,
1160

1161
	.read_cr0 = xen_read_cr0,
1162
	.write_cr0 = xen_write_cr0,
1163 1164 1165 1166 1167

	.read_cr4 = native_read_cr4,
	.read_cr4_safe = native_read_cr4_safe,
	.write_cr4 = xen_write_cr4,

1168 1169 1170 1171 1172
#ifdef CONFIG_X86_64
	.read_cr8 = xen_read_cr8,
	.write_cr8 = xen_write_cr8,
#endif

1173 1174
	.wbinvd = native_wbinvd,

1175
	.read_msr = xen_read_msr_safe,
1176
	.write_msr = xen_write_msr_safe,
1177

1178 1179 1180
	.read_tsc = native_read_tsc,
	.read_pmc = native_read_pmc,

1181 1182
	.read_tscp = native_read_tscp,

1183
	.iret = xen_iret,
1184
	.irq_enable_sysexit = xen_sysexit,
1185 1186 1187 1188
#ifdef CONFIG_X86_64
	.usergs_sysret32 = xen_sysret32,
	.usergs_sysret64 = xen_sysret64,
#endif
1189 1190 1191 1192 1193 1194

	.load_tr_desc = paravirt_nop,
	.set_ldt = xen_set_ldt,
	.load_gdt = xen_load_gdt,
	.load_idt = xen_load_idt,
	.load_tls = xen_load_tls,
1195 1196 1197
#ifdef CONFIG_X86_64
	.load_gs_index = xen_load_gs_index,
#endif
1198

1199 1200 1201
	.alloc_ldt = xen_alloc_ldt,
	.free_ldt = xen_free_ldt,

1202 1203 1204 1205 1206 1207
	.store_idt = native_store_idt,
	.store_tr = xen_store_tr,

	.write_ldt_entry = xen_write_ldt_entry,
	.write_gdt_entry = xen_write_gdt_entry,
	.write_idt_entry = xen_write_idt_entry,
1208
	.load_sp0 = xen_load_sp0,
1209 1210 1211 1212

	.set_iopl_mask = xen_set_iopl_mask,
	.io_delay = xen_io_delay,

J
Jeremy Fitzhardinge 已提交
1213 1214 1215
	/* Xen takes care of %gs when switching to usermode for us */
	.swapgs = paravirt_nop,

1216 1217
	.start_context_switch = paravirt_start_context_switch,
	.end_context_switch = xen_end_context_switch,
1218 1219
};

1220
static const struct pv_apic_ops xen_apic_ops __initconst = {
1221 1222 1223
#ifdef CONFIG_X86_LOCAL_APIC
	.startup_ipi_hook = paravirt_nop,
#endif
1224 1225
};

J
Jeremy Fitzhardinge 已提交
1226 1227
static void xen_reboot(int reason)
{
J
Jeremy Fitzhardinge 已提交
1228 1229 1230
	struct sched_shutdown r = { .reason = reason };

	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
J
Jeremy Fitzhardinge 已提交
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
		BUG();
}

static void xen_restart(char *msg)
{
	xen_reboot(SHUTDOWN_reboot);
}

static void xen_emergency_restart(void)
{
	xen_reboot(SHUTDOWN_reboot);
}

static void xen_machine_halt(void)
{
	xen_reboot(SHUTDOWN_poweroff);
}

1249 1250 1251 1252 1253 1254 1255
static void xen_machine_power_off(void)
{
	if (pm_power_off)
		pm_power_off();
	xen_reboot(SHUTDOWN_poweroff);
}

J
Jeremy Fitzhardinge 已提交
1256 1257 1258 1259 1260
static void xen_crash_shutdown(struct pt_regs *regs)
{
	xen_reboot(SHUTDOWN_crash);
}

1261 1262 1263
static int
xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr)
{
1264
	xen_reboot(SHUTDOWN_crash);
1265 1266 1267 1268 1269
	return NOTIFY_DONE;
}

static struct notifier_block xen_panic_block = {
	.notifier_call= xen_panic_event,
1270
	.priority = INT_MIN
1271 1272 1273 1274 1275 1276 1277 1278
};

int xen_panic_handler_init(void)
{
	atomic_notifier_chain_register(&panic_notifier_list, &xen_panic_block);
	return 0;
}

1279
static const struct machine_ops xen_machine_ops __initconst = {
J
Jeremy Fitzhardinge 已提交
1280 1281
	.restart = xen_restart,
	.halt = xen_machine_halt,
1282
	.power_off = xen_machine_power_off,
J
Jeremy Fitzhardinge 已提交
1283 1284 1285 1286 1287
	.shutdown = xen_machine_halt,
	.crash_shutdown = xen_crash_shutdown,
	.emergency_restart = xen_emergency_restart,
};

1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
static unsigned char xen_get_nmi_reason(void)
{
	unsigned char reason = 0;

	/* Construct a value which looks like it came from port 0x61. */
	if (test_bit(_XEN_NMIREASON_io_error,
		     &HYPERVISOR_shared_info->arch.nmi_reason))
		reason |= NMI_REASON_IOCHK;
	if (test_bit(_XEN_NMIREASON_pci_serr,
		     &HYPERVISOR_shared_info->arch.nmi_reason))
		reason |= NMI_REASON_SERR;

	return reason;
}

1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351
static void __init xen_boot_params_init_edd(void)
{
#if IS_ENABLED(CONFIG_EDD)
	struct xen_platform_op op;
	struct edd_info *edd_info;
	u32 *mbr_signature;
	unsigned nr;
	int ret;

	edd_info = boot_params.eddbuf;
	mbr_signature = boot_params.edd_mbr_sig_buffer;

	op.cmd = XENPF_firmware_info;

	op.u.firmware_info.type = XEN_FW_DISK_INFO;
	for (nr = 0; nr < EDDMAXNR; nr++) {
		struct edd_info *info = edd_info + nr;

		op.u.firmware_info.index = nr;
		info->params.length = sizeof(info->params);
		set_xen_guest_handle(op.u.firmware_info.u.disk_info.edd_params,
				     &info->params);
		ret = HYPERVISOR_dom0_op(&op);
		if (ret)
			break;

#define C(x) info->x = op.u.firmware_info.u.disk_info.x
		C(device);
		C(version);
		C(interface_support);
		C(legacy_max_cylinder);
		C(legacy_max_head);
		C(legacy_sectors_per_track);
#undef C
	}
	boot_params.eddbuf_entries = nr;

	op.u.firmware_info.type = XEN_FW_DISK_MBR_SIGNATURE;
	for (nr = 0; nr < EDD_MBR_SIG_MAX; nr++) {
		op.u.firmware_info.index = nr;
		ret = HYPERVISOR_dom0_op(&op);
		if (ret)
			break;
		mbr_signature[nr] = op.u.firmware_info.u.disk_mbr_signature.mbr_signature;
	}
	boot_params.edd_mbr_sig_buf_entries = nr;
#endif
}

1352 1353 1354 1355
/*
 * Set up the GDT and segment registers for -fstack-protector.  Until
 * we do this, we have to be careful not to call any stack-protected
 * function, which is most of the kernel.
1356 1357 1358 1359
 *
 * Note, that it is __ref because the only caller of this after init
 * is PVH which is not going to use xen_load_gdt_boot or other
 * __init functions.
1360
 */
1361
static void __ref xen_setup_gdt(int cpu)
1362
{
1363 1364 1365 1366
	if (xen_feature(XENFEAT_auto_translated_physmap)) {
#ifdef CONFIG_X86_64
		unsigned long dummy;

1367 1368
		load_percpu_segment(cpu); /* We need to access per-cpu area */
		switch_to_new_gdt(cpu); /* GDT and GS set */
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398

		/* We are switching of the Xen provided GDT to our HVM mode
		 * GDT. The new GDT has  __KERNEL_CS with CS.L = 1
		 * and we are jumping to reload it.
		 */
		asm volatile ("pushq %0\n"
			      "leaq 1f(%%rip),%0\n"
			      "pushq %0\n"
			      "lretq\n"
			      "1:\n"
			      : "=&r" (dummy) : "0" (__KERNEL_CS));

		/*
		 * While not needed, we also set the %es, %ds, and %fs
		 * to zero. We don't care about %ss as it is NULL.
		 * Strictly speaking this is not needed as Xen zeros those
		 * out (and also MSR_FS_BASE, MSR_GS_BASE, MSR_KERNEL_GS_BASE)
		 *
		 * Linux zeros them in cpu_init() and in secondary_startup_64
		 * (for BSP).
		 */
		loadsegment(es, 0);
		loadsegment(ds, 0);
		loadsegment(fs, 0);
#else
		/* PVH: TODO Implement. */
		BUG();
#endif
		return; /* PVH does not need any PV GDT ops. */
	}
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
	pv_cpu_ops.load_gdt = xen_load_gdt_boot;

	setup_stack_canary_segment(0);
	switch_to_new_gdt(0);

	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
	pv_cpu_ops.load_gdt = xen_load_gdt;
}

1409
#ifdef CONFIG_XEN_PVH
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420
/*
 * A PV guest starts with default flags that are not set for PVH, set them
 * here asap.
 */
static void xen_pvh_set_cr_flags(int cpu)
{

	/* Some of these are setup in 'secondary_startup_64'. The others:
	 * X86_CR0_TS, X86_CR0_PE, X86_CR0_ET are set by Xen for HVM guests
	 * (which PVH shared codepaths), while X86_CR0_PG is for PVH. */
	write_cr0(read_cr0() | X86_CR0_MP | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM);
M
Mukesh Rathor 已提交
1421 1422 1423 1424 1425 1426 1427 1428

	if (!cpu)
		return;
	/*
	 * For BSP, PSE PGE are set in probe_page_size_mask(), for APs
	 * set them here. For all, OSFXSR OSXMMEXCPT are set in fpu_init.
	*/
	if (cpu_has_pse)
A
Andy Lutomirski 已提交
1429
		cr4_set_bits_and_update_boot(X86_CR4_PSE);
M
Mukesh Rathor 已提交
1430 1431

	if (cpu_has_pge)
A
Andy Lutomirski 已提交
1432
		cr4_set_bits_and_update_boot(X86_CR4_PGE);
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
}

/*
 * Note, that it is ref - because the only caller of this after init
 * is PVH which is not going to use xen_load_gdt_boot or other
 * __init functions.
 */
void __ref xen_pvh_secondary_vcpu_init(int cpu)
{
	xen_setup_gdt(cpu);
	xen_pvh_set_cr_flags(cpu);
}

1446 1447 1448 1449 1450
static void __init xen_pvh_early_guest_init(void)
{
	if (!xen_feature(XENFEAT_auto_translated_physmap))
		return;

1451 1452 1453 1454
	if (!xen_feature(XENFEAT_hvm_callback_vector))
		return;

	xen_have_vector_callback = 1;
1455 1456

	xen_pvh_early_cpu_init(0, false);
1457
	xen_pvh_set_cr_flags(0);
1458 1459 1460 1461 1462

#ifdef CONFIG_X86_32
	BUG(); /* PVH: Implement proper support. */
#endif
}
1463
#endif    /* CONFIG_XEN_PVH */
1464

1465
/* First C function to be called on Xen boot */
1466
asmlinkage __visible void __init xen_start_kernel(void)
1467
{
1468
	struct physdev_set_iopl set_iopl;
1469
	unsigned long initrd_start = 0;
1470
	int rc;
1471 1472 1473 1474

	if (!xen_start_info)
		return;

1475 1476
	xen_domain_type = XEN_PV_DOMAIN;

1477
	xen_setup_features();
1478
#ifdef CONFIG_XEN_PVH
1479
	xen_pvh_early_guest_init();
1480
#endif
1481 1482
	xen_setup_machphys_mapping();

1483
	/* Install Xen paravirt ops */
1484 1485 1486
	pv_info = xen_info;
	pv_init_ops = xen_init_ops;
	pv_apic_ops = xen_apic_ops;
1487
	if (!xen_pvh_domain()) {
1488
		pv_cpu_ops = xen_cpu_ops;
1489

1490 1491 1492
		x86_platform.get_nmi_reason = xen_get_nmi_reason;
	}

1493 1494 1495 1496
	if (xen_feature(XENFEAT_auto_translated_physmap))
		x86_init.resources.memory_setup = xen_auto_xlated_memory_setup;
	else
		x86_init.resources.memory_setup = xen_memory_setup;
1497
	x86_init.oem.arch_setup = xen_arch_setup;
1498
	x86_init.oem.banner = xen_banner;
1499

1500
	xen_init_time_ops();
1501

1502
	/*
1503
	 * Set up some pagetable state before starting to set any ptes.
1504
	 */
1505

1506 1507
	xen_init_mmu_ops();

1508 1509 1510
	/* Prevent unwanted bits from being set in PTEs. */
	__supported_pte_mask &= ~_PAGE_GLOBAL;

1511 1512 1513 1514 1515 1516
	/*
	 * Prevent page tables from being allocated in highmem, even
	 * if CONFIG_HIGHPTE is enabled.
	 */
	__userpte_alloc_gfp &= ~__GFP_HIGHMEM;

1517
	/* Work out if we support NX */
1518
	x86_configure_nx();
1519

1520
	/* Get mfn list */
1521
	xen_build_dynamic_phys_to_machine();
1522 1523 1524 1525 1526

	/*
	 * Set up kernel GDT and segment registers, mainly so that
	 * -fstack-protector code can be executed.
	 */
1527
	xen_setup_gdt(0);
1528

1529
	xen_init_irq_ops();
J
Jeremy Fitzhardinge 已提交
1530 1531
	xen_init_cpuid_mask();

1532
#ifdef CONFIG_X86_LOCAL_APIC
1533
	/*
1534
	 * set up the basic apic ops.
1535
	 */
1536
	xen_init_apic();
1537
#endif
1538

1539 1540 1541 1542 1543
	if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
		pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
		pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
	}

J
Jeremy Fitzhardinge 已提交
1544 1545
	machine_ops = xen_machine_ops;

1546 1547 1548 1549 1550 1551
	/*
	 * The only reliable way to retain the initial address of the
	 * percpu gdt_page is to remember it here, so we can go and
	 * mark it RW later, when the initial percpu area is freed.
	 */
	xen_initial_gdt = &per_cpu(gdt_page, 0);
1552

1553
	xen_smp_init();
1554

1555 1556 1557 1558 1559 1560 1561
#ifdef CONFIG_ACPI_NUMA
	/*
	 * The pages we from Xen are not related to machine pages, so
	 * any NUMA information the kernel tries to get from ACPI will
	 * be meaningless.  Prevent it from trying.
	 */
	acpi_numa = -1;
1562
#endif
1563
	/* Don't do the full vcpu_info placement stuff until we have a
1564
	   possible map and a non-dummy shared_info. */
1565
	per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
1566

1567
	local_irq_disable();
1568
	early_boot_irqs_disabled = true;
1569

J
Jeremy Fitzhardinge 已提交
1570
	xen_raw_console_write("mapping kernel into physical memory\n");
1571
	xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base, xen_start_info->nr_pages);
1572

1573 1574 1575 1576 1577 1578 1579
	/*
	 * Modify the cache mode translation tables to match Xen's PAT
	 * configuration.
	 */

	pat_init_cache_modes();

1580 1581
	/* keep using Xen gdt for now; no urgent need to change it */

1582
#ifdef CONFIG_X86_32
1583
	pv_info.kernel_rpl = 1;
1584
	if (xen_feature(XENFEAT_supervisor_mode_kernel))
1585
		pv_info.kernel_rpl = 0;
1586 1587 1588
#else
	pv_info.kernel_rpl = 0;
#endif
1589
	/* set the limit of our address space */
1590
	xen_reserve_top();
1591

1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
	/* PVH: runs at default kernel iopl of 0 */
	if (!xen_pvh_domain()) {
		/*
		 * We used to do this in xen_arch_setup, but that is too late
		 * on AMD were early_cpu_init (run before ->arch_setup()) calls
		 * early_amd_init which pokes 0xcf8 port.
		 */
		set_iopl.iopl = 1;
		rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
		if (rc != 0)
			xen_raw_printk("physdev_op failed %d\n", rc);
	}
1604

1605
#ifdef CONFIG_X86_32
1606 1607
	/* set up basic CPUID stuff */
	cpu_detect(&new_cpu_data);
1608
	set_cpu_cap(&new_cpu_data, X86_FEATURE_FPU);
1609
	new_cpu_data.wp_works_ok = 1;
1610
	new_cpu_data.x86_capability[0] = cpuid_edx(1);
1611
#endif
1612

1613 1614 1615 1616 1617 1618 1619
	if (xen_start_info->mod_start) {
	    if (xen_start_info->flags & SIF_MOD_START_PFN)
		initrd_start = PFN_PHYS(xen_start_info->mod_start);
	    else
		initrd_start = __pa(xen_start_info->mod_start);
	}

1620
	/* Poke various useful things into boot_params */
1621
	boot_params.hdr.type_of_loader = (9 << 4) | 0;
1622
	boot_params.hdr.ramdisk_image = initrd_start;
1623
	boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
1624
	boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
1625

1626
	if (!xen_initial_domain()) {
1627
		add_preferred_console("xenboot", 0, NULL);
1628
		add_preferred_console("tty", 0, NULL);
1629
		add_preferred_console("hvc", 0, NULL);
1630 1631
		if (pci_xen)
			x86_init.pci.arch_init = pci_xen_init;
C
Chris Wright 已提交
1632
	} else {
1633 1634 1635
		const struct dom0_vga_console_info *info =
			(void *)((char *)xen_start_info +
				 xen_start_info->console.dom0.info_off);
1636 1637 1638 1639 1640
		struct xen_platform_op op = {
			.cmd = XENPF_firmware_info,
			.interface_version = XENPF_INTERFACE_VERSION,
			.u.firmware_info.type = XEN_FW_KBD_SHIFT_FLAGS,
		};
1641 1642 1643 1644 1645

		xen_init_vga(info, xen_start_info->console.dom0.info_size);
		xen_start_info->console.domU.mfn = 0;
		xen_start_info->console.domU.evtchn = 0;

1646 1647 1648
		if (HYPERVISOR_dom0_op(&op) == 0)
			boot_params.kbd_status = op.u.firmware_info.u.kbd_shift_flags;

C
Chris Wright 已提交
1649 1650
		/* Make sure ACS will be enabled */
		pci_request_acs();
1651 1652

		xen_acpi_sleep_register();
1653 1654 1655 1656

		/* Avoid searching for BIOS MP tables */
		x86_init.mpparse.find_smp_config = x86_init_noop;
		x86_init.mpparse.get_smp_config = x86_init_uint_noop;
1657 1658

		xen_boot_params_init_edd();
1659
	}
1660 1661 1662 1663
#ifdef CONFIG_PCI
	/* PCI BIOS service won't work from a PV guest. */
	pci_probe &= ~PCI_PROBE_BIOS;
#endif
J
Jeremy Fitzhardinge 已提交
1664 1665
	xen_raw_console_write("about to get started...\n");

1666 1667
	xen_setup_runstate_info(0);

1668
	xen_efi_init();
D
Daniel Kiper 已提交
1669

1670
	/* Start the world */
1671
#ifdef CONFIG_X86_32
1672
	i386_start_kernel();
1673
#else
1674
	cr4_init_shadow(); /* 32b kernel does this in i386_start_kernel() */
J
Jeremy Fitzhardinge 已提交
1675
	x86_64_start_reservations((char *)__pa_symbol(&boot_params));
1676
#endif
1677
}
1678

1679
void __ref xen_hvm_init_shared_info(void)
1680
{
1681
	int cpu;
1682
	struct xen_add_to_physmap xatp;
1683
	static struct shared_info *shared_info_page = 0;
1684

1685 1686 1687
	if (!shared_info_page)
		shared_info_page = (struct shared_info *)
			extend_brk(PAGE_SIZE, PAGE_SIZE);
1688 1689 1690
	xatp.domid = DOMID_SELF;
	xatp.idx = 0;
	xatp.space = XENMAPSPACE_shared_info;
1691
	xatp.gpfn = __pa(shared_info_page) >> PAGE_SHIFT;
1692 1693 1694
	if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
		BUG();

1695
	HYPERVISOR_shared_info = (struct shared_info *)shared_info_page;
1696

1697 1698 1699 1700
	/* xen_vcpu is a pointer to the vcpu_info struct in the shared_info
	 * page, we use it in the event channel upcall and in some pvclock
	 * related functions. We don't need the vcpu_info placement
	 * optimizations because we don't use any pv_mmu or pv_irq op on
1701 1702 1703 1704 1705
	 * HVM.
	 * When xen_hvm_init_shared_info is run at boot time only vcpu 0 is
	 * online but xen_hvm_init_shared_info is run at resume time too and
	 * in that case multiple vcpus might be online. */
	for_each_online_cpu(cpu) {
1706 1707 1708
		/* Leave it to be NULL. */
		if (cpu >= MAX_VIRT_CPUS)
			continue;
1709 1710
		per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
	}
1711 1712
}

1713
#ifdef CONFIG_XEN_PVHVM
O
Olaf Hering 已提交
1714 1715
static void __init init_hvm_pv_info(void)
{
1716
	int major, minor;
1717
	uint32_t eax, ebx, ecx, edx, pages, msr, base;
O
Olaf Hering 已提交
1718 1719 1720
	u64 pfn;

	base = xen_cpuid_base();
1721 1722 1723 1724 1725 1726
	cpuid(base + 1, &eax, &ebx, &ecx, &edx);

	major = eax >> 16;
	minor = eax & 0xffff;
	printk(KERN_INFO "Xen version %d.%d.\n", major, minor);

O
Olaf Hering 已提交
1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738
	cpuid(base + 2, &pages, &msr, &ecx, &edx);

	pfn = __pa(hypercall_page);
	wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));

	xen_setup_features();

	pv_info.name = "Xen HVM";

	xen_domain_type = XEN_HVM_DOMAIN;
}

1739 1740
static int xen_hvm_cpu_notify(struct notifier_block *self, unsigned long action,
			      void *hcpu)
1741 1742 1743 1744
{
	int cpu = (long)hcpu;
	switch (action) {
	case CPU_UP_PREPARE:
1745
		xen_vcpu_setup(cpu);
1746 1747 1748 1749
		if (xen_have_vector_callback) {
			if (xen_feature(XENFEAT_hvm_safe_pvclock))
				xen_setup_timer(cpu);
		}
1750 1751 1752 1753 1754 1755 1756
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

1757
static struct notifier_block xen_hvm_cpu_notifier = {
1758 1759 1760
	.notifier_call	= xen_hvm_cpu_notify,
};

1761 1762
static void __init xen_hvm_guest_init(void)
{
O
Olaf Hering 已提交
1763
	init_hvm_pv_info();
1764

1765
	xen_hvm_init_shared_info();
1766

1767 1768
	xen_panic_handler_init();

1769 1770
	if (xen_feature(XENFEAT_hvm_callback_vector))
		xen_have_vector_callback = 1;
1771
	xen_hvm_smp_init();
1772
	register_cpu_notifier(&xen_hvm_cpu_notifier);
1773
	xen_unplug_emulated_devices();
1774
	x86_init.irqs.intr_init = xen_init_IRQ;
1775
	xen_hvm_init_time_ops();
1776
	xen_hvm_init_mmu_ops();
1777 1778
}

1779 1780 1781 1782 1783 1784 1785 1786
static bool xen_nopv = false;
static __init int xen_parse_nopv(char *arg)
{
       xen_nopv = true;
       return 0;
}
early_param("xen_nopv", xen_parse_nopv);

J
Jason Wang 已提交
1787
static uint32_t __init xen_hvm_platform(void)
1788
{
1789 1790 1791
	if (xen_nopv)
		return 0;

1792
	if (xen_pv_domain())
J
Jason Wang 已提交
1793
		return 0;
1794

J
Jason Wang 已提交
1795
	return xen_cpuid_base();
1796 1797
}

S
Sheng Yang 已提交
1798 1799
bool xen_hvm_need_lapic(void)
{
1800 1801
	if (xen_nopv)
		return false;
S
Sheng Yang 已提交
1802 1803 1804 1805 1806 1807 1808 1809 1810 1811
	if (xen_pv_domain())
		return false;
	if (!xen_hvm_domain())
		return false;
	if (xen_feature(XENFEAT_hvm_pirqs) && xen_have_vector_callback)
		return false;
	return true;
}
EXPORT_SYMBOL_GPL(xen_hvm_need_lapic);

1812
const struct hypervisor_x86 x86_hyper_xen_hvm __refconst = {
1813 1814 1815
	.name			= "Xen HVM",
	.detect			= xen_hvm_platform,
	.init_platform		= xen_hvm_guest_init,
1816
	.x2apic_available	= xen_x2apic_para_available,
1817 1818
};
EXPORT_SYMBOL(x86_hyper_xen_hvm);
1819
#endif