enlighten.c 29.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Core of Xen paravirt_ops implementation.
 *
 * This file contains the xen_paravirt_ops structure itself, and the
 * implementations for:
 * - privileged instructions
 * - interrupt flags
 * - segment operations
 * - booting and setup
 *
 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/preempt.h>
18
#include <linux/hardirq.h>
19 20 21 22 23 24
#include <linux/percpu.h>
#include <linux/delay.h>
#include <linux/start_kernel.h>
#include <linux/sched.h>
#include <linux/bootmem.h>
#include <linux/module.h>
25 26 27
#include <linux/mm.h>
#include <linux/page-flags.h>
#include <linux/highmem.h>
28 29 30 31

#include <xen/interface/xen.h>
#include <xen/interface/physdev.h>
#include <xen/interface/vcpu.h>
J
Jeremy Fitzhardinge 已提交
32
#include <xen/interface/sched.h>
33 34 35 36 37 38 39 40 41 42 43 44
#include <xen/features.h>
#include <xen/page.h>

#include <asm/paravirt.h>
#include <asm/page.h>
#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>
#include <asm/fixmap.h>
#include <asm/processor.h>
#include <asm/setup.h>
#include <asm/desc.h>
#include <asm/pgtable.h>
J
Jeremy Fitzhardinge 已提交
45
#include <asm/tlbflush.h>
J
Jeremy Fitzhardinge 已提交
46
#include <asm/reboot.h>
47 48

#include "xen-ops.h"
J
Jeremy Fitzhardinge 已提交
49
#include "mmu.h"
50 51 52 53 54 55
#include "multicalls.h"

EXPORT_SYMBOL_GPL(hypercall_page);

DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72

/*
 * Note about cr3 (pagetable base) values:
 *
 * xen_cr3 contains the current logical cr3 value; it contains the
 * last set cr3.  This may not be the current effective cr3, because
 * its update may be being lazily deferred.  However, a vcpu looking
 * at its own cr3 can use this value knowing that it everything will
 * be self-consistent.
 *
 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
 * hypercall to set the vcpu cr3 is complete (so it may be a little
 * out of date, but it will never be set early).  If one vcpu is
 * looking at another vcpu's cr3 value, it should use this variable.
 */
DEFINE_PER_CPU(unsigned long, xen_cr3);	 /* cr3 stored as physaddr */
DEFINE_PER_CPU(unsigned long, xen_current_cr3);	 /* actual vcpu cr3 */
73 74 75 76

struct start_info *xen_start_info;
EXPORT_SYMBOL_GPL(xen_start_info);

77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
static /* __initdata */ struct shared_info dummy_shared_info;

/*
 * Point at some empty memory to start with. We map the real shared_info
 * page as soon as fixmap is up and running.
 */
struct shared_info *HYPERVISOR_shared_info = (void *)&dummy_shared_info;

/*
 * Flag to determine whether vcpu info placement is available on all
 * VCPUs.  We assume it is to start with, and then set it to zero on
 * the first failure.  This is because it can succeed on some VCPUs
 * and not others, since it can involve hypervisor memory allocation,
 * or because the guest failed to guarantee all the appropriate
 * constraints on all VCPUs (ie buffer can't cross a page boundary).
 *
 * Note that any particular CPU may be using a placed vcpu structure,
 * but we can only optimise if the all are.
 *
 * 0: not available, 1: available
 */
static int have_vcpu_info_placement = 1;

static void __init xen_vcpu_setup(int cpu)
101
{
102 103 104 105
	struct vcpu_register_vcpu_info info;
	int err;
	struct vcpu_info *vcpup;

106
	per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
107 108 109 110 111 112 113 114 115

	if (!have_vcpu_info_placement)
		return;		/* already tested, not available */

	vcpup = &per_cpu(xen_vcpu_info, cpu);

	info.mfn = virt_to_mfn(vcpup);
	info.offset = offset_in_page(vcpup);

116
	printk(KERN_DEBUG "trying to map vcpu_info %d at %p, mfn %llx, offset %d\n",
117 118 119 120 121 122 123 124 125 126 127 128 129 130
	       cpu, vcpup, info.mfn, info.offset);

	/* Check to see if the hypervisor will put the vcpu_info
	   structure where we want it, which allows direct access via
	   a percpu-variable. */
	err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);

	if (err) {
		printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
		have_vcpu_info_placement = 0;
	} else {
		/* This cpu is using the registered vcpu info, even if
		   later ones fail to. */
		per_cpu(xen_vcpu, cpu) = vcpup;
131

132 133 134
		printk(KERN_DEBUG "cpu %d using vcpu_info at %p\n",
		       cpu, vcpup);
	}
135 136 137 138 139
}

static void __init xen_banner(void)
{
	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
140
	       pv_info.name);
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
	printk(KERN_INFO "Hypervisor signature: %s\n", xen_start_info->magic);
}

static void xen_cpuid(unsigned int *eax, unsigned int *ebx,
		      unsigned int *ecx, unsigned int *edx)
{
	unsigned maskedx = ~0;

	/*
	 * Mask out inconvenient features, to try and disable as many
	 * unsupported kernel subsystems as possible.
	 */
	if (*eax == 1)
		maskedx = ~((1 << X86_FEATURE_APIC) |  /* disable APIC */
			    (1 << X86_FEATURE_ACPI) |  /* disable ACPI */
			    (1 << X86_FEATURE_ACC));   /* thermal monitoring */

	asm(XEN_EMULATE_PREFIX "cpuid"
		: "=a" (*eax),
		  "=b" (*ebx),
		  "=c" (*ecx),
		  "=d" (*edx)
		: "0" (*eax), "2" (*ecx));
	*edx &= maskedx;
}

static void xen_set_debugreg(int reg, unsigned long val)
{
	HYPERVISOR_set_debugreg(reg, val);
}

static unsigned long xen_get_debugreg(int reg)
{
	return HYPERVISOR_get_debugreg(reg);
}

static unsigned long xen_save_fl(void)
{
	struct vcpu_info *vcpu;
	unsigned long flags;

	vcpu = x86_read_percpu(xen_vcpu);
183

184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
	/* flag has opposite sense of mask */
	flags = !vcpu->evtchn_upcall_mask;

	/* convert to IF type flag
	   -0 -> 0x00000000
	   -1 -> 0xffffffff
	*/
	return (-flags) & X86_EFLAGS_IF;
}

static void xen_restore_fl(unsigned long flags)
{
	struct vcpu_info *vcpu;

	/* convert from IF type flag */
	flags = !(flags & X86_EFLAGS_IF);
200 201 202 203 204

	/* There's a one instruction preempt window here.  We need to
	   make sure we're don't switch CPUs between getting the vcpu
	   pointer and updating the mask. */
	preempt_disable();
205 206
	vcpu = x86_read_percpu(xen_vcpu);
	vcpu->evtchn_upcall_mask = flags;
207
	preempt_enable_no_resched();
208

209 210
	/* Doesn't matter if we get preempted here, because any
	   pending event will get dealt with anyway. */
211

212 213 214
	if (flags == 0) {
		preempt_check_resched();
		barrier(); /* unmask then check (avoid races) */
215 216
		if (unlikely(vcpu->evtchn_upcall_pending))
			force_evtchn_callback();
217
	}
218 219 220 221
}

static void xen_irq_disable(void)
{
222 223 224
	/* There's a one instruction preempt window here.  We need to
	   make sure we're don't switch CPUs between getting the vcpu
	   pointer and updating the mask. */
225
	preempt_disable();
226
	x86_read_percpu(xen_vcpu)->evtchn_upcall_mask = 1;
227 228 229 230 231 232 233
	preempt_enable_no_resched();
}

static void xen_irq_enable(void)
{
	struct vcpu_info *vcpu;

234 235 236
	/* There's a one instruction preempt window here.  We need to
	   make sure we're don't switch CPUs between getting the vcpu
	   pointer and updating the mask. */
237 238 239
	preempt_disable();
	vcpu = x86_read_percpu(xen_vcpu);
	vcpu->evtchn_upcall_mask = 0;
240
	preempt_enable_no_resched();
241

242 243
	/* Doesn't matter if we get preempted here, because any
	   pending event will get dealt with anyway. */
244

245
	barrier(); /* unmask then check (avoid races) */
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
	if (unlikely(vcpu->evtchn_upcall_pending))
		force_evtchn_callback();
}

static void xen_safe_halt(void)
{
	/* Blocking includes an implicit local_irq_enable(). */
	if (HYPERVISOR_sched_op(SCHEDOP_block, 0) != 0)
		BUG();
}

static void xen_halt(void)
{
	if (irqs_disabled())
		HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
	else
		xen_safe_halt();
}

265
static void xen_leave_lazy(void)
266
{
267
	paravirt_leave_lazy(paravirt_get_lazy_mode());
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
	xen_mc_flush();
}

static unsigned long xen_store_tr(void)
{
	return 0;
}

static void xen_set_ldt(const void *addr, unsigned entries)
{
	unsigned long linear_addr = (unsigned long)addr;
	struct mmuext_op *op;
	struct multicall_space mcs = xen_mc_entry(sizeof(*op));

	op = mcs.args;
	op->cmd = MMUEXT_SET_LDT;
	if (linear_addr) {
		/* ldt my be vmalloced, use arbitrary_virt_to_machine */
		xmaddr_t maddr;
		maddr = arbitrary_virt_to_machine((unsigned long)addr);
		linear_addr = (unsigned long)maddr.maddr;
	}
	op->arg1.linear_addr = linear_addr;
	op->arg2.nr_ents = entries;

	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

static void xen_load_gdt(const struct Xgt_desc_struct *dtr)
{
	unsigned long *frames;
	unsigned long va = dtr->address;
	unsigned int size = dtr->size + 1;
	unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
	int f;
	struct multicall_space mcs;

	/* A GDT can be up to 64k in size, which corresponds to 8192
	   8-byte entries, or 16 4k pages.. */

	BUG_ON(size > 65536);
	BUG_ON(va & ~PAGE_MASK);

	mcs = xen_mc_entry(sizeof(*frames) * pages);
	frames = mcs.args;

	for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
		frames[f] = virt_to_mfn(va);
		make_lowmem_page_readonly((void *)va);
	}

	MULTI_set_gdt(mcs.mc, frames, size / sizeof(struct desc_struct));

	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

static void load_TLS_descriptor(struct thread_struct *t,
				unsigned int cpu, unsigned int i)
{
	struct desc_struct *gdt = get_cpu_gdt_table(cpu);
	xmaddr_t maddr = virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
	struct multicall_space mc = __xen_mc_entry(0);

	MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
}

static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
{
	xen_mc_batch();

	load_TLS_descriptor(t, cpu, 0);
	load_TLS_descriptor(t, cpu, 1);
	load_TLS_descriptor(t, cpu, 2);

	xen_mc_issue(PARAVIRT_LAZY_CPU);
345 346 347 348 349 350 351 352 353 354

	/*
	 * XXX sleazy hack: If we're being called in a lazy-cpu zone,
	 * it means we're in a context switch, and %gs has just been
	 * saved.  This means we can zero it out to prevent faults on
	 * exit from the hypervisor if the next process has no %gs.
	 * Either way, it has been saved, and the new value will get
	 * loaded properly.  This will go away as soon as Xen has been
	 * modified to not save/restore %gs for normal hypercalls.
	 */
355
	if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU)
356
		loadsegment(gs, 0);
357 358 359 360 361 362 363 364 365
}

static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
				u32 low, u32 high)
{
	unsigned long lp = (unsigned long)&dt[entrynum];
	xmaddr_t mach_lp = virt_to_machine(lp);
	u64 entry = (u64)high << 32 | low;

366 367
	preempt_disable();

368 369 370
	xen_mc_flush();
	if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
		BUG();
371 372

	preempt_enable();
373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
}

static int cvt_gate_to_trap(int vector, u32 low, u32 high,
			    struct trap_info *info)
{
	u8 type, dpl;

	type = (high >> 8) & 0x1f;
	dpl = (high >> 13) & 3;

	if (type != 0xf && type != 0xe)
		return 0;

	info->vector = vector;
	info->address = (high & 0xffff0000) | (low & 0x0000ffff);
	info->cs = low >> 16;
	info->flags = dpl;
	/* interrupt gates clear IF */
	if (type == 0xe)
		info->flags |= 4;

	return 1;
}

/* Locations of each CPU's IDT */
static DEFINE_PER_CPU(struct Xgt_desc_struct, idt_desc);

/* Set an IDT entry.  If the entry is part of the current IDT, then
   also update Xen. */
static void xen_write_idt_entry(struct desc_struct *dt, int entrynum,
				u32 low, u32 high)
{
	unsigned long p = (unsigned long)&dt[entrynum];
406 407 408 409 410 411
	unsigned long start, end;

	preempt_disable();

	start = __get_cpu_var(idt_desc).address;
	end = start + __get_cpu_var(idt_desc).size + 1;
412 413 414 415 416 417 418 419 420 421 422 423 424 425

	xen_mc_flush();

	write_dt_entry(dt, entrynum, low, high);

	if (p >= start && (p + 8) <= end) {
		struct trap_info info[2];

		info[1].address = 0;

		if (cvt_gate_to_trap(entrynum, low, high, &info[0]))
			if (HYPERVISOR_set_trap_table(info))
				BUG();
	}
426 427

	preempt_enable();
428 429
}

J
Jeremy Fitzhardinge 已提交
430 431
static void xen_convert_trap_info(const struct Xgt_desc_struct *desc,
				  struct trap_info *traps)
432 433 434 435 436 437 438 439 440 441 442 443 444
{
	unsigned in, out, count;

	count = (desc->size+1) / 8;
	BUG_ON(count > 256);

	for (in = out = 0; in < count; in++) {
		const u32 *entry = (u32 *)(desc->address + in * 8);

		if (cvt_gate_to_trap(in, entry[0], entry[1], &traps[out]))
			out++;
	}
	traps[out].address = 0;
J
Jeremy Fitzhardinge 已提交
445 446 447 448
}

void xen_copy_trap_info(struct trap_info *traps)
{
449
	const struct Xgt_desc_struct *desc = &__get_cpu_var(idt_desc);
J
Jeremy Fitzhardinge 已提交
450 451 452 453 454 455 456 457 458 459 460 461 462 463

	xen_convert_trap_info(desc, traps);
}

/* Load a new IDT into Xen.  In principle this can be per-CPU, so we
   hold a spinlock to protect the static traps[] array (static because
   it avoids allocation, and saves stack space). */
static void xen_load_idt(const struct Xgt_desc_struct *desc)
{
	static DEFINE_SPINLOCK(lock);
	static struct trap_info traps[257];

	spin_lock(&lock);

464 465
	__get_cpu_var(idt_desc) = *desc;

J
Jeremy Fitzhardinge 已提交
466
	xen_convert_trap_info(desc, traps);
467 468 469 470 471 472 473 474 475 476 477 478 479

	xen_mc_flush();
	if (HYPERVISOR_set_trap_table(traps))
		BUG();

	spin_unlock(&lock);
}

/* Write a GDT descriptor entry.  Ignore LDT descriptors, since
   they're handled differently. */
static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
				u32 low, u32 high)
{
480 481
	preempt_disable();

482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
	switch ((high >> 8) & 0xff) {
	case DESCTYPE_LDT:
	case DESCTYPE_TSS:
		/* ignore */
		break;

	default: {
		xmaddr_t maddr = virt_to_machine(&dt[entry]);
		u64 desc = (u64)high << 32 | low;

		xen_mc_flush();
		if (HYPERVISOR_update_descriptor(maddr.maddr, desc))
			BUG();
	}

	}
498 499

	preempt_enable();
500 501 502
}

static void xen_load_esp0(struct tss_struct *tss,
503
			  struct thread_struct *thread)
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
{
	struct multicall_space mcs = xen_mc_entry(0);
	MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->esp0);
	xen_mc_issue(PARAVIRT_LAZY_CPU);
}

static void xen_set_iopl_mask(unsigned mask)
{
	struct physdev_set_iopl set_iopl;

	/* Force the change at ring 0. */
	set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
	HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
}

static void xen_io_delay(void)
{
}

#ifdef CONFIG_X86_LOCAL_APIC
static unsigned long xen_apic_read(unsigned long reg)
{
	return 0;
}
J
Jeremy Fitzhardinge 已提交
528 529 530 531 532 533

static void xen_apic_write(unsigned long reg, unsigned long val)
{
	/* Warn to see if there's any stray references */
	WARN_ON(1);
}
534 535 536 537
#endif

static void xen_flush_tlb(void)
{
J
Jeremy Fitzhardinge 已提交
538 539
	struct mmuext_op *op;
	struct multicall_space mcs = xen_mc_entry(sizeof(*op));
540

J
Jeremy Fitzhardinge 已提交
541 542 543 544 545
	op = mcs.args;
	op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);

	xen_mc_issue(PARAVIRT_LAZY_MMU);
546 547 548 549
}

static void xen_flush_tlb_single(unsigned long addr)
{
J
Jeremy Fitzhardinge 已提交
550 551
	struct mmuext_op *op;
	struct multicall_space mcs = xen_mc_entry(sizeof(*op));
552

J
Jeremy Fitzhardinge 已提交
553 554 555 556 557 558
	op = mcs.args;
	op->cmd = MMUEXT_INVLPG_LOCAL;
	op->arg1.linear_addr = addr & PAGE_MASK;
	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);

	xen_mc_issue(PARAVIRT_LAZY_MMU);
559 560
}

J
Jeremy Fitzhardinge 已提交
561 562 563
static void xen_flush_tlb_others(const cpumask_t *cpus, struct mm_struct *mm,
				 unsigned long va)
{
J
Jeremy Fitzhardinge 已提交
564 565 566 567
	struct {
		struct mmuext_op op;
		cpumask_t mask;
	} *args;
J
Jeremy Fitzhardinge 已提交
568
	cpumask_t cpumask = *cpus;
J
Jeremy Fitzhardinge 已提交
569
	struct multicall_space mcs;
J
Jeremy Fitzhardinge 已提交
570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585

	/*
	 * A couple of (to be removed) sanity checks:
	 *
	 * - current CPU must not be in mask
	 * - mask must exist :)
	 */
	BUG_ON(cpus_empty(cpumask));
	BUG_ON(cpu_isset(smp_processor_id(), cpumask));
	BUG_ON(!mm);

	/* If a CPU which we ran on has gone down, OK. */
	cpus_and(cpumask, cpumask, cpu_online_map);
	if (cpus_empty(cpumask))
		return;

J
Jeremy Fitzhardinge 已提交
586 587 588 589 590
	mcs = xen_mc_entry(sizeof(*args));
	args = mcs.args;
	args->mask = cpumask;
	args->op.arg2.vcpumask = &args->mask;

J
Jeremy Fitzhardinge 已提交
591
	if (va == TLB_FLUSH_ALL) {
J
Jeremy Fitzhardinge 已提交
592
		args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
J
Jeremy Fitzhardinge 已提交
593
	} else {
J
Jeremy Fitzhardinge 已提交
594 595
		args->op.cmd = MMUEXT_INVLPG_MULTI;
		args->op.arg1.linear_addr = va;
J
Jeremy Fitzhardinge 已提交
596 597
	}

J
Jeremy Fitzhardinge 已提交
598 599 600
	MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);

	xen_mc_issue(PARAVIRT_LAZY_MMU);
J
Jeremy Fitzhardinge 已提交
601 602
}

603 604 605 606 607
static void xen_write_cr2(unsigned long cr2)
{
	x86_read_percpu(xen_vcpu)->arch.cr2 = cr2;
}

608 609 610 611 612
static unsigned long xen_read_cr2(void)
{
	return x86_read_percpu(xen_vcpu)->arch.cr2;
}

613 614 615 616 617
static unsigned long xen_read_cr2_direct(void)
{
	return x86_read_percpu(xen_vcpu_info.arch.cr2);
}

618 619
static void xen_write_cr4(unsigned long cr4)
{
620 621
	/* Just ignore cr4 changes; Xen doesn't allow us to do
	   anything anyway. */
622 623 624 625 626 627 628
}

static unsigned long xen_read_cr3(void)
{
	return x86_read_percpu(xen_cr3);
}

629 630 631 632 633
static void set_current_cr3(void *v)
{
	x86_write_percpu(xen_current_cr3, (unsigned long)v);
}

634 635
static void xen_write_cr3(unsigned long cr3)
{
636 637 638 639
	struct mmuext_op *op;
	struct multicall_space mcs;
	unsigned long mfn = pfn_to_mfn(PFN_DOWN(cr3));

640 641
	BUG_ON(preemptible());

642
	mcs = xen_mc_entry(sizeof(*op));  /* disables interrupts */
643

644 645
	/* Update while interrupts are disabled, so its atomic with
	   respect to ipis */
646 647
	x86_write_percpu(xen_cr3, cr3);

648 649 650
	op = mcs.args;
	op->cmd = MMUEXT_NEW_BASEPTR;
	op->arg1.mfn = mfn;
651

652
	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
653

654 655 656
	/* Update xen_update_cr3 once the batch has actually
	   been submitted. */
	xen_mc_callback(set_current_cr3, (void *)cr3);
657

658
	xen_mc_issue(PARAVIRT_LAZY_CPU);  /* interrupts restored */
659 660
}

661 662
/* Early in boot, while setting up the initial pagetable, assume
   everything is pinned. */
663
static __init void xen_alloc_pt_init(struct mm_struct *mm, u32 pfn)
664
{
665
	BUG_ON(mem_map);	/* should only be used early */
666 667 668
	make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
}

669 670 671 672 673 674 675 676 677
static void pin_pagetable_pfn(unsigned level, unsigned long pfn)
{
	struct mmuext_op op;
	op.cmd = level;
	op.arg1.mfn = pfn_to_mfn(pfn);
	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
		BUG();
}

678 679 680
/* This needs to make sure the new pte page is pinned iff its being
   attached to a pinned pagetable. */
static void xen_alloc_pt(struct mm_struct *mm, u32 pfn)
681
{
682
	struct page *page = pfn_to_page(pfn);
683

684 685 686
	if (PagePinned(virt_to_page(mm->pgd))) {
		SetPagePinned(page);

687
		if (!PageHighMem(page)) {
688
			make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
689 690
			pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
		} else
691 692 693 694
			/* make sure there are no stray mappings of
			   this page */
			kmap_flush_unused();
	}
695 696
}

697
/* This should never happen until we're OK to use struct page */
698 699
static void xen_release_pt(u32 pfn)
{
700 701 702
	struct page *page = pfn_to_page(pfn);

	if (PagePinned(page)) {
703 704
		if (!PageHighMem(page)) {
			pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
705
			make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
706
		}
707
	}
708 709
}

710 711
#ifdef CONFIG_HIGHPTE
static void *xen_kmap_atomic_pte(struct page *page, enum km_type type)
712
{
713 714 715 716 717 718 719 720 721 722 723
	pgprot_t prot = PAGE_KERNEL;

	if (PagePinned(page))
		prot = PAGE_KERNEL_RO;

	if (0 && PageHighMem(page))
		printk("mapping highpte %lx type %d prot %s\n",
		       page_to_pfn(page), type,
		       (unsigned long)pgprot_val(prot) & _PAGE_RW ? "WRITE" : "READ");

	return kmap_atomic_prot(page, type, prot);
724
}
725
#endif
726

727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte)
{
	/* If there's an existing pte, then don't allow _PAGE_RW to be set */
	if (pte_val_ma(*ptep) & _PAGE_PRESENT)
		pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
			       pte_val_ma(pte));

	return pte;
}

/* Init-time set_pte while constructing initial pagetables, which
   doesn't allow RO pagetable pages to be remapped RW */
static __init void xen_set_pte_init(pte_t *ptep, pte_t pte)
{
	pte = mask_rw_pte(ptep, pte);

	xen_set_pte(ptep, pte);
}

746 747 748 749
static __init void xen_pagetable_setup_start(pgd_t *base)
{
	pgd_t *xen_pgd = (pgd_t *)xen_start_info->pt_base;

750
	/* special set_pte for pagetable initialization */
751
	pv_mmu_ops.set_pte = xen_set_pte_init;
752

753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774
	init_mm.pgd = base;
	/*
	 * copy top-level of Xen-supplied pagetable into place.	 For
	 * !PAE we can use this as-is, but for PAE it is a stand-in
	 * while we copy the pmd pages.
	 */
	memcpy(base, xen_pgd, PTRS_PER_PGD * sizeof(pgd_t));

	if (PTRS_PER_PMD > 1) {
		int i;
		/*
		 * For PAE, need to allocate new pmds, rather than
		 * share Xen's, since Xen doesn't like pmd's being
		 * shared between address spaces.
		 */
		for (i = 0; i < PTRS_PER_PGD; i++) {
			if (pgd_val_ma(xen_pgd[i]) & _PAGE_PRESENT) {
				pmd_t *pmd = (pmd_t *)alloc_bootmem_low_pages(PAGE_SIZE);

				memcpy(pmd, (void *)pgd_page_vaddr(xen_pgd[i]),
				       PAGE_SIZE);

775
				make_lowmem_page_readonly(pmd);
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795

				set_pgd(&base[i], __pgd(1 + __pa(pmd)));
			} else
				pgd_clear(&base[i]);
		}
	}

	/* make sure zero_page is mapped RO so we can use it in pagetables */
	make_lowmem_page_readonly(empty_zero_page);
	make_lowmem_page_readonly(base);
	/*
	 * Switch to new pagetable.  This is done before
	 * pagetable_init has done anything so that the new pages
	 * added to the table can be prepared properly for Xen.
	 */
	xen_write_cr3(__pa(base));
}

static __init void xen_pagetable_setup_done(pgd_t *base)
{
796 797
	/* This will work as long as patching hasn't happened yet
	   (which it hasn't) */
798 799
	pv_mmu_ops.alloc_pt = xen_alloc_pt;
	pv_mmu_ops.set_pte = xen_set_pte;
800

801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
		/*
		 * Create a mapping for the shared info page.
		 * Should be set_fixmap(), but shared_info is a machine
		 * address with no corresponding pseudo-phys address.
		 */
		set_pte_mfn(fix_to_virt(FIX_PARAVIRT_BOOTMAP),
			    PFN_DOWN(xen_start_info->shared_info),
			    PAGE_KERNEL);

		HYPERVISOR_shared_info =
			(struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);

	} else
		HYPERVISOR_shared_info =
			(struct shared_info *)__va(xen_start_info->shared_info);

818 819 820
	/* Actually pin the pagetable down, but we can't set PG_pinned
	   yet because the page structures don't exist yet. */
	{
821 822
		unsigned level;

823
#ifdef CONFIG_X86_PAE
824
		level = MMUEXT_PIN_L3_TABLE;
825
#else
826
		level = MMUEXT_PIN_L2_TABLE;
827
#endif
828 829

		pin_pagetable_pfn(level, PFN_DOWN(__pa(base)));
830
	}
831
}
832

833 834 835 836 837 838 839 840 841 842 843 844 845
/* This is called once we have the cpu_possible_map */
void __init xen_setup_vcpu_info_placement(void)
{
	int cpu;

	for_each_possible_cpu(cpu)
		xen_vcpu_setup(cpu);

	/* xen_vcpu_setup managed to place the vcpu_info within the
	   percpu area for all cpus, so make use of it */
	if (have_vcpu_info_placement) {
		printk(KERN_INFO "Xen: using vcpu_info placement\n");

846 847 848 849 850 851
		pv_irq_ops.save_fl = xen_save_fl_direct;
		pv_irq_ops.restore_fl = xen_restore_fl_direct;
		pv_irq_ops.irq_disable = xen_irq_disable_direct;
		pv_irq_ops.irq_enable = xen_irq_enable_direct;
		pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
		pv_cpu_ops.iret = xen_iret_direct;
852
	}
853 854
}

855 856
static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
			  unsigned long addr, unsigned len)
857 858 859 860 861 862
{
	char *start, *end, *reloc;
	unsigned ret;

	start = end = reloc = NULL;

863 864
#define SITE(op, x)							\
	case PARAVIRT_PATCH(op.x):					\
865 866 867 868 869 870 871 872
	if (have_vcpu_info_placement) {					\
		start = (char *)xen_##x##_direct;			\
		end = xen_##x##_direct_end;				\
		reloc = xen_##x##_direct_reloc;				\
	}								\
	goto patch_site

	switch (type) {
873 874 875 876
		SITE(pv_irq_ops, irq_enable);
		SITE(pv_irq_ops, irq_disable);
		SITE(pv_irq_ops, save_fl);
		SITE(pv_irq_ops, restore_fl);
877 878 879 880 881 882
#undef SITE

	patch_site:
		if (start == NULL || (end-start) > len)
			goto default_patch;

883
		ret = paravirt_patch_insns(insnbuf, len, start, end);
884 885 886 887 888 889 890

		/* Note: because reloc is assigned from something that
		   appears to be an array, gcc assumes it's non-null,
		   but doesn't know its relationship with start and
		   end. */
		if (reloc > start && reloc < end) {
			int reloc_off = reloc - start;
891 892
			long *relocp = (long *)(insnbuf + reloc_off);
			long delta = start - (char *)addr;
893 894 895 896 897 898 899

			*relocp += delta;
		}
		break;

	default_patch:
	default:
900 901
		ret = paravirt_patch_default(type, clobbers, insnbuf,
					     addr, len);
902 903 904 905 906 907
		break;
	}

	return ret;
}

908
static const struct pv_info xen_info __initdata = {
909 910 911 912
	.paravirt_enabled = 1,
	.shared_kernel_pmd = 0,

	.name = "Xen",
913
};
914

915
static const struct pv_init_ops xen_init_ops __initdata = {
916
	.patch = xen_patch,
917

918
	.banner = xen_banner,
919 920
	.memory_setup = xen_memory_setup,
	.arch_setup = xen_arch_setup,
921
	.post_allocator_init = xen_mark_init_mm_pinned,
922
};
923

924
static const struct pv_time_ops xen_time_ops __initdata = {
J
Jeremy Fitzhardinge 已提交
925
	.time_init = xen_time_init,
926

J
Jeremy Fitzhardinge 已提交
927 928 929
	.set_wallclock = xen_set_wallclock,
	.get_wallclock = xen_get_wallclock,
	.get_cpu_khz = xen_cpu_khz,
J
Jeremy Fitzhardinge 已提交
930
	.sched_clock = xen_sched_clock,
931
};
J
Jeremy Fitzhardinge 已提交
932

933
static const struct pv_cpu_ops xen_cpu_ops __initdata = {
934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975
	.cpuid = xen_cpuid,

	.set_debugreg = xen_set_debugreg,
	.get_debugreg = xen_get_debugreg,

	.clts = native_clts,

	.read_cr0 = native_read_cr0,
	.write_cr0 = native_write_cr0,

	.read_cr4 = native_read_cr4,
	.read_cr4_safe = native_read_cr4_safe,
	.write_cr4 = xen_write_cr4,

	.wbinvd = native_wbinvd,

	.read_msr = native_read_msr_safe,
	.write_msr = native_write_msr_safe,
	.read_tsc = native_read_tsc,
	.read_pmc = native_read_pmc,

	.iret = (void *)&hypercall_page[__HYPERVISOR_iret],
	.irq_enable_sysexit = NULL,  /* never called */

	.load_tr_desc = paravirt_nop,
	.set_ldt = xen_set_ldt,
	.load_gdt = xen_load_gdt,
	.load_idt = xen_load_idt,
	.load_tls = xen_load_tls,

	.store_gdt = native_store_gdt,
	.store_idt = native_store_idt,
	.store_tr = xen_store_tr,

	.write_ldt_entry = xen_write_ldt_entry,
	.write_gdt_entry = xen_write_gdt_entry,
	.write_idt_entry = xen_write_idt_entry,
	.load_esp0 = xen_load_esp0,

	.set_iopl_mask = xen_set_iopl_mask,
	.io_delay = xen_io_delay,

976 977 978 979
	.lazy_mode = {
		.enter = paravirt_enter_lazy_cpu,
		.leave = xen_leave_lazy,
	},
980 981 982 983 984 985 986 987 988 989 990
};

static const struct pv_irq_ops xen_irq_ops __initdata = {
	.init_IRQ = xen_init_IRQ,
	.save_fl = xen_save_fl,
	.restore_fl = xen_restore_fl,
	.irq_disable = xen_irq_disable,
	.irq_enable = xen_irq_enable,
	.safe_halt = xen_safe_halt,
	.halt = xen_halt,
};
991

992
static const struct pv_apic_ops xen_apic_ops __initdata = {
993
#ifdef CONFIG_X86_LOCAL_APIC
J
Jeremy Fitzhardinge 已提交
994 995
	.apic_write = xen_apic_write,
	.apic_write_atomic = xen_apic_write,
996 997 998 999 1000
	.apic_read = xen_apic_read,
	.setup_boot_clock = paravirt_nop,
	.setup_secondary_clock = paravirt_nop,
	.startup_ipi_hook = paravirt_nop,
#endif
1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
};

static const struct pv_mmu_ops xen_mmu_ops __initdata = {
	.pagetable_setup_start = xen_pagetable_setup_start,
	.pagetable_setup_done = xen_pagetable_setup_done,

	.read_cr2 = xen_read_cr2,
	.write_cr2 = xen_write_cr2,

	.read_cr3 = xen_read_cr3,
	.write_cr3 = xen_write_cr3,
1012 1013 1014 1015

	.flush_tlb_user = xen_flush_tlb,
	.flush_tlb_kernel = xen_flush_tlb,
	.flush_tlb_single = xen_flush_tlb_single,
J
Jeremy Fitzhardinge 已提交
1016
	.flush_tlb_others = xen_flush_tlb_others,
1017 1018 1019 1020

	.pte_update = paravirt_nop,
	.pte_update_defer = paravirt_nop,

1021
	.alloc_pt = xen_alloc_pt_init,
1022
	.release_pt = xen_release_pt,
1023 1024 1025 1026 1027 1028 1029
	.alloc_pd = paravirt_nop,
	.alloc_pd_clone = paravirt_nop,
	.release_pd = paravirt_nop,

#ifdef CONFIG_HIGHPTE
	.kmap_atomic_pte = xen_kmap_atomic_pte,
#endif
1030

1031
	.set_pte = NULL,	/* see xen_pagetable_setup_* */
J
Jeremy Fitzhardinge 已提交
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055
	.set_pte_at = xen_set_pte_at,
	.set_pmd = xen_set_pmd,

	.pte_val = xen_pte_val,
	.pgd_val = xen_pgd_val,

	.make_pte = xen_make_pte,
	.make_pgd = xen_make_pgd,

#ifdef CONFIG_X86_PAE
	.set_pte_atomic = xen_set_pte_atomic,
	.set_pte_present = xen_set_pte_at,
	.set_pud = xen_set_pud,
	.pte_clear = xen_pte_clear,
	.pmd_clear = xen_pmd_clear,

	.make_pmd = xen_make_pmd,
	.pmd_val = xen_pmd_val,
#endif	/* PAE */

	.activate_mm = xen_activate_mm,
	.dup_mmap = xen_dup_mmap,
	.exit_mmap = xen_exit_mmap,

1056 1057 1058 1059
	.lazy_mode = {
		.enter = paravirt_enter_lazy_mmu,
		.leave = xen_leave_lazy,
	},
1060 1061
};

J
Jeremy Fitzhardinge 已提交
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
#ifdef CONFIG_SMP
static const struct smp_ops xen_smp_ops __initdata = {
	.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
	.smp_prepare_cpus = xen_smp_prepare_cpus,
	.cpu_up = xen_cpu_up,
	.smp_cpus_done = xen_smp_cpus_done,

	.smp_send_stop = xen_smp_send_stop,
	.smp_send_reschedule = xen_smp_send_reschedule,
	.smp_call_function_mask = xen_smp_call_function_mask,
};
#endif	/* CONFIG_SMP */

J
Jeremy Fitzhardinge 已提交
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
static void xen_reboot(int reason)
{
#ifdef CONFIG_SMP
	smp_send_stop();
#endif

	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, reason))
		BUG();
}

static void xen_restart(char *msg)
{
	xen_reboot(SHUTDOWN_reboot);
}

static void xen_emergency_restart(void)
{
	xen_reboot(SHUTDOWN_reboot);
}

static void xen_machine_halt(void)
{
	xen_reboot(SHUTDOWN_poweroff);
}

static void xen_crash_shutdown(struct pt_regs *regs)
{
	xen_reboot(SHUTDOWN_crash);
}

static const struct machine_ops __initdata xen_machine_ops = {
	.restart = xen_restart,
	.halt = xen_machine_halt,
	.power_off = xen_machine_halt,
	.shutdown = xen_machine_halt,
	.crash_shutdown = xen_crash_shutdown,
	.emergency_restart = xen_emergency_restart,
};

1114

1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
static void __init xen_reserve_top(void)
{
	unsigned long top = HYPERVISOR_VIRT_START;
	struct xen_platform_parameters pp;

	if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
		top = pp.virt_start;

	reserve_top_address(-top + 2 * PAGE_SIZE);
}

1126 1127 1128 1129 1130 1131 1132 1133
/* First C function to be called on Xen boot */
asmlinkage void __init xen_start_kernel(void)
{
	pgd_t *pgd;

	if (!xen_start_info)
		return;

J
Jeremy Fitzhardinge 已提交
1134
	BUG_ON(memcmp(xen_start_info->magic, "xen-3", 5) != 0);
1135 1136

	/* Install Xen paravirt ops */
1137 1138 1139 1140 1141 1142 1143 1144
	pv_info = xen_info;
	pv_init_ops = xen_init_ops;
	pv_time_ops = xen_time_ops;
	pv_cpu_ops = xen_cpu_ops;
	pv_irq_ops = xen_irq_ops;
	pv_apic_ops = xen_apic_ops;
	pv_mmu_ops = xen_mmu_ops;

J
Jeremy Fitzhardinge 已提交
1145 1146
	machine_ops = xen_machine_ops;

J
Jeremy Fitzhardinge 已提交
1147 1148 1149
#ifdef CONFIG_SMP
	smp_ops = xen_smp_ops;
#endif
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165

	xen_setup_features();

	/* Get mfn list */
	if (!xen_feature(XENFEAT_auto_translated_physmap))
		phys_to_machine_mapping = (unsigned long *)xen_start_info->mfn_list;

	pgd = (pgd_t *)xen_start_info->pt_base;

	init_pg_tables_end = __pa(pgd) + xen_start_info->nr_pt_frames*PAGE_SIZE;

	init_mm.pgd = pgd; /* use the Xen pagetables to start */

	/* keep using Xen gdt for now; no urgent need to change it */

	x86_write_percpu(xen_cr3, __pa(pgd));
1166
	x86_write_percpu(xen_current_cr3, __pa(pgd));
1167 1168 1169 1170 1171 1172 1173 1174 1175 1176

#ifdef CONFIG_SMP
	/* Don't do the full vcpu_info placement stuff until we have a
	   possible map. */
	per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
#else
	/* May as well do it now, since there's no good time to call
	   it later on UP. */
	xen_setup_vcpu_info_placement();
#endif
1177

1178
	pv_info.kernel_rpl = 1;
1179
	if (xen_feature(XENFEAT_supervisor_mode_kernel))
1180
		pv_info.kernel_rpl = 0;
1181 1182

	/* set the limit of our address space */
1183
	xen_reserve_top();
1184 1185 1186 1187 1188 1189 1190

	/* set up basic CPUID stuff */
	cpu_detect(&new_cpu_data);
	new_cpu_data.hard_math = 1;
	new_cpu_data.x86_capability[0] = cpuid_edx(1);

	/* Poke various useful things into boot_params */
1191 1192 1193 1194
	boot_params.hdr.type_of_loader = (9 << 4) | 0;
	boot_params.hdr.ramdisk_image = xen_start_info->mod_start
		? __pa(xen_start_info->mod_start) : 0;
	boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
1195 1196 1197 1198

	/* Start the world */
	start_kernel();
}