kvm_main.c 49.6 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

18
#include "iodev.h"
A
Avi Kivity 已提交
19

20
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
21 22 23 24 25 26 27 28 29 30 31
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
32
#include <linux/sysdev.h>
A
Avi Kivity 已提交
33
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
34
#include <linux/sched.h>
35 36
#include <linux/cpumask.h>
#include <linux/smp.h>
37
#include <linux/anon_inodes.h>
38
#include <linux/profile.h>
39
#include <linux/kvm_para.h>
40
#include <linux/pagemap.h>
41
#include <linux/mman.h>
42
#include <linux/swap.h>
43
#include <linux/bitops.h>
44
#include <linux/spinlock.h>
45
#include <linux/compat.h>
46
#include <linux/srcu.h>
J
Joerg Roedel 已提交
47
#include <linux/hugetlb.h>
48
#include <linux/slab.h>
A
Avi Kivity 已提交
49

A
Avi Kivity 已提交
50 51 52
#include <asm/processor.h>
#include <asm/io.h>
#include <asm/uaccess.h>
53
#include <asm/pgtable.h>
54
#include <asm-generic/bitops/le.h>
A
Avi Kivity 已提交
55

56 57
#include "coalesced_mmio.h"

58 59 60
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

A
Avi Kivity 已提交
61 62 63
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

64 65 66
/*
 * Ordering of locks:
 *
67
 * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
68 69
 */

70 71
DEFINE_SPINLOCK(kvm_lock);
LIST_HEAD(vm_list);
72

73
static cpumask_var_t cpus_hardware_enabled;
74 75
static int kvm_usage_count = 0;
static atomic_t hardware_enable_failed;
76

77 78
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
79

80 81
static __read_mostly struct preempt_ops kvm_preempt_ops;

82
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
83

A
Avi Kivity 已提交
84 85
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
86 87
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
88

M
Marcelo Tosatti 已提交
89 90
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);

H
Hannes Eder 已提交
91
static bool kvm_rebooting;
92

93 94
static bool largepages_enabled = true;

95
inline int kvm_is_mmio_pfn(pfn_t pfn)
B
Ben-Ami Yassour 已提交
96
{
97 98 99 100
	if (pfn_valid(pfn)) {
		struct page *page = compound_head(pfn_to_page(pfn));
		return PageReserved(page);
	}
B
Ben-Ami Yassour 已提交
101 102 103 104

	return true;
}

A
Avi Kivity 已提交
105 106 107
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
108
void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
109
{
110 111
	int cpu;

A
Avi Kivity 已提交
112
	mutex_lock(&vcpu->mutex);
113 114
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
115
	kvm_arch_vcpu_load(vcpu, cpu);
116
	put_cpu();
A
Avi Kivity 已提交
117 118
}

119
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
120
{
121
	preempt_disable();
122
	kvm_arch_vcpu_put(vcpu);
123 124
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
125 126 127
	mutex_unlock(&vcpu->mutex);
}

128 129 130 131
static void ack_flush(void *_completed)
{
}

132
static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
133
{
134
	int i, cpu, me;
135 136
	cpumask_var_t cpus;
	bool called = true;
137 138
	struct kvm_vcpu *vcpu;

139
	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
140

141
	raw_spin_lock(&kvm->requests_lock);
142
	me = smp_processor_id();
143
	kvm_for_each_vcpu(i, vcpu, kvm) {
144
		if (test_and_set_bit(req, &vcpu->requests))
145 146
			continue;
		cpu = vcpu->cpu;
147 148
		if (cpus != NULL && cpu != -1 && cpu != me)
			cpumask_set_cpu(cpu, cpus);
149
	}
150 151 152 153 154 155
	if (unlikely(cpus == NULL))
		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
	else if (!cpumask_empty(cpus))
		smp_call_function_many(cpus, ack_flush, NULL, 1);
	else
		called = false;
156
	raw_spin_unlock(&kvm->requests_lock);
157
	free_cpumask_var(cpus);
158
	return called;
159 160
}

161
void kvm_flush_remote_tlbs(struct kvm *kvm)
162
{
163 164
	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
		++kvm->stat.remote_tlb_flush;
165 166
}

167 168 169 170
void kvm_reload_remote_mmus(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
}
171

R
Rusty Russell 已提交
172 173 174 175 176 177 178 179 180
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
E
Eddie Dong 已提交
181
	init_waitqueue_head(&vcpu->wq);
R
Rusty Russell 已提交
182 183 184 185 186 187 188 189

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

190
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
191
	if (r < 0)
192
		goto fail_free_run;
R
Rusty Russell 已提交
193 194 195 196 197
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
198
	return r;
R
Rusty Russell 已提交
199 200 201 202 203
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
204
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
205 206 207 208
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

209 210 211 212 213 214 215 216 217 218 219
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
220
	int need_tlb_flush, idx;
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
240
	idx = srcu_read_lock(&kvm->srcu);
241 242 243 244
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	need_tlb_flush = kvm_unmap_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
245
	srcu_read_unlock(&kvm->srcu, idx);
246 247 248 249 250 251 252

	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

}

253 254 255 256 257 258
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
259
	int idx;
260

261
	idx = srcu_read_lock(&kvm->srcu);
262 263 264 265
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	kvm_set_spte_hva(kvm, address, pte);
	spin_unlock(&kvm->mmu_lock);
266
	srcu_read_unlock(&kvm->srcu, idx);
267 268
}

269 270 271 272 273 274
static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
275
	int need_tlb_flush = 0, idx;
276

277
	idx = srcu_read_lock(&kvm->srcu);
278 279 280 281 282 283 284 285 286 287
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
	for (; start < end; start += PAGE_SIZE)
		need_tlb_flush |= kvm_unmap_hva(kvm, start);
	spin_unlock(&kvm->mmu_lock);
288
	srcu_read_unlock(&kvm->srcu, idx);
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325

	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
	/*
	 * The above sequence increase must be visible before the
	 * below count decrease but both values are read by the kvm
	 * page fault under mmu_lock spinlock so we don't need to add
	 * a smb_wmb() here in between the two.
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
					      unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
326
	int young, idx;
327

328
	idx = srcu_read_lock(&kvm->srcu);
329 330 331
	spin_lock(&kvm->mmu_lock);
	young = kvm_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
332
	srcu_read_unlock(&kvm->srcu, idx);
333 334 335 336 337 338 339

	if (young)
		kvm_flush_remote_tlbs(kvm);

	return young;
}

340 341 342 343
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
344 345 346
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
347
	kvm_arch_flush_shadow(kvm);
348
	srcu_read_unlock(&kvm->srcu, idx);
349 350
}

351 352 353 354 355
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
356
	.change_pte		= kvm_mmu_notifier_change_pte,
357
	.release		= kvm_mmu_notifier_release,
358
};
359 360 361 362 363 364 365 366 367 368 369 370 371 372

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

373 374
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

375
static struct kvm *kvm_create_vm(void)
A
Avi Kivity 已提交
376
{
M
Marcelo Tosatti 已提交
377
	int r = 0, i;
378
	struct kvm *kvm = kvm_arch_create_vm();
A
Avi Kivity 已提交
379

380 381
	if (IS_ERR(kvm))
		goto out;
382 383 384 385 386

	r = hardware_enable_all();
	if (r)
		goto out_err_nodisable;

387 388
#ifdef CONFIG_HAVE_KVM_IRQCHIP
	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
389
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
390
#endif
A
Avi Kivity 已提交
391

392 393 394 395
	r = -ENOMEM;
	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
	if (!kvm->memslots)
		goto out_err;
396 397
	if (init_srcu_struct(&kvm->srcu))
		goto out_err;
M
Marcelo Tosatti 已提交
398 399 400 401 402 403 404 405
	for (i = 0; i < KVM_NR_BUSES; i++) {
		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
					GFP_KERNEL);
		if (!kvm->buses[i]) {
			cleanup_srcu_struct(&kvm->srcu);
			goto out_err;
		}
	}
406

407
	r = kvm_init_mmu_notifier(kvm);
408
	if (r) {
409
		cleanup_srcu_struct(&kvm->srcu);
410
		goto out_err;
411 412
	}

413 414
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
415
	spin_lock_init(&kvm->mmu_lock);
416
	raw_spin_lock_init(&kvm->requests_lock);
G
Gregory Haskins 已提交
417
	kvm_eventfd_init(kvm);
S
Shaohua Li 已提交
418
	mutex_init(&kvm->lock);
419
	mutex_init(&kvm->irq_lock);
420
	mutex_init(&kvm->slots_lock);
I
Izik Eidus 已提交
421
	atomic_set(&kvm->users_count, 1);
422 423 424
	spin_lock(&kvm_lock);
	list_add(&kvm->vm_list, &vm_list);
	spin_unlock(&kvm_lock);
425
out:
426
	return kvm;
427 428 429 430

out_err:
	hardware_disable_all();
out_err_nodisable:
M
Marcelo Tosatti 已提交
431 432
	for (i = 0; i < KVM_NR_BUSES; i++)
		kfree(kvm->buses[i]);
433
	kfree(kvm->memslots);
434 435
	kfree(kvm);
	return ERR_PTR(r);
436 437
}

A
Avi Kivity 已提交
438 439 440 441 442 443
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
444 445
	int i;

446 447
	if (!dont || free->rmap != dont->rmap)
		vfree(free->rmap);
A
Avi Kivity 已提交
448 449 450 451

	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		vfree(free->dirty_bitmap);

452 453 454 455 456 457 458

	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
		if (!dont || free->lpage_info[i] != dont->lpage_info[i]) {
			vfree(free->lpage_info[i]);
			free->lpage_info[i] = NULL;
		}
	}
M
Marcelo Tosatti 已提交
459

A
Avi Kivity 已提交
460
	free->npages = 0;
A
Al Viro 已提交
461
	free->dirty_bitmap = NULL;
462
	free->rmap = NULL;
A
Avi Kivity 已提交
463 464
}

465
void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
466 467
{
	int i;
468 469 470 471
	struct kvm_memslots *slots = kvm->memslots;

	for (i = 0; i < slots->nmemslots; ++i)
		kvm_free_physmem_slot(&slots->memslots[i], NULL);
A
Avi Kivity 已提交
472

473
	kfree(kvm->memslots);
A
Avi Kivity 已提交
474 475
}

476 477
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
478
	int i;
479 480
	struct mm_struct *mm = kvm->mm;

481
	kvm_arch_sync_events(kvm);
482 483 484
	spin_lock(&kvm_lock);
	list_del(&kvm->vm_list);
	spin_unlock(&kvm_lock);
485
	kvm_free_irq_routing(kvm);
M
Marcelo Tosatti 已提交
486 487
	for (i = 0; i < KVM_NR_BUSES; i++)
		kvm_io_bus_destroy(kvm->buses[i]);
488
	kvm_coalesced_mmio_free(kvm);
489 490
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
491 492
#else
	kvm_arch_flush_shadow(kvm);
493
#endif
494
	kvm_arch_destroy_vm(kvm);
495
	hardware_disable_all();
496
	mmdrop(mm);
497 498
}

I
Izik Eidus 已提交
499 500 501 502 503 504 505 506 507 508 509 510 511 512
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


513 514 515 516
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
517 518
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
519
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
520 521 522 523 524 525 526 527
	return 0;
}

/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
528
 *
529
 * Must be called holding mmap_sem for write.
A
Avi Kivity 已提交
530
 */
531 532 533
int __kvm_set_memory_region(struct kvm *kvm,
			    struct kvm_userspace_memory_region *mem,
			    int user_alloc)
A
Avi Kivity 已提交
534
{
535
	int r, flush_shadow = 0;
A
Avi Kivity 已提交
536
	gfn_t base_gfn;
537 538
	unsigned long npages;
	unsigned long i;
A
Avi Kivity 已提交
539 540
	struct kvm_memory_slot *memslot;
	struct kvm_memory_slot old, new;
541
	struct kvm_memslots *slots, *old_memslots;
A
Avi Kivity 已提交
542 543 544 545 546 547 548

	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
S
Sheng Yang 已提交
549
	if (user_alloc && (mem->userspace_addr & (PAGE_SIZE - 1)))
550
		goto out;
551
	if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
A
Avi Kivity 已提交
552 553 554 555
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

556
	memslot = &kvm->memslots->memslots[mem->slot];
A
Avi Kivity 已提交
557 558 559
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

560 561 562 563
	r = -EINVAL;
	if (npages > KVM_MEM_MAX_NR_PAGES)
		goto out;

A
Avi Kivity 已提交
564 565 566 567 568 569 570 571 572 573 574 575
	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

	new = old = *memslot;

	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	/* Disallow changing a memory slot's size. */
	r = -EINVAL;
	if (npages && old.npages && npages != old.npages)
576
		goto out_free;
A
Avi Kivity 已提交
577 578 579 580

	/* Check for overlaps */
	r = -EEXIST;
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
581
		struct kvm_memory_slot *s = &kvm->memslots->memslots[i];
A
Avi Kivity 已提交
582

583
		if (s == memslot || !s->npages)
A
Avi Kivity 已提交
584 585 586
			continue;
		if (!((base_gfn + npages <= s->base_gfn) ||
		      (base_gfn >= s->base_gfn + s->npages)))
587
			goto out_free;
A
Avi Kivity 已提交
588 589 590 591
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
592
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
593 594 595 596

	r = -ENOMEM;

	/* Allocate if a slot is being created */
597
#ifndef CONFIG_S390
598
	if (npages && !new.rmap) {
M
Mike Day 已提交
599
		new.rmap = vmalloc(npages * sizeof(struct page *));
600 601

		if (!new.rmap)
602
			goto out_free;
603 604

		memset(new.rmap, 0, npages * sizeof(*new.rmap));
605

606
		new.user_alloc = user_alloc;
607
		new.userspace_addr = mem->userspace_addr;
A
Avi Kivity 已提交
608
	}
609 610
	if (!npages)
		goto skip_lpage;
M
Marcelo Tosatti 已提交
611

612
	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
613 614 615
		unsigned long ugfn;
		unsigned long j;
		int lpages;
616
		int level = i + 2;
M
Marcelo Tosatti 已提交
617

618 619 620 621 622 623 624 625 626 627 628 629 630
		/* Avoid unused variable warning if no large pages */
		(void)level;

		if (new.lpage_info[i])
			continue;

		lpages = 1 + (base_gfn + npages - 1) /
			     KVM_PAGES_PER_HPAGE(level);
		lpages -= base_gfn / KVM_PAGES_PER_HPAGE(level);

		new.lpage_info[i] = vmalloc(lpages * sizeof(*new.lpage_info[i]));

		if (!new.lpage_info[i])
M
Marcelo Tosatti 已提交
631 632
			goto out_free;

633 634
		memset(new.lpage_info[i], 0,
		       lpages * sizeof(*new.lpage_info[i]));
M
Marcelo Tosatti 已提交
635

636 637 638 639
		if (base_gfn % KVM_PAGES_PER_HPAGE(level))
			new.lpage_info[i][0].write_count = 1;
		if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE(level))
			new.lpage_info[i][lpages - 1].write_count = 1;
640 641 642
		ugfn = new.userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
643 644
		 * other, or if explicitly asked to, disable large page
		 * support for this slot
645
		 */
646
		if ((base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
647
		    !largepages_enabled)
648 649
			for (j = 0; j < lpages; ++j)
				new.lpage_info[i][j].write_count = 1;
M
Marcelo Tosatti 已提交
650
	}
A
Avi Kivity 已提交
651

652 653
skip_lpage:

A
Avi Kivity 已提交
654 655
	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
656
		unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(&new);
A
Avi Kivity 已提交
657 658 659

		new.dirty_bitmap = vmalloc(dirty_bytes);
		if (!new.dirty_bitmap)
660
			goto out_free;
A
Avi Kivity 已提交
661
		memset(new.dirty_bitmap, 0, dirty_bytes);
662
		/* destroy any largepage mappings for dirty tracking */
663
		if (old.npages)
664
			flush_shadow = 1;
A
Avi Kivity 已提交
665
	}
666 667 668 669
#else  /* not defined CONFIG_S390 */
	new.user_alloc = user_alloc;
	if (user_alloc)
		new.userspace_addr = mem->userspace_addr;
670
#endif /* not defined CONFIG_S390 */
A
Avi Kivity 已提交
671

672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691
	if (!npages) {
		r = -ENOMEM;
		slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
		if (!slots)
			goto out_free;
		memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
		if (mem->slot >= slots->nmemslots)
			slots->nmemslots = mem->slot + 1;
		slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID;

		old_memslots = kvm->memslots;
		rcu_assign_pointer(kvm->memslots, slots);
		synchronize_srcu_expedited(&kvm->srcu);
		/* From this point no new shadow pages pointing to a deleted
		 * memslot will be created.
		 *
		 * validation of sp->gfn happens in:
		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
		 * 	- kvm_is_visible_gfn (mmu_check_roots)
		 */
692
		kvm_arch_flush_shadow(kvm);
693 694
		kfree(old_memslots);
	}
695

696 697 698 699
	r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
	if (r)
		goto out_free;

700 701 702 703 704 705 706 707
#ifdef CONFIG_DMAR
	/* map the pages in iommu page table */
	if (npages) {
		r = kvm_iommu_map_pages(kvm, &new);
		if (r)
			goto out_free;
	}
#endif
708

709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
	r = -ENOMEM;
	slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
	if (!slots)
		goto out_free;
	memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
	if (mem->slot >= slots->nmemslots)
		slots->nmemslots = mem->slot + 1;

	/* actual memory is freed via old in kvm_free_physmem_slot below */
	if (!npages) {
		new.rmap = NULL;
		new.dirty_bitmap = NULL;
		for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i)
			new.lpage_info[i] = NULL;
	}

	slots->memslots[mem->slot] = new;
	old_memslots = kvm->memslots;
	rcu_assign_pointer(kvm->memslots, slots);
	synchronize_srcu_expedited(&kvm->srcu);
729

730
	kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
731

732 733 734 735 736 737
	kvm_free_physmem_slot(&old, &new);
	kfree(old_memslots);

	if (flush_shadow)
		kvm_arch_flush_shadow(kvm);

A
Avi Kivity 已提交
738 739
	return 0;

740
out_free:
A
Avi Kivity 已提交
741 742 743
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
744 745

}
746 747 748 749 750 751 752 753
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
			  struct kvm_userspace_memory_region *mem,
			  int user_alloc)
{
	int r;

754
	mutex_lock(&kvm->slots_lock);
755
	r = __kvm_set_memory_region(kvm, mem, user_alloc);
756
	mutex_unlock(&kvm->slots_lock);
757 758
	return r;
}
759 760
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

761 762 763 764
int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
				   struct
				   kvm_userspace_memory_region *mem,
				   int user_alloc)
765
{
766 767
	if (mem->slot >= KVM_MEMORY_SLOTS)
		return -EINVAL;
768
	return kvm_set_memory_region(kvm, mem, user_alloc);
A
Avi Kivity 已提交
769 770
}

771 772
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
773 774 775
{
	struct kvm_memory_slot *memslot;
	int r, i;
776
	unsigned long n;
A
Avi Kivity 已提交
777 778 779 780 781 782
	unsigned long any = 0;

	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

783
	memslot = &kvm->memslots->memslots[log->slot];
A
Avi Kivity 已提交
784 785 786 787
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

788
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
789

790
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
791 792 793 794 795 796
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

797 798
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
799 800 801 802 803 804

	r = 0;
out:
	return r;
}

805 806 807 808 809 810
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

811 812 813 814 815 816
int is_error_page(struct page *page)
{
	return page == bad_page;
}
EXPORT_SYMBOL_GPL(is_error_page);

817 818 819 820 821 822
int is_error_pfn(pfn_t pfn)
{
	return pfn == bad_pfn;
}
EXPORT_SYMBOL_GPL(is_error_pfn);

I
Izik Eidus 已提交
823 824 825 826 827 828 829 830 831 832 833
static inline unsigned long bad_hva(void)
{
	return PAGE_OFFSET;
}

int kvm_is_error_hva(unsigned long addr)
{
	return addr == bad_hva();
}
EXPORT_SYMBOL_GPL(kvm_is_error_hva);

834
struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
835 836
{
	int i;
837
	struct kvm_memslots *slots = kvm_memslots(kvm);
A
Avi Kivity 已提交
838

839 840
	for (i = 0; i < slots->nmemslots; ++i) {
		struct kvm_memory_slot *memslot = &slots->memslots[i];
A
Avi Kivity 已提交
841 842 843 844 845

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return memslot;
	}
A
Al Viro 已提交
846
	return NULL;
A
Avi Kivity 已提交
847
}
848
EXPORT_SYMBOL_GPL(gfn_to_memslot_unaliased);
849 850 851 852

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	gfn = unalias_gfn(kvm, gfn);
853
	return gfn_to_memslot_unaliased(kvm, gfn);
854
}
A
Avi Kivity 已提交
855

856 857 858
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
	int i;
859
	struct kvm_memslots *slots = kvm_memslots(kvm);
860

861
	gfn = unalias_gfn_instantiation(kvm, gfn);
862
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
863
		struct kvm_memory_slot *memslot = &slots->memslots[i];
864

865 866 867
		if (memslot->flags & KVM_MEMSLOT_INVALID)
			continue;

868 869 870 871 872 873 874 875
		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

J
Joerg Roedel 已提交
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

900 901 902
int memslot_id(struct kvm *kvm, gfn_t gfn)
{
	int i;
903
	struct kvm_memslots *slots = kvm_memslots(kvm);
904 905 906 907 908 909 910 911 912 913 914 915 916 917
	struct kvm_memory_slot *memslot = NULL;

	gfn = unalias_gfn(kvm, gfn);
	for (i = 0; i < slots->nmemslots; ++i) {
		memslot = &slots->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			break;
	}

	return memslot - slots->memslots;
}

918 919 920 921 922
static unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
{
	return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE;
}

M
Marcelo Tosatti 已提交
923
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
I
Izik Eidus 已提交
924 925 926
{
	struct kvm_memory_slot *slot;

927
	gfn = unalias_gfn_instantiation(kvm, gfn);
928
	slot = gfn_to_memslot_unaliased(kvm, gfn);
929
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
I
Izik Eidus 已提交
930
		return bad_hva();
931
	return gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
932
}
933
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
934

935
static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr)
A
Avi Kivity 已提交
936
{
937 938
	struct page *page[1];
	int npages;
939
	pfn_t pfn;
A
Avi Kivity 已提交
940

941 942
	might_sleep();

943
	npages = get_user_pages_fast(addr, 1, 1, page);
I
Izik Eidus 已提交
944

945 946 947
	if (unlikely(npages != 1)) {
		struct vm_area_struct *vma;

948
		down_read(&current->mm->mmap_sem);
949
		vma = find_vma(current->mm, addr);
950

951 952
		if (vma == NULL || addr < vma->vm_start ||
		    !(vma->vm_flags & VM_PFNMAP)) {
953
			up_read(&current->mm->mmap_sem);
954 955 956 957 958
			get_page(bad_page);
			return page_to_pfn(bad_page);
		}

		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
959
		up_read(&current->mm->mmap_sem);
960
		BUG_ON(!kvm_is_mmio_pfn(pfn));
961 962
	} else
		pfn = page_to_pfn(page[0]);
963

964
	return pfn;
965 966
}

967 968 969 970 971 972 973 974 975 976 977 978
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
{
	unsigned long addr;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr)) {
		get_page(bad_page);
		return page_to_pfn(bad_page);
	}

	return hva_to_pfn(kvm, addr);
}
979 980
EXPORT_SYMBOL_GPL(gfn_to_pfn);

981 982 983 984 985 986 987
pfn_t gfn_to_pfn_memslot(struct kvm *kvm,
			 struct kvm_memory_slot *slot, gfn_t gfn)
{
	unsigned long addr = gfn_to_hva_memslot(slot, gfn);
	return hva_to_pfn(kvm, addr);
}

988 989
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
990 991 992
	pfn_t pfn;

	pfn = gfn_to_pfn(kvm, gfn);
993
	if (!kvm_is_mmio_pfn(pfn))
994 995
		return pfn_to_page(pfn);

996
	WARN_ON(kvm_is_mmio_pfn(pfn));
997 998 999

	get_page(bad_page);
	return bad_page;
A
Avi Kivity 已提交
1000
}
1001

A
Avi Kivity 已提交
1002 1003
EXPORT_SYMBOL_GPL(gfn_to_page);

1004 1005
void kvm_release_page_clean(struct page *page)
{
1006
	kvm_release_pfn_clean(page_to_pfn(page));
1007 1008 1009
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

1010 1011
void kvm_release_pfn_clean(pfn_t pfn)
{
1012
	if (!kvm_is_mmio_pfn(pfn))
1013
		put_page(pfn_to_page(pfn));
1014 1015 1016
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1017
void kvm_release_page_dirty(struct page *page)
1018
{
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

void kvm_release_pfn_dirty(pfn_t pfn)
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);

void kvm_set_page_dirty(struct page *page)
{
	kvm_set_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_set_page_dirty);

void kvm_set_pfn_dirty(pfn_t pfn)
{
1038
	if (!kvm_is_mmio_pfn(pfn)) {
1039 1040 1041 1042
		struct page *page = pfn_to_page(pfn);
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1043
}
1044 1045 1046 1047
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
1048
	if (!kvm_is_mmio_pfn(pfn))
1049
		mark_page_accessed(pfn_to_page(pfn));
1050 1051 1052 1053 1054
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
1055
	if (!kvm_is_mmio_pfn(pfn))
1056
		get_page(pfn_to_page(pfn));
1057 1058
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1059

1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
1071 1072
	int r;
	unsigned long addr;
1073

1074 1075 1076 1077 1078
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_from_user(data, (void __user *)addr + offset, len);
	if (r)
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1115
	pagefault_disable();
1116
	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1117
	pagefault_enable();
1118 1119 1120 1121 1122 1123
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

1124 1125 1126
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
1127 1128
	int r;
	unsigned long addr;
1129

1130 1131 1132 1133 1134
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_to_user((void __user *)addr + offset, data, len);
	if (r)
1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
1163
	return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

A
Avi Kivity 已提交
1186 1187
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
1188
	struct kvm_memory_slot *memslot;
A
Avi Kivity 已提交
1189

1190
	gfn = unalias_gfn(kvm, gfn);
1191
	memslot = gfn_to_memslot_unaliased(kvm, gfn);
R
Rusty Russell 已提交
1192 1193
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1194

1195
		generic___set_le_bit(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1196 1197 1198
	}
}

E
Eddie Dong 已提交
1199 1200 1201
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1202
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1203
{
1204 1205 1206 1207 1208
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

1209
		if (kvm_arch_vcpu_runnable(vcpu)) {
1210
			set_bit(KVM_REQ_UNHALT, &vcpu->requests);
1211
			break;
1212
		}
1213 1214
		if (kvm_cpu_has_pending_timer(vcpu))
			break;
1215 1216 1217
		if (signal_pending(current))
			break;

E
Eddie Dong 已提交
1218 1219
		schedule();
	}
1220

1221
	finish_wait(&vcpu->wq, &wait);
E
Eddie Dong 已提交
1222 1223
}

A
Avi Kivity 已提交
1224 1225
void kvm_resched(struct kvm_vcpu *vcpu)
{
1226 1227
	if (!need_resched())
		return;
A
Avi Kivity 已提交
1228 1229 1230 1231
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

Z
Zhai, Edwin 已提交
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu)
{
	ktime_t expires;
	DEFINE_WAIT(wait);

	prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

	/* Sleep for 100 us, and hope lock-holder got scheduled */
	expires = ktime_add_ns(ktime_get(), 100000UL);
	schedule_hrtimeout(&expires, HRTIMER_MODE_ABS);

	finish_wait(&vcpu->wq, &wait);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

1247
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1248 1249 1250 1251
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

1252
	if (vmf->pgoff == 0)
1253
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
1254
#ifdef CONFIG_X86
1255
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1256
		page = virt_to_page(vcpu->arch.pio_data);
1257 1258 1259 1260
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
1261
#endif
1262
	else
1263
		return VM_FAULT_SIGBUS;
1264
	get_page(page);
1265 1266
	vmf->page = page;
	return 0;
1267 1268
}

1269
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1270
	.fault = kvm_vcpu_fault,
1271 1272 1273 1274 1275 1276 1277 1278
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
1279 1280 1281 1282
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
1283
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1284 1285 1286
	return 0;
}

1287
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
1288 1289 1290
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
	.compat_ioctl   = kvm_vcpu_ioctl,
1291
	.mmap           = kvm_vcpu_mmap,
A
Avi Kivity 已提交
1292 1293 1294 1295 1296 1297 1298
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
1299
	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
A
Avi Kivity 已提交
1300 1301
}

1302 1303 1304
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
1305
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1306 1307
{
	int r;
1308
	struct kvm_vcpu *vcpu, *v;
1309

1310
	vcpu = kvm_arch_vcpu_create(kvm, id);
R
Rusty Russell 已提交
1311 1312
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
1313

1314 1315
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

1316 1317
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
1318
		return r;
1319

S
Shaohua Li 已提交
1320
	mutex_lock(&kvm->lock);
1321 1322
	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
		r = -EINVAL;
1323
		goto vcpu_destroy;
R
Rusty Russell 已提交
1324
	}
1325

1326 1327
	kvm_for_each_vcpu(r, v, kvm)
		if (v->vcpu_id == id) {
1328 1329 1330 1331 1332
			r = -EEXIST;
			goto vcpu_destroy;
		}

	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1333

R
Rusty Russell 已提交
1334
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
1335
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
1336
	r = create_vcpu_fd(vcpu);
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
	if (r < 0) {
		kvm_put_kvm(kvm);
		goto vcpu_destroy;
	}

	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	if (kvm->bsp_vcpu_id == id)
		kvm->bsp_vcpu = vcpu;
#endif
	mutex_unlock(&kvm->lock);
R
Rusty Russell 已提交
1351
	return r;
1352

1353
vcpu_destroy:
1354
	mutex_unlock(&kvm->lock);
1355
	kvm_arch_vcpu_destroy(vcpu);
1356 1357 1358
	return r;
}

A
Avi Kivity 已提交
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
1370 1371
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
1372
{
A
Avi Kivity 已提交
1373
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
1374
	void __user *argp = (void __user *)arg;
1375
	int r;
1376 1377
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
1378

1379 1380
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
1381
	switch (ioctl) {
1382
	case KVM_RUN:
1383 1384 1385
		r = -EINVAL;
		if (arg)
			goto out;
1386
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
A
Avi Kivity 已提交
1387 1388
		break;
	case KVM_GET_REGS: {
1389
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1390

1391 1392 1393
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1394
			goto out;
1395 1396 1397
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
1398
		r = -EFAULT;
1399 1400
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
1401
		r = 0;
1402 1403
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
1404 1405 1406
		break;
	}
	case KVM_SET_REGS: {
1407
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1408

1409 1410 1411
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1412
			goto out;
1413 1414 1415 1416
		r = -EFAULT;
		if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
			goto out_free2;
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
A
Avi Kivity 已提交
1417
		if (r)
1418
			goto out_free2;
A
Avi Kivity 已提交
1419
		r = 0;
1420 1421
out_free2:
		kfree(kvm_regs);
A
Avi Kivity 已提交
1422 1423 1424
		break;
	}
	case KVM_GET_SREGS: {
1425 1426 1427 1428 1429
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
1430 1431 1432
		if (r)
			goto out;
		r = -EFAULT;
1433
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
1434 1435 1436 1437 1438
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
1439 1440 1441 1442
		kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
A
Avi Kivity 已提交
1443
		r = -EFAULT;
1444
		if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
1445
			goto out;
1446
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
1447 1448 1449 1450 1451
		if (r)
			goto out;
		r = 0;
		break;
	}
1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1476 1477 1478 1479
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
1480
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
1481
			goto out;
1482
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
1483 1484 1485
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
1486
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
1487 1488 1489 1490
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
1491 1492
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
1493 1494

		r = -EFAULT;
A
Al Viro 已提交
1495
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
1496
			goto out;
J
Jan Kiszka 已提交
1497
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
1498 1499 1500 1501 1502
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		break;
	}
A
Avi Kivity 已提交
1526
	case KVM_GET_FPU: {
1527 1528 1529 1530 1531
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
1532 1533 1534
		if (r)
			goto out;
		r = -EFAULT;
1535
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
1536 1537 1538 1539 1540
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
1541 1542 1543 1544
		fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
A
Avi Kivity 已提交
1545
		r = -EFAULT;
1546
		if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
1547
			goto out;
1548
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
1549 1550 1551 1552 1553
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1554
	default:
1555
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1556 1557
	}
out:
1558 1559
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
1560 1561 1562 1563 1564 1565 1566 1567
	return r;
}

static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
1568
	int r;
A
Avi Kivity 已提交
1569

1570 1571
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
1572 1573 1574 1575 1576 1577
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		if (r < 0)
			goto out;
		break;
1578 1579 1580 1581 1582 1583 1584 1585 1586
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
A
Avi Kivity 已提交
1587 1588 1589 1590 1591 1592 1593 1594
		if (r)
			goto out;
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
1595
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
1596
			goto out;
1597
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
1598 1599 1600 1601
		if (r)
			goto out;
		break;
	}
1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		if (r)
			goto out;
		r = 0;
		break;
	}
#endif
G
Gregory Haskins 已提交
1626 1627 1628 1629 1630 1631 1632 1633 1634
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
		r = kvm_irqfd(kvm, data.fd, data.gsi, data.flags);
		break;
	}
G
Gregory Haskins 已提交
1635 1636 1637 1638 1639 1640 1641 1642 1643
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
1644 1645 1646
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
1647
		mutex_lock(&kvm->lock);
1648 1649 1650 1651
		if (atomic_read(&kvm->online_vcpus) != 0)
			r = -EBUSY;
		else
			kvm->bsp_vcpu_id = arg;
1652
		mutex_unlock(&kvm->lock);
1653 1654
		break;
#endif
1655
	default:
1656
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1657 1658
		if (r == -ENOTTY)
			r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
1659 1660 1661 1662 1663
	}
out:
	return r;
}

1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709
#ifdef CONFIG_COMPAT
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		r = -EFAULT;
		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
			goto out;
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		if (r)
			goto out;
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

1710
static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1711
{
1712 1713 1714 1715
	struct page *page[1];
	unsigned long addr;
	int npages;
	gfn_t gfn = vmf->pgoff;
1716 1717
	struct kvm *kvm = vma->vm_file->private_data;

1718 1719
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
1720
		return VM_FAULT_SIGBUS;
1721 1722 1723 1724

	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
				NULL);
	if (unlikely(npages != 1))
1725
		return VM_FAULT_SIGBUS;
1726 1727

	vmf->page = page[0];
1728
	return 0;
1729 1730
}

1731
static const struct vm_operations_struct kvm_vm_vm_ops = {
1732
	.fault = kvm_vm_fault,
1733 1734 1735 1736 1737 1738 1739 1740
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

1741
static struct file_operations kvm_vm_fops = {
1742 1743
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
1744 1745 1746
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vm_compat_ioctl,
#endif
1747 1748 1749 1750 1751
	.mmap           = kvm_vm_mmap,
};

static int kvm_dev_ioctl_create_vm(void)
{
1752
	int fd, r;
1753 1754 1755
	struct kvm *kvm;

	kvm = kvm_create_vm();
1756 1757
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
1758 1759 1760 1761 1762 1763 1764
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	r = kvm_coalesced_mmio_init(kvm);
	if (r < 0) {
		kvm_put_kvm(kvm);
		return r;
	}
#endif
1765
	fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
A
Al Viro 已提交
1766
	if (fd < 0)
A
Al Viro 已提交
1767
		kvm_put_kvm(kvm);
1768 1769 1770 1771

	return fd;
}

1772 1773 1774
static long kvm_dev_ioctl_check_extension_generic(long arg)
{
	switch (arg) {
1775
	case KVM_CAP_USER_MEMORY:
1776
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
1777
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
1778 1779 1780
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_CAP_SET_BOOT_CPU_ID:
#endif
1781
	case KVM_CAP_INTERNAL_ERROR_DATA:
1782
		return 1;
1783 1784
#ifdef CONFIG_HAVE_KVM_IRQCHIP
	case KVM_CAP_IRQ_ROUTING:
1785
		return KVM_MAX_IRQ_ROUTES;
1786
#endif
1787 1788 1789 1790 1791 1792
	default:
		break;
	}
	return kvm_dev_ioctl_check_extension(arg);
}

1793 1794 1795
static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
1796
	long r = -EINVAL;
1797 1798 1799

	switch (ioctl) {
	case KVM_GET_API_VERSION:
1800 1801 1802
		r = -EINVAL;
		if (arg)
			goto out;
1803 1804 1805
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
1806 1807 1808
		r = -EINVAL;
		if (arg)
			goto out;
1809 1810
		r = kvm_dev_ioctl_create_vm();
		break;
1811
	case KVM_CHECK_EXTENSION:
1812
		r = kvm_dev_ioctl_check_extension_generic(arg);
1813
		break;
1814 1815 1816 1817
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
1818 1819 1820
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
1821 1822 1823
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
		r += PAGE_SIZE;    /* coalesced mmio ring page */
1824
#endif
1825
		break;
1826 1827 1828
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
1829
		r = -EOPNOTSUPP;
1830
		break;
A
Avi Kivity 已提交
1831
	default:
1832
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
1844
	KVM_MINOR,
A
Avi Kivity 已提交
1845 1846 1847 1848
	"kvm",
	&kvm_chardev_ops,
};

1849 1850 1851
static void hardware_enable(void *junk)
{
	int cpu = raw_smp_processor_id();
1852
	int r;
1853

1854
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
1855
		return;
1856

1857
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
1858 1859 1860 1861 1862 1863 1864 1865 1866

	r = kvm_arch_hardware_enable(NULL);

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
		printk(KERN_INFO "kvm: enabling virtualization on "
				 "CPU%d failed\n", cpu);
	}
1867 1868 1869 1870 1871 1872
}

static void hardware_disable(void *junk)
{
	int cpu = raw_smp_processor_id();

1873
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
1874
		return;
1875
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
1876
	kvm_arch_hardware_disable(NULL);
1877 1878
}

1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
		on_each_cpu(hardware_disable, NULL, 1);
}

static void hardware_disable_all(void)
{
	spin_lock(&kvm_lock);
	hardware_disable_all_nolock();
	spin_unlock(&kvm_lock);
}

static int hardware_enable_all(void)
{
	int r = 0;

	spin_lock(&kvm_lock);

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
		on_each_cpu(hardware_enable, NULL, 1);

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

	spin_unlock(&kvm_lock);

	return r;
}

A
Avi Kivity 已提交
1917 1918 1919 1920 1921
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

1922 1923 1924
	if (!kvm_usage_count)
		return NOTIFY_OK;

1925
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
1926
	switch (val) {
1927
	case CPU_DYING:
1928 1929 1930 1931
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
1932 1933 1934
	case CPU_ONLINE:
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
1935
		smp_call_function_single(cpu, hardware_enable, NULL, 1);
A
Avi Kivity 已提交
1936 1937 1938 1939 1940
		break;
	}
	return NOTIFY_OK;
}

1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952

asmlinkage void kvm_handle_fault_on_reboot(void)
{
	if (kvm_rebooting)
		/* spin while reset goes on */
		while (true)
			;
	/* Fault while not rebooting.  We want the trace. */
	BUG();
}
EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot);

1953
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
1954
		      void *v)
1955
{
1956 1957 1958 1959 1960 1961 1962 1963 1964
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
	kvm_rebooting = true;
	on_each_cpu(hardware_disable, NULL, 1);
1965 1966 1967 1968 1969 1970 1971 1972
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
1973
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
1974 1975 1976 1977 1978 1979 1980 1981
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
1982
	kfree(bus);
1983 1984
}

1985
/* kvm_io_bus_write - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
1986
int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
1987
		     int len, const void *val)
1988 1989
{
	int i;
1990 1991 1992
	struct kvm_io_bus *bus;

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
1993 1994 1995 1996 1997
	for (i = 0; i < bus->dev_count; i++)
		if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
			return 0;
	return -EOPNOTSUPP;
}
1998

1999
/* kvm_io_bus_read - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2000 2001
int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
		    int len, void *val)
2002 2003
{
	int i;
2004
	struct kvm_io_bus *bus;
M
Marcelo Tosatti 已提交
2005

2006
	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2007 2008 2009 2010
	for (i = 0; i < bus->dev_count; i++)
		if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
			return 0;
	return -EOPNOTSUPP;
2011 2012
}

2013
/* Caller must hold slots_lock. */
M
Marcelo Tosatti 已提交
2014 2015
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			    struct kvm_io_device *dev)
2016
{
M
Marcelo Tosatti 已提交
2017
	struct kvm_io_bus *new_bus, *bus;
2018

M
Marcelo Tosatti 已提交
2019
	bus = kvm->buses[bus_idx];
2020 2021
	if (bus->dev_count > NR_IOBUS_DEVS-1)
		return -ENOSPC;
2022

M
Marcelo Tosatti 已提交
2023 2024 2025 2026 2027 2028 2029 2030
	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
	if (!new_bus)
		return -ENOMEM;
	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
	new_bus->devs[new_bus->dev_count++] = dev;
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
2031 2032 2033 2034

	return 0;
}

2035
/* Caller must hold slots_lock. */
M
Marcelo Tosatti 已提交
2036 2037
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			      struct kvm_io_device *dev)
2038
{
M
Marcelo Tosatti 已提交
2039 2040
	int i, r;
	struct kvm_io_bus *new_bus, *bus;
2041

M
Marcelo Tosatti 已提交
2042 2043 2044
	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
	if (!new_bus)
		return -ENOMEM;
2045

M
Marcelo Tosatti 已提交
2046 2047 2048 2049 2050 2051 2052 2053
	bus = kvm->buses[bus_idx];
	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));

	r = -ENOENT;
	for (i = 0; i < new_bus->dev_count; i++)
		if (new_bus->devs[i] == dev) {
			r = 0;
			new_bus->devs[i] = new_bus->devs[--new_bus->dev_count];
2054 2055
			break;
		}
M
Marcelo Tosatti 已提交
2056 2057 2058 2059 2060 2061 2062 2063 2064 2065

	if (r) {
		kfree(new_bus);
		return r;
	}

	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
	return r;
2066 2067
}

A
Avi Kivity 已提交
2068 2069 2070 2071 2072
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
	.priority = 20, /* must be > scheduler priority */
};

2073
static int vm_stat_get(void *_offset, u64 *val)
2074 2075 2076 2077
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

2078
	*val = 0;
2079 2080
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
2081
		*val += *(u32 *)((void *)kvm + offset);
2082
	spin_unlock(&kvm_lock);
2083
	return 0;
2084 2085 2086 2087
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

2088
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
2089 2090 2091 2092 2093 2094
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

2095
	*val = 0;
A
Avi Kivity 已提交
2096 2097
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
2098 2099 2100
		kvm_for_each_vcpu(i, vcpu, kvm)
			*val += *(u32 *)((void *)vcpu + offset);

A
Avi Kivity 已提交
2101
	spin_unlock(&kvm_lock);
2102
	return 0;
A
Avi Kivity 已提交
2103 2104
}

2105 2106
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

2107
static const struct file_operations *stat_fops[] = {
2108 2109 2110
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
2111

2112
static void kvm_init_debug(void)
A
Avi Kivity 已提交
2113 2114 2115
{
	struct kvm_stats_debugfs_item *p;

2116
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
A
Avi Kivity 已提交
2117
	for (p = debugfs_entries; p->name; ++p)
2118
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
2119
						(void *)(long)p->offset,
2120
						stat_fops[p->kind]);
A
Avi Kivity 已提交
2121 2122 2123 2124 2125 2126 2127 2128
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
2129
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
2130 2131
}

2132 2133
static int kvm_suspend(struct sys_device *dev, pm_message_t state)
{
2134 2135
	if (kvm_usage_count)
		hardware_disable(NULL);
2136 2137 2138 2139 2140
	return 0;
}

static int kvm_resume(struct sys_device *dev)
{
2141 2142
	if (kvm_usage_count)
		hardware_enable(NULL);
2143 2144 2145 2146
	return 0;
}

static struct sysdev_class kvm_sysdev_class = {
2147
	.name = "kvm",
2148 2149 2150 2151 2152 2153 2154 2155 2156
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

static struct sys_device kvm_sysdev = {
	.id = 0,
	.cls = &kvm_sysdev_class,
};

2157
struct page *bad_page;
2158
pfn_t bad_pfn;
A
Avi Kivity 已提交
2159

2160 2161 2162 2163 2164 2165 2166 2167 2168 2169
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2170
	kvm_arch_vcpu_load(vcpu, cpu);
2171 2172 2173 2174 2175 2176 2177
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2178
	kvm_arch_vcpu_put(vcpu);
2179 2180
}

2181
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
2182
		  struct module *module)
A
Avi Kivity 已提交
2183 2184
{
	int r;
Y
Yang, Sheng 已提交
2185
	int cpu;
A
Avi Kivity 已提交
2186

2187 2188
	r = kvm_arch_init(opaque);
	if (r)
2189
		goto out_fail;
2190 2191 2192 2193 2194 2195 2196 2197

	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

	if (bad_page == NULL) {
		r = -ENOMEM;
		goto out;
	}

2198 2199
	bad_pfn = page_to_pfn(bad_page);

2200
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2201 2202 2203 2204
		r = -ENOMEM;
		goto out_free_0;
	}

2205
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
2206
	if (r < 0)
2207
		goto out_free_0a;
A
Avi Kivity 已提交
2208

Y
Yang, Sheng 已提交
2209 2210
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
2211
				kvm_arch_check_processor_compat,
2212
				&r, 1);
Y
Yang, Sheng 已提交
2213
		if (r < 0)
2214
			goto out_free_1;
Y
Yang, Sheng 已提交
2215 2216
	}

A
Avi Kivity 已提交
2217 2218
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
2219
		goto out_free_2;
A
Avi Kivity 已提交
2220 2221
	register_reboot_notifier(&kvm_reboot_notifier);

2222 2223
	r = sysdev_class_register(&kvm_sysdev_class);
	if (r)
2224
		goto out_free_3;
2225 2226 2227

	r = sysdev_register(&kvm_sysdev);
	if (r)
2228
		goto out_free_4;
2229

2230
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
2231 2232 2233
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
J
Joe Perches 已提交
2234
					   0, NULL);
2235 2236
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
2237
		goto out_free_5;
2238 2239
	}

A
Avi Kivity 已提交
2240
	kvm_chardev_ops.owner = module;
2241 2242
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
2243 2244 2245

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
2246
		printk(KERN_ERR "kvm: misc device register failed\n");
A
Avi Kivity 已提交
2247 2248 2249
		goto out_free;
	}

2250 2251 2252
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

2253 2254
	kvm_init_debug();

2255
	return 0;
A
Avi Kivity 已提交
2256 2257

out_free:
2258
	kmem_cache_destroy(kvm_vcpu_cache);
2259
out_free_5:
2260
	sysdev_unregister(&kvm_sysdev);
2261
out_free_4:
2262
	sysdev_class_unregister(&kvm_sysdev_class);
2263
out_free_3:
A
Avi Kivity 已提交
2264
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
2265
	unregister_cpu_notifier(&kvm_cpu_notifier);
2266 2267
out_free_2:
out_free_1:
2268
	kvm_arch_hardware_unsetup();
2269 2270
out_free_0a:
	free_cpumask_var(cpus_hardware_enabled);
2271 2272
out_free_0:
	__free_page(bad_page);
2273
out:
2274
	kvm_arch_exit();
2275
out_fail:
A
Avi Kivity 已提交
2276 2277
	return r;
}
2278
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
2279

2280
void kvm_exit(void)
A
Avi Kivity 已提交
2281
{
2282
	kvm_exit_debug();
A
Avi Kivity 已提交
2283
	misc_deregister(&kvm_dev);
2284
	kmem_cache_destroy(kvm_vcpu_cache);
2285 2286
	sysdev_unregister(&kvm_sysdev);
	sysdev_class_unregister(&kvm_sysdev_class);
A
Avi Kivity 已提交
2287
	unregister_reboot_notifier(&kvm_reboot_notifier);
2288
	unregister_cpu_notifier(&kvm_cpu_notifier);
2289
	on_each_cpu(hardware_disable, NULL, 1);
2290
	kvm_arch_hardware_unsetup();
2291
	kvm_arch_exit();
2292
	free_cpumask_var(cpus_hardware_enabled);
2293
	__free_page(bad_page);
A
Avi Kivity 已提交
2294
}
2295
EXPORT_SYMBOL_GPL(kvm_exit);