kvm_main.c 67.9 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "iodev.h"
A
Avi Kivity 已提交
20

21
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/syscore_ops.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
35
#include <linux/sched.h>
36 37
#include <linux/cpumask.h>
#include <linux/smp.h>
38
#include <linux/anon_inodes.h>
39
#include <linux/profile.h>
40
#include <linux/kvm_para.h>
41
#include <linux/pagemap.h>
42
#include <linux/mman.h>
43
#include <linux/swap.h>
44
#include <linux/bitops.h>
45
#include <linux/spinlock.h>
46
#include <linux/compat.h>
47
#include <linux/srcu.h>
J
Joerg Roedel 已提交
48
#include <linux/hugetlb.h>
49
#include <linux/slab.h>
50 51
#include <linux/sort.h>
#include <linux/bsearch.h>
A
Avi Kivity 已提交
52

A
Avi Kivity 已提交
53 54 55
#include <asm/processor.h>
#include <asm/io.h>
#include <asm/uaccess.h>
56
#include <asm/pgtable.h>
A
Avi Kivity 已提交
57

58
#include "coalesced_mmio.h"
59
#include "async_pf.h"
60

61 62 63
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

A
Avi Kivity 已提交
64 65 66
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

67 68 69
/*
 * Ordering of locks:
 *
70
 * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
71 72
 */

73
DEFINE_RAW_SPINLOCK(kvm_lock);
74
LIST_HEAD(vm_list);
75

76
static cpumask_var_t cpus_hardware_enabled;
77 78
static int kvm_usage_count = 0;
static atomic_t hardware_enable_failed;
79

80 81
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
82

83 84
static __read_mostly struct preempt_ops kvm_preempt_ops;

85
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
86

A
Avi Kivity 已提交
87 88
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
89 90 91 92
#ifdef CONFIG_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
				  unsigned long arg);
#endif
93 94
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
95

M
Marcelo Tosatti 已提交
96 97
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);

98 99
bool kvm_rebooting;
EXPORT_SYMBOL_GPL(kvm_rebooting);
100

101 102
static bool largepages_enabled = true;

X
Xiao Guangrong 已提交
103
bool kvm_is_mmio_pfn(pfn_t pfn)
B
Ben-Ami Yassour 已提交
104
{
105
	if (pfn_valid(pfn)) {
106
		int reserved;
107
		struct page *tail = pfn_to_page(pfn);
108 109
		struct page *head = compound_trans_head(tail);
		reserved = PageReserved(head);
110 111
		if (head != tail) {
			/*
112 113 114 115 116 117 118 119
			 * "head" is not a dangling pointer
			 * (compound_trans_head takes care of that)
			 * but the hugepage may have been splitted
			 * from under us (and we may not hold a
			 * reference count on the head page so it can
			 * be reused before we run PageReferenced), so
			 * we've to check PageTail before returning
			 * what we just read.
120
			 */
121 122 123
			smp_rmb();
			if (PageTail(tail))
				return reserved;
124 125
		}
		return PageReserved(tail);
126
	}
B
Ben-Ami Yassour 已提交
127 128 129 130

	return true;
}

A
Avi Kivity 已提交
131 132 133
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
134
int vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
135
{
136 137
	int cpu;

138 139
	if (mutex_lock_killable(&vcpu->mutex))
		return -EINTR;
140 141 142 143 144 145 146 147
	if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
		/* The thread running this VCPU changed. */
		struct pid *oldpid = vcpu->pid;
		struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
		rcu_assign_pointer(vcpu->pid, newpid);
		synchronize_rcu();
		put_pid(oldpid);
	}
148 149
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
150
	kvm_arch_vcpu_load(vcpu, cpu);
151
	put_cpu();
152
	return 0;
A
Avi Kivity 已提交
153 154
}

155
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
156
{
157
	preempt_disable();
158
	kvm_arch_vcpu_put(vcpu);
159 160
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
161 162 163
	mutex_unlock(&vcpu->mutex);
}

164 165 166 167
static void ack_flush(void *_completed)
{
}

168
static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
169
{
170
	int i, cpu, me;
171 172
	cpumask_var_t cpus;
	bool called = true;
173 174
	struct kvm_vcpu *vcpu;

175
	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
176

177
	me = get_cpu();
178
	kvm_for_each_vcpu(i, vcpu, kvm) {
179
		kvm_make_request(req, vcpu);
180
		cpu = vcpu->cpu;
181 182 183 184 185 186

		/* Set ->requests bit before we read ->mode */
		smp_mb();

		if (cpus != NULL && cpu != -1 && cpu != me &&
		      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
187
			cpumask_set_cpu(cpu, cpus);
188
	}
189 190 191 192 193 194
	if (unlikely(cpus == NULL))
		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
	else if (!cpumask_empty(cpus))
		smp_call_function_many(cpus, ack_flush, NULL, 1);
	else
		called = false;
195
	put_cpu();
196
	free_cpumask_var(cpus);
197
	return called;
198 199
}

200
void kvm_flush_remote_tlbs(struct kvm *kvm)
201
{
202
	long dirty_count = kvm->tlbs_dirty;
203 204

	smp_mb();
205 206
	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
		++kvm->stat.remote_tlb_flush;
207
	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
208 209
}

210 211 212 213
void kvm_reload_remote_mmus(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
}
214

215 216 217 218 219
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}

220 221 222 223 224
void kvm_make_update_eoibitmap_request(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_EOIBITMAP);
}

R
Rusty Russell 已提交
225 226 227 228 229 230 231 232 233
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
234
	vcpu->pid = NULL;
E
Eddie Dong 已提交
235
	init_waitqueue_head(&vcpu->wq);
236
	kvm_async_pf_vcpu_init(vcpu);
R
Rusty Russell 已提交
237 238 239 240 241 242 243 244

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

245 246 247
	kvm_vcpu_set_in_spin_loop(vcpu, false);
	kvm_vcpu_set_dy_eligible(vcpu, false);

248
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
249
	if (r < 0)
250
		goto fail_free_run;
R
Rusty Russell 已提交
251 252 253 254 255
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
256
	return r;
R
Rusty Russell 已提交
257 258 259 260 261
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
262
	put_pid(vcpu->pid);
263
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
264 265 266 267
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

268 269 270 271 272 273 274 275 276 277 278
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
279
	int need_tlb_flush, idx;
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
299
	idx = srcu_read_lock(&kvm->srcu);
300
	spin_lock(&kvm->mmu_lock);
301

302
	kvm->mmu_notifier_seq++;
303
	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
304 305 306 307
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

308 309
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
310 311
}

312 313 314 315 316 317
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
318
	int idx;
319

320
	idx = srcu_read_lock(&kvm->srcu);
321 322 323 324
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	kvm_set_spte_hva(kvm, address, pte);
	spin_unlock(&kvm->mmu_lock);
325
	srcu_read_unlock(&kvm->srcu, idx);
326 327
}

328 329 330 331 332 333
static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
334
	int need_tlb_flush = 0, idx;
335

336
	idx = srcu_read_lock(&kvm->srcu);
337 338 339 340 341 342 343
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
344
	need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
345
	need_tlb_flush |= kvm->tlbs_dirty;
346 347 348
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
349 350 351

	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
368
	smp_wmb();
369 370
	/*
	 * The above sequence increase must be visible before the
371 372
	 * below count decrease, which is ensured by the smp_wmb above
	 * in conjunction with the smp_rmb in mmu_notifier_retry().
373 374 375 376 377 378 379 380 381 382 383 384
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
					      unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
385
	int young, idx;
386

387
	idx = srcu_read_lock(&kvm->srcu);
388 389
	spin_lock(&kvm->mmu_lock);

390
	young = kvm_age_hva(kvm, address);
391 392 393
	if (young)
		kvm_flush_remote_tlbs(kvm);

394 395 396
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

397 398 399
	return young;
}

A
Andrea Arcangeli 已提交
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
				       struct mm_struct *mm,
				       unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	young = kvm_test_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

416 417 418 419
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
420 421 422
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
423
	kvm_arch_flush_shadow_all(kvm);
424
	srcu_read_unlock(&kvm->srcu, idx);
425 426
}

427 428 429 430 431
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
A
Andrea Arcangeli 已提交
432
	.test_young		= kvm_mmu_notifier_test_young,
433
	.change_pte		= kvm_mmu_notifier_change_pte,
434
	.release		= kvm_mmu_notifier_release,
435
};
436 437 438 439 440 441 442 443 444 445 446 447 448 449

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

450 451
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

452 453 454 455 456 457
static void kvm_init_memslots_id(struct kvm *kvm)
{
	int i;
	struct kvm_memslots *slots = kvm->memslots;

	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
458
		slots->id_to_index[i] = slots->memslots[i].id = i;
459 460
}

461
static struct kvm *kvm_create_vm(unsigned long type)
A
Avi Kivity 已提交
462
{
463 464
	int r, i;
	struct kvm *kvm = kvm_arch_alloc_vm();
A
Avi Kivity 已提交
465

466 467 468
	if (!kvm)
		return ERR_PTR(-ENOMEM);

469
	r = kvm_arch_init_vm(kvm, type);
470 471
	if (r)
		goto out_err_nodisable;
472 473 474 475 476

	r = hardware_enable_all();
	if (r)
		goto out_err_nodisable;

477 478
#ifdef CONFIG_HAVE_KVM_IRQCHIP
	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
479
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
480
#endif
A
Avi Kivity 已提交
481

482 483
	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);

484 485 486
	r = -ENOMEM;
	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
	if (!kvm->memslots)
487
		goto out_err_nosrcu;
488
	kvm_init_memslots_id(kvm);
489
	if (init_srcu_struct(&kvm->srcu))
490
		goto out_err_nosrcu;
M
Marcelo Tosatti 已提交
491 492 493
	for (i = 0; i < KVM_NR_BUSES; i++) {
		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
					GFP_KERNEL);
494
		if (!kvm->buses[i])
M
Marcelo Tosatti 已提交
495 496
			goto out_err;
	}
497

498
	spin_lock_init(&kvm->mmu_lock);
499 500
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
G
Gregory Haskins 已提交
501
	kvm_eventfd_init(kvm);
S
Shaohua Li 已提交
502
	mutex_init(&kvm->lock);
503
	mutex_init(&kvm->irq_lock);
504
	mutex_init(&kvm->slots_lock);
I
Izik Eidus 已提交
505
	atomic_set(&kvm->users_count, 1);
506 507 508 509 510

	r = kvm_init_mmu_notifier(kvm);
	if (r)
		goto out_err;

511
	raw_spin_lock(&kvm_lock);
512
	list_add(&kvm->vm_list, &vm_list);
513
	raw_spin_unlock(&kvm_lock);
514

515
	return kvm;
516 517

out_err:
518 519
	cleanup_srcu_struct(&kvm->srcu);
out_err_nosrcu:
520 521
	hardware_disable_all();
out_err_nodisable:
M
Marcelo Tosatti 已提交
522 523
	for (i = 0; i < KVM_NR_BUSES; i++)
		kfree(kvm->buses[i]);
524
	kfree(kvm->memslots);
525
	kvm_arch_free_vm(kvm);
526
	return ERR_PTR(r);
527 528
}

529 530 531 532
/*
 * Avoid using vmalloc for a small buffer.
 * Should not be used when the size is statically known.
 */
533
void *kvm_kvzalloc(unsigned long size)
534 535 536 537 538 539 540
{
	if (size > PAGE_SIZE)
		return vzalloc(size);
	else
		return kzalloc(size, GFP_KERNEL);
}

541
void kvm_kvfree(const void *addr)
542 543 544 545 546 547 548
{
	if (is_vmalloc_addr(addr))
		vfree(addr);
	else
		kfree(addr);
}

549 550 551 552 553
static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
	if (!memslot->dirty_bitmap)
		return;

554
	kvm_kvfree(memslot->dirty_bitmap);
555 556 557
	memslot->dirty_bitmap = NULL;
}

A
Avi Kivity 已提交
558 559 560 561 562 563 564
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
565
		kvm_destroy_dirty_bitmap(free);
A
Avi Kivity 已提交
566

567
	kvm_arch_free_memslot(free, dont);
M
Marcelo Tosatti 已提交
568

A
Avi Kivity 已提交
569 570 571
	free->npages = 0;
}

572
void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
573
{
574
	struct kvm_memslots *slots = kvm->memslots;
575
	struct kvm_memory_slot *memslot;
576

577 578
	kvm_for_each_memslot(memslot, slots)
		kvm_free_physmem_slot(memslot, NULL);
A
Avi Kivity 已提交
579

580
	kfree(kvm->memslots);
A
Avi Kivity 已提交
581 582
}

583 584
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
585
	int i;
586 587
	struct mm_struct *mm = kvm->mm;

588
	kvm_arch_sync_events(kvm);
589
	raw_spin_lock(&kvm_lock);
590
	list_del(&kvm->vm_list);
591
	raw_spin_unlock(&kvm_lock);
592
	kvm_free_irq_routing(kvm);
M
Marcelo Tosatti 已提交
593 594
	for (i = 0; i < KVM_NR_BUSES; i++)
		kvm_io_bus_destroy(kvm->buses[i]);
595
	kvm_coalesced_mmio_free(kvm);
596 597
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
598
#else
599
	kvm_arch_flush_shadow_all(kvm);
600
#endif
601
	kvm_arch_destroy_vm(kvm);
602 603 604
	kvm_free_physmem(kvm);
	cleanup_srcu_struct(&kvm->srcu);
	kvm_arch_free_vm(kvm);
605
	hardware_disable_all();
606
	mmdrop(mm);
607 608
}

I
Izik Eidus 已提交
609 610 611 612 613 614 615 616 617 618 619 620 621 622
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


623 624 625 626
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
627 628
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
629
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
630 631 632
	return 0;
}

633 634
/*
 * Allocation size is twice as large as the actual dirty bitmap size.
635
 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
636
 */
637 638
static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
{
639
#ifndef CONFIG_S390
640
	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
641

642
	memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
643 644 645
	if (!memslot->dirty_bitmap)
		return -ENOMEM;

646
#endif /* !CONFIG_S390 */
647 648 649
	return 0;
}

650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
static int cmp_memslot(const void *slot1, const void *slot2)
{
	struct kvm_memory_slot *s1, *s2;

	s1 = (struct kvm_memory_slot *)slot1;
	s2 = (struct kvm_memory_slot *)slot2;

	if (s1->npages < s2->npages)
		return 1;
	if (s1->npages > s2->npages)
		return -1;

	return 0;
}

/*
 * Sort the memslots base on its size, so the larger slots
 * will get better fit.
 */
static void sort_memslots(struct kvm_memslots *slots)
{
671 672
	int i;

673 674
	sort(slots->memslots, KVM_MEM_SLOTS_NUM,
	      sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
675 676 677

	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
		slots->id_to_index[slots->memslots[i].id] = i;
678 679
}

680 681
void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new,
		     u64 last_generation)
682 683 684
{
	if (new) {
		int id = new->id;
685
		struct kvm_memory_slot *old = id_to_memslot(slots, id);
686
		unsigned long npages = old->npages;
687

688
		*old = *new;
689 690
		if (new->npages != npages)
			sort_memslots(slots);
691 692
	}

693
	slots->generation = last_generation + 1;
694 695
}

696 697
static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
{
X
Xiao Guangrong 已提交
698 699 700 701 702 703 704
	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;

#ifdef KVM_CAP_READONLY_MEM
	valid_flags |= KVM_MEM_READONLY;
#endif

	if (mem->flags & ~valid_flags)
705 706 707 708 709
		return -EINVAL;

	return 0;
}

710 711 712 713 714 715 716 717 718 719 720
static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
		struct kvm_memslots *slots, struct kvm_memory_slot *new)
{
	struct kvm_memslots *old_memslots = kvm->memslots;

	update_memslots(slots, new, kvm->memslots->generation);
	rcu_assign_pointer(kvm->memslots, slots);
	synchronize_srcu_expedited(&kvm->srcu);
	return old_memslots; 
}

721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
/*
 * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations:
 * - create a new memory slot
 * - delete an existing memory slot
 * - modify an existing memory slot
 *   -- move it in the guest physical memory space
 *   -- just change its flags
 *
 * Since flags can be changed by some of these operations, the following
 * differentiation is the best we can do for __kvm_set_memory_region():
 */
enum kvm_mr_change {
	KVM_MR_CREATE,
	KVM_MR_DELETE,
	KVM_MR_MOVE,
	KVM_MR_FLAGS_ONLY,
};

A
Avi Kivity 已提交
739 740 741 742 743
/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
744
 *
745
 * Must be called holding mmap_sem for write.
A
Avi Kivity 已提交
746
 */
747 748
int __kvm_set_memory_region(struct kvm *kvm,
			    struct kvm_userspace_memory_region *mem,
749
			    bool user_alloc)
A
Avi Kivity 已提交
750
{
751
	int r;
A
Avi Kivity 已提交
752
	gfn_t base_gfn;
753
	unsigned long npages;
754
	struct kvm_memory_slot *slot;
A
Avi Kivity 已提交
755
	struct kvm_memory_slot old, new;
756
	struct kvm_memslots *slots = NULL, *old_memslots;
757
	bool old_iommu_mapped;
758
	enum kvm_mr_change change;
A
Avi Kivity 已提交
759

760 761 762 763
	r = check_memory_region_flags(mem);
	if (r)
		goto out;

A
Avi Kivity 已提交
764 765 766 767 768 769
	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
770 771 772
	/* We can read the guest memory with __xxx_user() later on. */
	if (user_alloc &&
	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
773 774 775
	     !access_ok(VERIFY_WRITE,
			(void __user *)(unsigned long)mem->userspace_addr,
			mem->memory_size)))
776
		goto out;
777
	if (mem->slot >= KVM_MEM_SLOTS_NUM)
A
Avi Kivity 已提交
778 779 780 781
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

782
	slot = id_to_memslot(kvm->memslots, mem->slot);
A
Avi Kivity 已提交
783 784 785
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

786 787 788 789
	r = -EINVAL;
	if (npages > KVM_MEM_MAX_NR_PAGES)
		goto out;

A
Avi Kivity 已提交
790 791 792
	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

793
	new = old = *slot;
A
Avi Kivity 已提交
794

795
	new.id = mem->slot;
A
Avi Kivity 已提交
796 797 798 799
	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

800 801
	old_iommu_mapped = old.npages;

A
Avi Kivity 已提交
802
	r = -EINVAL;
803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
	if (npages) {
		if (!old.npages)
			change = KVM_MR_CREATE;
		else { /* Modify an existing slot. */
			if ((mem->userspace_addr != old.userspace_addr) ||
			    (npages != old.npages))
				goto out;

			if (base_gfn != old.base_gfn)
				change = KVM_MR_MOVE;
			else if (new.flags != old.flags)
				change = KVM_MR_FLAGS_ONLY;
			else { /* Nothing to change. */
				r = 0;
				goto out;
			}
		}
	} else if (old.npages) {
		change = KVM_MR_DELETE;
	} else /* Modify a non-existent slot: disallowed. */
823
		goto out;
A
Avi Kivity 已提交
824

825
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
826 827 828
		/* Check for overlaps */
		r = -EEXIST;
		kvm_for_each_memslot(slot, kvm->memslots) {
829 830
			if ((slot->id >= KVM_USER_MEM_SLOTS) ||
			    (slot->id == mem->slot))
831 832 833 834 835
				continue;
			if (!((base_gfn + npages <= slot->base_gfn) ||
			      (base_gfn >= slot->base_gfn + slot->npages)))
				goto out;
		}
A
Avi Kivity 已提交
836 837 838 839
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
840
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
841 842

	r = -ENOMEM;
843
	if (change == KVM_MR_CREATE) {
844 845
		new.user_alloc = user_alloc;
		new.userspace_addr = mem->userspace_addr;
846

847 848
		if (kvm_arch_create_memslot(&new, npages))
			goto out_free;
A
Avi Kivity 已提交
849
	}
850

A
Avi Kivity 已提交
851 852
	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
853
		if (kvm_create_dirty_bitmap(&new) < 0)
854
			goto out_free;
A
Avi Kivity 已提交
855 856
	}

857
	if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
858
		r = -ENOMEM;
859 860
		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
				GFP_KERNEL);
861 862
		if (!slots)
			goto out_free;
863 864 865
		slot = id_to_memslot(slots, mem->slot);
		slot->flags |= KVM_MEMSLOT_INVALID;

866
		old_memslots = install_new_memslots(kvm, slots, NULL);
867

868 869
		/* slot was deleted or moved, clear iommu mapping */
		kvm_iommu_unmap_pages(kvm, &old);
870
		old_iommu_mapped = false;
871 872
		/* From this point no new shadow pages pointing to a deleted,
		 * or moved, memslot will be created.
873 874 875 876 877
		 *
		 * validation of sp->gfn happens in:
		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
		 * 	- kvm_is_visible_gfn (mmu_check_roots)
		 */
878
		kvm_arch_flush_shadow_memslot(kvm, slot);
879
		slots = old_memslots;
880
	}
881

882 883
	r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
	if (r)
884
		goto out_slots;
885

886
	r = -ENOMEM;
887 888 889 890 891 892 893 894 895 896 897
	/*
	 * We can re-use the old_memslots from above, the only difference
	 * from the currently installed memslots is the invalid flag.  This
	 * will get overwritten by update_memslots anyway.
	 */
	if (!slots) {
		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
				GFP_KERNEL);
		if (!slots)
			goto out_free;
	}
898

899 900 901 902 903 904 905 906 907
	/*
	 * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
	 * un-mapped and re-mapped if their base changes or if flags that the
	 * iommu cares about change (read-only).  Base change unmapping is
	 * handled above with slot deletion, so we only unmap incompatible
	 * flags here.  Anything else the iommu might care about for existing
	 * slots (size changes, userspace addr changes) is disallowed above,
	 * so any other attribute changes getting here can be skipped.
	 */
908
	if (change != KVM_MR_DELETE) {
909 910 911 912 913 914 915 916 917 918 919
		if (old_iommu_mapped &&
		    ((new.flags ^ old.flags) & KVM_MEM_READONLY)) {
			kvm_iommu_unmap_pages(kvm, &old);
			old_iommu_mapped = false;
		}

		if (!old_iommu_mapped) {
			r = kvm_iommu_map_pages(kvm, &new);
			if (r)
				goto out_slots;
		}
920 921
	}

922
	/* actual memory is freed via old in kvm_free_physmem_slot below */
923
	if (change == KVM_MR_DELETE) {
924
		new.dirty_bitmap = NULL;
925
		memset(&new.arch, 0, sizeof(new.arch));
926 927
	}

928
	old_memslots = install_new_memslots(kvm, slots, &new);
929

930
	kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
931

932 933 934
	kvm_free_physmem_slot(&old, &new);
	kfree(old_memslots);

A
Avi Kivity 已提交
935 936
	return 0;

937 938
out_slots:
	kfree(slots);
939
out_free:
A
Avi Kivity 已提交
940 941 942
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
943
}
944 945 946 947
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
			  struct kvm_userspace_memory_region *mem,
948
			  bool user_alloc)
949 950 951
{
	int r;

952
	mutex_lock(&kvm->slots_lock);
953
	r = __kvm_set_memory_region(kvm, mem, user_alloc);
954
	mutex_unlock(&kvm->slots_lock);
955 956
	return r;
}
957 958
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

959 960 961
int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
				   struct
				   kvm_userspace_memory_region *mem,
962
				   bool user_alloc)
963
{
964
	if (mem->slot >= KVM_USER_MEM_SLOTS)
965
		return -EINVAL;
966
	return kvm_set_memory_region(kvm, mem, user_alloc);
A
Avi Kivity 已提交
967 968
}

969 970
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
971 972 973
{
	struct kvm_memory_slot *memslot;
	int r, i;
974
	unsigned long n;
A
Avi Kivity 已提交
975 976 977
	unsigned long any = 0;

	r = -EINVAL;
978
	if (log->slot >= KVM_USER_MEM_SLOTS)
A
Avi Kivity 已提交
979 980
		goto out;

981
	memslot = id_to_memslot(kvm->memslots, log->slot);
A
Avi Kivity 已提交
982 983 984 985
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

986
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
987

988
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
989 990 991 992 993 994
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

995 996
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
997 998 999 1000 1001 1002

	r = 0;
out:
	return r;
}

1003 1004 1005 1006 1007
bool kvm_largepages_enabled(void)
{
	return largepages_enabled;
}

1008 1009 1010 1011 1012 1013
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

1014 1015 1016 1017
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
A
Avi Kivity 已提交
1018
EXPORT_SYMBOL_GPL(gfn_to_memslot);
A
Avi Kivity 已提交
1019

1020 1021
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
1022
	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1023

1024
	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1025 1026
	      memslot->flags & KVM_MEMSLOT_INVALID)
		return 0;
1027

1028
	return 1;
1029 1030 1031
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

J
Joerg Roedel 已提交
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

X
Xiao Guangrong 已提交
1056 1057 1058 1059 1060 1061 1062
static bool memslot_is_readonly(struct kvm_memory_slot *slot)
{
	return slot->flags & KVM_MEM_READONLY;
}

static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				       gfn_t *nr_pages, bool write)
I
Izik Eidus 已提交
1063
{
1064
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
X
Xiao Guangrong 已提交
1065
		return KVM_HVA_ERR_BAD;
1066

X
Xiao Guangrong 已提交
1067 1068
	if (memslot_is_readonly(slot) && write)
		return KVM_HVA_ERR_RO_BAD;
1069 1070 1071 1072

	if (nr_pages)
		*nr_pages = slot->npages - (gfn - slot->base_gfn);

X
Xiao Guangrong 已提交
1073
	return __gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
1074
}
1075

X
Xiao Guangrong 已提交
1076 1077 1078 1079
static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				     gfn_t *nr_pages)
{
	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
I
Izik Eidus 已提交
1080
}
1081

X
Xiao Guangrong 已提交
1082 1083 1084 1085 1086 1087 1088
unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
				 gfn_t gfn)
{
	return gfn_to_hva_many(slot, gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);

1089 1090
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
1091
	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1092
}
1093
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1094

1095 1096 1097 1098 1099 1100
/*
 * The hva returned by this function is only allowed to be read.
 * It should pair with kvm_read_hva() or kvm_read_hva_atomic().
 */
static unsigned long gfn_to_hva_read(struct kvm *kvm, gfn_t gfn)
{
X
Xiao Guangrong 已提交
1101
	return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false);
1102 1103 1104
}

static int kvm_read_hva(void *data, void __user *hva, int len)
1105
{
1106 1107 1108 1109 1110 1111
	return __copy_from_user(data, hva, len);
}

static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
{
	return __copy_from_user_inatomic(data, hva, len);
1112 1113
}

1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
	unsigned long start, int write, struct page **page)
{
	int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;

	if (write)
		flags |= FOLL_WRITE;

	return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
}

1125 1126 1127 1128 1129 1130 1131 1132 1133
static inline int check_user_page_hwpoison(unsigned long addr)
{
	int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;

	rc = __get_user_pages(current, current->mm, addr, 1,
			      flags, NULL, NULL, NULL);
	return rc == -EHWPOISON;
}

X
Xiao Guangrong 已提交
1134 1135 1136 1137 1138 1139
/*
 * The atomic path to get the writable pfn which will be stored in @pfn,
 * true indicates success, otherwise false is returned.
 */
static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
			    bool write_fault, bool *writable, pfn_t *pfn)
A
Avi Kivity 已提交
1140
{
1141
	struct page *page[1];
X
Xiao Guangrong 已提交
1142
	int npages;
A
Avi Kivity 已提交
1143

X
Xiao Guangrong 已提交
1144 1145
	if (!(async || atomic))
		return false;
1146

1147 1148 1149 1150 1151 1152 1153
	/*
	 * Fast pin a writable pfn only if it is a write fault request
	 * or the caller allows to map a writable pfn for a read fault
	 * request.
	 */
	if (!(write_fault || writable))
		return false;
1154

X
Xiao Guangrong 已提交
1155 1156 1157
	npages = __get_user_pages_fast(addr, 1, 1, page);
	if (npages == 1) {
		*pfn = page_to_pfn(page[0]);
1158

X
Xiao Guangrong 已提交
1159 1160 1161 1162
		if (writable)
			*writable = true;
		return true;
	}
1163

X
Xiao Guangrong 已提交
1164 1165
	return false;
}
1166

X
Xiao Guangrong 已提交
1167 1168 1169 1170 1171 1172 1173 1174 1175
/*
 * The slow path to get the pfn of the specified host virtual address,
 * 1 indicates success, -errno is returned if error is detected.
 */
static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
			   bool *writable, pfn_t *pfn)
{
	struct page *page[1];
	int npages = 0;
1176

X
Xiao Guangrong 已提交
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193
	might_sleep();

	if (writable)
		*writable = write_fault;

	if (async) {
		down_read(&current->mm->mmap_sem);
		npages = get_user_page_nowait(current, current->mm,
					      addr, write_fault, page);
		up_read(&current->mm->mmap_sem);
	} else
		npages = get_user_pages_fast(addr, 1, write_fault,
					     page);
	if (npages != 1)
		return npages;

	/* map read fault as writable if possible */
1194
	if (unlikely(!write_fault) && writable) {
X
Xiao Guangrong 已提交
1195 1196 1197 1198 1199 1200 1201
		struct page *wpage[1];

		npages = __get_user_pages_fast(addr, 1, 1, wpage);
		if (npages == 1) {
			*writable = true;
			put_page(page[0]);
			page[0] = wpage[0];
1202
		}
X
Xiao Guangrong 已提交
1203 1204

		npages = 1;
1205
	}
X
Xiao Guangrong 已提交
1206 1207 1208
	*pfn = page_to_pfn(page[0]);
	return npages;
}
I
Izik Eidus 已提交
1209

X
Xiao Guangrong 已提交
1210 1211 1212 1213
static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
{
	if (unlikely(!(vma->vm_flags & VM_READ)))
		return false;
1214

X
Xiao Guangrong 已提交
1215 1216
	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
		return false;
1217

X
Xiao Guangrong 已提交
1218 1219
	return true;
}
1220

1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
/*
 * Pin guest page in memory and return its pfn.
 * @addr: host virtual address which maps memory to the guest
 * @atomic: whether this function can sleep
 * @async: whether this function need to wait IO complete if the
 *         host page is not in the memory
 * @write_fault: whether we should get a writable host page
 * @writable: whether it allows to map a writable host page for !@write_fault
 *
 * The function will map a writable host page for these two cases:
 * 1): @write_fault = true
 * 2): @write_fault = false && @writable, @writable will tell the caller
 *     whether the mapping is writable.
 */
X
Xiao Guangrong 已提交
1235 1236 1237 1238 1239 1240
static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
			bool write_fault, bool *writable)
{
	struct vm_area_struct *vma;
	pfn_t pfn = 0;
	int npages;
1241

X
Xiao Guangrong 已提交
1242 1243
	/* we can do it either atomically or asynchronously, not both */
	BUG_ON(atomic && async);
1244

X
Xiao Guangrong 已提交
1245 1246 1247 1248 1249 1250 1251 1252 1253
	if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
		return pfn;

	if (atomic)
		return KVM_PFN_ERR_FAULT;

	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
	if (npages == 1)
		return pfn;
1254

X
Xiao Guangrong 已提交
1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
	down_read(&current->mm->mmap_sem);
	if (npages == -EHWPOISON ||
	      (!async && check_user_page_hwpoison(addr))) {
		pfn = KVM_PFN_ERR_HWPOISON;
		goto exit;
	}

	vma = find_vma_intersection(current->mm, addr, addr + 1);

	if (vma == NULL)
		pfn = KVM_PFN_ERR_FAULT;
	else if ((vma->vm_flags & VM_PFNMAP)) {
		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
			vma->vm_pgoff;
		BUG_ON(!kvm_is_mmio_pfn(pfn));
	} else {
X
Xiao Guangrong 已提交
1271
		if (async && vma_is_valid(vma, write_fault))
X
Xiao Guangrong 已提交
1272 1273 1274 1275 1276
			*async = true;
		pfn = KVM_PFN_ERR_FAULT;
	}
exit:
	up_read(&current->mm->mmap_sem);
1277
	return pfn;
1278 1279
}

X
Xiao Guangrong 已提交
1280 1281 1282
static pfn_t
__gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
		     bool *async, bool write_fault, bool *writable)
1283
{
X
Xiao Guangrong 已提交
1284 1285 1286 1287 1288 1289
	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);

	if (addr == KVM_HVA_ERR_RO_BAD)
		return KVM_PFN_ERR_RO_FAULT;

	if (kvm_is_error_hva(addr))
1290
		return KVM_PFN_NOSLOT;
X
Xiao Guangrong 已提交
1291 1292 1293 1294 1295 1296 1297 1298 1299

	/* Do not map writable pfn in the readonly memslot. */
	if (writable && memslot_is_readonly(slot)) {
		*writable = false;
		writable = NULL;
	}

	return hva_to_pfn(addr, atomic, async, write_fault,
			  writable);
1300 1301
}

1302 1303
static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
			  bool write_fault, bool *writable)
1304
{
X
Xiao Guangrong 已提交
1305
	struct kvm_memory_slot *slot;
1306

1307 1308 1309
	if (async)
		*async = false;

X
Xiao Guangrong 已提交
1310
	slot = gfn_to_memslot(kvm, gfn);
1311

X
Xiao Guangrong 已提交
1312 1313
	return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
				    writable);
1314 1315 1316 1317
}

pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
{
1318
	return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1319 1320 1321
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);

1322 1323
pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
		       bool write_fault, bool *writable)
1324
{
1325
	return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1326 1327 1328
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_async);

1329 1330
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
{
1331
	return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1332
}
1333 1334
EXPORT_SYMBOL_GPL(gfn_to_pfn);

1335 1336 1337 1338 1339 1340 1341
pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
		      bool *writable)
{
	return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);

1342
pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1343
{
X
Xiao Guangrong 已提交
1344
	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1345 1346
}

1347
pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1348
{
X
Xiao Guangrong 已提交
1349
	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1350
}
1351
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1352

1353 1354 1355 1356 1357 1358
int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
								  int nr_pages)
{
	unsigned long addr;
	gfn_t entry;

1359
	addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
	if (kvm_is_error_hva(addr))
		return -1;

	if (entry < nr_pages)
		return 0;

	return __get_user_pages_fast(addr, nr_pages, 1, pages);
}
EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);

X
Xiao Guangrong 已提交
1370 1371
static struct page *kvm_pfn_to_page(pfn_t pfn)
{
1372
	if (is_error_noslot_pfn(pfn))
1373
		return KVM_ERR_PTR_BAD_PAGE;
X
Xiao Guangrong 已提交
1374

1375 1376
	if (kvm_is_mmio_pfn(pfn)) {
		WARN_ON(1);
1377
		return KVM_ERR_PTR_BAD_PAGE;
1378
	}
X
Xiao Guangrong 已提交
1379 1380 1381 1382

	return pfn_to_page(pfn);
}

1383 1384
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
1385 1386 1387 1388
	pfn_t pfn;

	pfn = gfn_to_pfn(kvm, gfn);

X
Xiao Guangrong 已提交
1389
	return kvm_pfn_to_page(pfn);
A
Avi Kivity 已提交
1390
}
1391

A
Avi Kivity 已提交
1392 1393
EXPORT_SYMBOL_GPL(gfn_to_page);

1394 1395
void kvm_release_page_clean(struct page *page)
{
1396 1397
	WARN_ON(is_error_page(page));

1398
	kvm_release_pfn_clean(page_to_pfn(page));
1399 1400 1401
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

1402 1403
void kvm_release_pfn_clean(pfn_t pfn)
{
1404
	if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn))
1405
		put_page(pfn_to_page(pfn));
1406 1407 1408
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1409
void kvm_release_page_dirty(struct page *page)
1410
{
X
Xiao Guangrong 已提交
1411 1412
	WARN_ON(is_error_page(page));

1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

void kvm_release_pfn_dirty(pfn_t pfn)
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);

void kvm_set_page_dirty(struct page *page)
{
	kvm_set_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_set_page_dirty);

void kvm_set_pfn_dirty(pfn_t pfn)
{
1432
	if (!kvm_is_mmio_pfn(pfn)) {
1433 1434 1435 1436
		struct page *page = pfn_to_page(pfn);
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1437
}
1438 1439 1440 1441
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
1442
	if (!kvm_is_mmio_pfn(pfn))
1443
		mark_page_accessed(pfn_to_page(pfn));
1444 1445 1446 1447 1448
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
1449
	if (!kvm_is_mmio_pfn(pfn))
1450
		get_page(pfn_to_page(pfn));
1451 1452
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1453

1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
1465 1466
	int r;
	unsigned long addr;
1467

1468
	addr = gfn_to_hva_read(kvm, gfn);
1469 1470
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1471
	r = kvm_read_hva(data, (void __user *)addr + offset, len);
1472
	if (r)
1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1498 1499 1500 1501 1502 1503 1504 1505
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

1506
	addr = gfn_to_hva_read(kvm, gfn);
1507 1508
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1509
	pagefault_disable();
1510
	r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
1511
	pagefault_enable();
1512 1513 1514 1515 1516 1517
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

1518 1519 1520
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
1521 1522
	int r;
	unsigned long addr;
1523

1524 1525 1526
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1527
	r = __copy_to_user((void __user *)addr + offset, data, len);
1528
	if (r)
1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

1555 1556 1557 1558 1559 1560 1561 1562 1563
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			      gpa_t gpa)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int offset = offset_in_page(gpa);
	gfn_t gfn = gpa >> PAGE_SHIFT;

	ghc->gpa = gpa;
	ghc->generation = slots->generation;
1564
	ghc->memslot = gfn_to_memslot(kvm, gfn);
1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
	ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
	if (!kvm_is_error_hva(ghc->hva))
		ghc->hva += offset;
	else
		return -EFAULT;

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);

int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

	if (slots->generation != ghc->generation)
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

1587
	r = __copy_to_user((void __user *)ghc->hva, data, len);
1588 1589 1590 1591 1592 1593 1594 1595
	if (r)
		return -EFAULT;
	mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);

1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

	if (slots->generation != ghc->generation)
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

	r = __copy_from_user(data, (void __user *)ghc->hva, len);
	if (r)
		return -EFAULT;

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_cached);

1616 1617
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
1618 1619
	return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
				    offset, len);
1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

1642 1643
void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
			     gfn_t gfn)
A
Avi Kivity 已提交
1644
{
R
Rusty Russell 已提交
1645 1646
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1647

1648
		set_bit_le(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1649 1650 1651
	}
}

1652 1653 1654 1655 1656 1657 1658 1659
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = gfn_to_memslot(kvm, gfn);
	mark_page_dirty_in_slot(kvm, memslot, gfn);
}

E
Eddie Dong 已提交
1660 1661 1662
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1663
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1664
{
1665 1666 1667 1668 1669
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

1670
		if (kvm_arch_vcpu_runnable(vcpu)) {
1671
			kvm_make_request(KVM_REQ_UNHALT, vcpu);
1672
			break;
1673
		}
1674 1675
		if (kvm_cpu_has_pending_timer(vcpu))
			break;
1676 1677 1678
		if (signal_pending(current))
			break;

E
Eddie Dong 已提交
1679 1680
		schedule();
	}
1681

1682
	finish_wait(&vcpu->wq, &wait);
E
Eddie Dong 已提交
1683 1684
}

1685
#ifndef CONFIG_S390
1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706
/*
 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
 */
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
	int me;
	int cpu = vcpu->cpu;
	wait_queue_head_t *wqp;

	wqp = kvm_arch_vcpu_wq(vcpu);
	if (waitqueue_active(wqp)) {
		wake_up_interruptible(wqp);
		++vcpu->stat.halt_wakeup;
	}

	me = get_cpu();
	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
		if (kvm_arch_vcpu_should_kick(vcpu))
			smp_send_reschedule(cpu);
	put_cpu();
}
1707
#endif /* !CONFIG_S390 */
1708

A
Avi Kivity 已提交
1709 1710
void kvm_resched(struct kvm_vcpu *vcpu)
{
1711 1712
	if (!need_resched())
		return;
A
Avi Kivity 已提交
1713 1714 1715 1716
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

1717 1718 1719 1720
bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
{
	struct pid *pid;
	struct task_struct *task = NULL;
1721
	bool ret = false;
1722 1723 1724 1725 1726 1727 1728

	rcu_read_lock();
	pid = rcu_dereference(target->pid);
	if (pid)
		task = get_pid_task(target->pid, PIDTYPE_PID);
	rcu_read_unlock();
	if (!task)
1729
		return ret;
1730 1731
	if (task->flags & PF_VCPU) {
		put_task_struct(task);
1732
		return ret;
1733
	}
1734
	ret = yield_to(task, 1);
1735
	put_task_struct(task);
1736 1737

	return ret;
1738 1739 1740
}
EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);

1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
/*
 * Helper that checks whether a VCPU is eligible for directed yield.
 * Most eligible candidate to yield is decided by following heuristics:
 *
 *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
 *  (preempted lock holder), indicated by @in_spin_loop.
 *  Set at the beiginning and cleared at the end of interception/PLE handler.
 *
 *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
 *  chance last time (mostly it has become eligible now since we have probably
 *  yielded to lockholder in last iteration. This is done by toggling
 *  @dy_eligible each time a VCPU checked for eligibility.)
 *
 *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
 *  to preempted lock-holder could result in wrong VCPU selection and CPU
 *  burning. Giving priority for a potential lock-holder increases lock
 *  progress.
 *
 *  Since algorithm is based on heuristics, accessing another VCPU data without
 *  locking does not harm. It may result in trying to yield to  same VCPU, fail
 *  and continue with next VCPU and so on.
 */
bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
{
	bool eligible;

	eligible = !vcpu->spin_loop.in_spin_loop ||
			(vcpu->spin_loop.in_spin_loop &&
			 vcpu->spin_loop.dy_eligible);

	if (vcpu->spin_loop.in_spin_loop)
		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);

	return eligible;
}
#endif
1778

1779
void kvm_vcpu_on_spin(struct kvm_vcpu *me)
Z
Zhai, Edwin 已提交
1780
{
1781 1782 1783 1784
	struct kvm *kvm = me->kvm;
	struct kvm_vcpu *vcpu;
	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
	int yielded = 0;
1785
	int try = 3;
1786 1787
	int pass;
	int i;
Z
Zhai, Edwin 已提交
1788

1789
	kvm_vcpu_set_in_spin_loop(me, true);
1790 1791 1792 1793 1794 1795 1796
	/*
	 * We boost the priority of a VCPU that is runnable but not
	 * currently running, because it got preempted by something
	 * else and called schedule in __vcpu_run.  Hopefully that
	 * VCPU is holding the lock that we need and will release it.
	 * We approximate round-robin by starting at the last boosted VCPU.
	 */
1797
	for (pass = 0; pass < 2 && !yielded && try; pass++) {
1798
		kvm_for_each_vcpu(i, vcpu, kvm) {
1799
			if (!pass && i <= last_boosted_vcpu) {
1800 1801 1802 1803 1804 1805 1806 1807
				i = last_boosted_vcpu;
				continue;
			} else if (pass && i > last_boosted_vcpu)
				break;
			if (vcpu == me)
				continue;
			if (waitqueue_active(&vcpu->wq))
				continue;
1808 1809
			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
				continue;
1810 1811 1812

			yielded = kvm_vcpu_yield_to(vcpu);
			if (yielded > 0) {
1813 1814
				kvm->last_boosted_vcpu = i;
				break;
1815 1816 1817 1818
			} else if (yielded < 0) {
				try--;
				if (!try)
					break;
1819 1820 1821
			}
		}
	}
1822
	kvm_vcpu_set_in_spin_loop(me, false);
1823 1824 1825

	/* Ensure vcpu is not eligible during next spinloop */
	kvm_vcpu_set_dy_eligible(me, false);
Z
Zhai, Edwin 已提交
1826 1827 1828
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

1829
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1830 1831 1832 1833
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

1834
	if (vmf->pgoff == 0)
1835
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
1836
#ifdef CONFIG_X86
1837
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1838
		page = virt_to_page(vcpu->arch.pio_data);
1839 1840 1841 1842
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
1843
#endif
1844
	else
1845
		return kvm_arch_vcpu_fault(vcpu, vmf);
1846
	get_page(page);
1847 1848
	vmf->page = page;
	return 0;
1849 1850
}

1851
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1852
	.fault = kvm_vcpu_fault,
1853 1854 1855 1856 1857 1858 1859 1860
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
1861 1862 1863 1864
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
1865
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1866 1867 1868
	return 0;
}

1869
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
1870 1871
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
1872 1873 1874
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vcpu_compat_ioctl,
#endif
1875
	.mmap           = kvm_vcpu_mmap,
1876
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
1877 1878 1879 1880 1881 1882 1883
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
1884
	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
A
Avi Kivity 已提交
1885 1886
}

1887 1888 1889
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
1890
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1891 1892
{
	int r;
1893
	struct kvm_vcpu *vcpu, *v;
1894

1895
	vcpu = kvm_arch_vcpu_create(kvm, id);
R
Rusty Russell 已提交
1896 1897
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
1898

1899 1900
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

1901 1902
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
1903
		goto vcpu_destroy;
1904

S
Shaohua Li 已提交
1905
	mutex_lock(&kvm->lock);
1906 1907 1908 1909
	if (!kvm_vcpu_compatible(vcpu)) {
		r = -EINVAL;
		goto unlock_vcpu_destroy;
	}
1910 1911
	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
		r = -EINVAL;
1912
		goto unlock_vcpu_destroy;
R
Rusty Russell 已提交
1913
	}
1914

1915 1916
	kvm_for_each_vcpu(r, v, kvm)
		if (v->vcpu_id == id) {
1917
			r = -EEXIST;
1918
			goto unlock_vcpu_destroy;
1919 1920 1921
		}

	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1922

R
Rusty Russell 已提交
1923
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
1924
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
1925
	r = create_vcpu_fd(vcpu);
1926 1927
	if (r < 0) {
		kvm_put_kvm(kvm);
1928
		goto unlock_vcpu_destroy;
1929 1930 1931 1932 1933 1934 1935
	}

	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

	mutex_unlock(&kvm->lock);
1936
	kvm_arch_vcpu_postcreate(vcpu);
R
Rusty Russell 已提交
1937
	return r;
1938

1939
unlock_vcpu_destroy:
1940
	mutex_unlock(&kvm->lock);
1941
vcpu_destroy:
1942
	kvm_arch_vcpu_destroy(vcpu);
1943 1944 1945
	return r;
}

A
Avi Kivity 已提交
1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
1957 1958
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
1959
{
A
Avi Kivity 已提交
1960
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
1961
	void __user *argp = (void __user *)arg;
1962
	int r;
1963 1964
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
1965

1966 1967
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978

#if defined(CONFIG_S390) || defined(CONFIG_PPC)
	/*
	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
	 * so vcpu_load() would break it.
	 */
	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
#endif


1979 1980 1981
	r = vcpu_load(vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
1982
	switch (ioctl) {
1983
	case KVM_RUN:
1984 1985 1986
		r = -EINVAL;
		if (arg)
			goto out;
1987
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1988
		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
A
Avi Kivity 已提交
1989 1990
		break;
	case KVM_GET_REGS: {
1991
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1992

1993 1994 1995
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1996
			goto out;
1997 1998 1999
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
2000
		r = -EFAULT;
2001 2002
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
2003
		r = 0;
2004 2005
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
2006 2007 2008
		break;
	}
	case KVM_SET_REGS: {
2009
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2010

2011
		r = -ENOMEM;
2012 2013 2014
		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
		if (IS_ERR(kvm_regs)) {
			r = PTR_ERR(kvm_regs);
A
Avi Kivity 已提交
2015
			goto out;
2016
		}
2017 2018
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
		kfree(kvm_regs);
A
Avi Kivity 已提交
2019 2020 2021
		break;
	}
	case KVM_GET_SREGS: {
2022 2023 2024 2025 2026
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2027 2028 2029
		if (r)
			goto out;
		r = -EFAULT;
2030
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
2031 2032 2033 2034 2035
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
2036 2037 2038
		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
		if (IS_ERR(kvm_sregs)) {
			r = PTR_ERR(kvm_sregs);
G
Guo Chao 已提交
2039
			kvm_sregs = NULL;
A
Avi Kivity 已提交
2040
			goto out;
2041
		}
2042
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2043 2044
		break;
	}
2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		break;
	}
A
Avi Kivity 已提交
2066 2067 2068 2069
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
2070
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
2071
			goto out;
2072
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2073 2074 2075
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2076
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
2077 2078 2079 2080
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
2081 2082
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
2083 2084

		r = -EFAULT;
A
Al Viro 已提交
2085
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
2086
			goto out;
J
Jan Kiszka 已提交
2087
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
2088 2089
		break;
	}
A
Avi Kivity 已提交
2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
2110
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
A
Avi Kivity 已提交
2111 2112
		break;
	}
A
Avi Kivity 已提交
2113
	case KVM_GET_FPU: {
2114 2115 2116 2117 2118
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2119 2120 2121
		if (r)
			goto out;
		r = -EFAULT;
2122
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
2123 2124 2125 2126 2127
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
2128 2129 2130
		fpu = memdup_user(argp, sizeof(*fpu));
		if (IS_ERR(fpu)) {
			r = PTR_ERR(fpu);
G
Guo Chao 已提交
2131
			fpu = NULL;
A
Avi Kivity 已提交
2132
			goto out;
2133
		}
2134
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2135 2136
		break;
	}
A
Avi Kivity 已提交
2137
	default:
2138
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2139 2140
	}
out:
2141
	vcpu_put(vcpu);
2142 2143
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
2144 2145 2146
	return r;
}

2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176
#ifdef CONFIG_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *filp,
				  unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = compat_ptr(arg);
	int r;

	if (vcpu->kvm->mm != current->mm)
		return -EIO;

	switch (ioctl) {
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		compat_sigset_t csigset;
		sigset_t sigset;

		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof csigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&csigset, sigmask_arg->sigset,
					   sizeof csigset))
				goto out;
2177 2178 2179 2180
			sigset_from_compat(&sigset, &csigset);
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		} else
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
		break;
	}
	default:
		r = kvm_vcpu_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

A
Avi Kivity 已提交
2192 2193 2194 2195 2196
static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
2197
	int r;
A
Avi Kivity 已提交
2198

2199 2200
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
2201 2202 2203 2204
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		break;
2205 2206 2207 2208 2209 2210 2211 2212
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

2213
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, true);
A
Avi Kivity 已提交
2214 2215 2216 2217 2218 2219
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
2220
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
2221
			goto out;
2222
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
2223 2224
		break;
	}
2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		break;
	}
#endif
G
Gregory Haskins 已提交
2243 2244 2245 2246 2247 2248
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
2249
		r = kvm_irqfd(kvm, &data);
G
Gregory Haskins 已提交
2250 2251
		break;
	}
G
Gregory Haskins 已提交
2252 2253 2254 2255 2256 2257 2258 2259 2260
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
2261 2262 2263
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
2264
		mutex_lock(&kvm->lock);
2265 2266 2267 2268
		if (atomic_read(&kvm->online_vcpus) != 0)
			r = -EBUSY;
		else
			kvm->bsp_vcpu_id = arg;
2269
		mutex_unlock(&kvm->lock);
2270
		break;
2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281
#endif
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_SIGNAL_MSI: {
		struct kvm_msi msi;

		r = -EFAULT;
		if (copy_from_user(&msi, argp, sizeof msi))
			goto out;
		r = kvm_send_userspace_msi(kvm, &msi);
		break;
	}
2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304
#endif
#ifdef __KVM_HAVE_IRQ_LINE
	case KVM_IRQ_LINE_STATUS:
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
		if (copy_from_user(&irq_event, argp, sizeof irq_event))
			goto out;

		r = kvm_vm_ioctl_irq_line(kvm, &irq_event);
		if (r)
			goto out;

		r = -EFAULT;
		if (ioctl == KVM_IRQ_LINE_STATUS) {
			if (copy_to_user(argp, &irq_event, sizeof irq_event))
				goto out;
		}

		r = 0;
		break;
	}
2305
#endif
2306
	default:
2307
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2308 2309
		if (r == -ENOTTY)
			r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2310 2311 2312 2313 2314
	}
out:
	return r;
}

2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358
#ifdef CONFIG_COMPAT
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		r = -EFAULT;
		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
			goto out;
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

2359
static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2360
{
2361 2362 2363 2364
	struct page *page[1];
	unsigned long addr;
	int npages;
	gfn_t gfn = vmf->pgoff;
2365 2366
	struct kvm *kvm = vma->vm_file->private_data;

2367 2368
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
2369
		return VM_FAULT_SIGBUS;
2370 2371 2372 2373

	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
				NULL);
	if (unlikely(npages != 1))
2374
		return VM_FAULT_SIGBUS;
2375 2376

	vmf->page = page[0];
2377
	return 0;
2378 2379
}

2380
static const struct vm_operations_struct kvm_vm_vm_ops = {
2381
	.fault = kvm_vm_fault,
2382 2383 2384 2385 2386 2387 2388 2389
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

2390
static struct file_operations kvm_vm_fops = {
2391 2392
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
2393 2394 2395
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vm_compat_ioctl,
#endif
2396
	.mmap           = kvm_vm_mmap,
2397
	.llseek		= noop_llseek,
2398 2399
};

2400
static int kvm_dev_ioctl_create_vm(unsigned long type)
2401
{
2402
	int r;
2403 2404
	struct kvm *kvm;

2405
	kvm = kvm_create_vm(type);
2406 2407
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
2408 2409 2410 2411 2412 2413 2414
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	r = kvm_coalesced_mmio_init(kvm);
	if (r < 0) {
		kvm_put_kvm(kvm);
		return r;
	}
#endif
2415 2416
	r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
	if (r < 0)
A
Al Viro 已提交
2417
		kvm_put_kvm(kvm);
2418

2419
	return r;
2420 2421
}

2422 2423 2424
static long kvm_dev_ioctl_check_extension_generic(long arg)
{
	switch (arg) {
2425
	case KVM_CAP_USER_MEMORY:
2426
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2427
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2428 2429 2430
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_CAP_SET_BOOT_CPU_ID:
#endif
2431
	case KVM_CAP_INTERNAL_ERROR_DATA:
2432 2433 2434
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_CAP_SIGNAL_MSI:
#endif
2435
		return 1;
2436
#ifdef KVM_CAP_IRQ_ROUTING
2437
	case KVM_CAP_IRQ_ROUTING:
2438
		return KVM_MAX_IRQ_ROUTES;
2439
#endif
2440 2441 2442 2443 2444 2445
	default:
		break;
	}
	return kvm_dev_ioctl_check_extension(arg);
}

2446 2447 2448
static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
2449
	long r = -EINVAL;
2450 2451 2452

	switch (ioctl) {
	case KVM_GET_API_VERSION:
2453 2454 2455
		r = -EINVAL;
		if (arg)
			goto out;
2456 2457 2458
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
2459
		r = kvm_dev_ioctl_create_vm(arg);
2460
		break;
2461
	case KVM_CHECK_EXTENSION:
2462
		r = kvm_dev_ioctl_check_extension_generic(arg);
2463
		break;
2464 2465 2466 2467
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
2468 2469 2470
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
2471 2472 2473
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
		r += PAGE_SIZE;    /* coalesced mmio ring page */
2474
#endif
2475
		break;
2476 2477 2478
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
2479
		r = -EOPNOTSUPP;
2480
		break;
A
Avi Kivity 已提交
2481
	default:
2482
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2483 2484 2485 2486 2487 2488 2489 2490
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
2491
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
2492 2493 2494
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
2495
	KVM_MINOR,
A
Avi Kivity 已提交
2496 2497 2498 2499
	"kvm",
	&kvm_chardev_ops,
};

2500
static void hardware_enable_nolock(void *junk)
2501 2502
{
	int cpu = raw_smp_processor_id();
2503
	int r;
2504

2505
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2506
		return;
2507

2508
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
2509 2510 2511 2512 2513 2514 2515 2516 2517

	r = kvm_arch_hardware_enable(NULL);

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
		printk(KERN_INFO "kvm: enabling virtualization on "
				 "CPU%d failed\n", cpu);
	}
2518 2519
}

2520 2521
static void hardware_enable(void *junk)
{
2522
	raw_spin_lock(&kvm_lock);
2523
	hardware_enable_nolock(junk);
2524
	raw_spin_unlock(&kvm_lock);
2525 2526 2527
}

static void hardware_disable_nolock(void *junk)
2528 2529 2530
{
	int cpu = raw_smp_processor_id();

2531
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2532
		return;
2533
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2534
	kvm_arch_hardware_disable(NULL);
2535 2536
}

2537 2538
static void hardware_disable(void *junk)
{
2539
	raw_spin_lock(&kvm_lock);
2540
	hardware_disable_nolock(junk);
2541
	raw_spin_unlock(&kvm_lock);
2542 2543
}

2544 2545 2546 2547 2548 2549
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
2550
		on_each_cpu(hardware_disable_nolock, NULL, 1);
2551 2552 2553 2554
}

static void hardware_disable_all(void)
{
2555
	raw_spin_lock(&kvm_lock);
2556
	hardware_disable_all_nolock();
2557
	raw_spin_unlock(&kvm_lock);
2558 2559 2560 2561 2562 2563
}

static int hardware_enable_all(void)
{
	int r = 0;

2564
	raw_spin_lock(&kvm_lock);
2565 2566 2567 2568

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
2569
		on_each_cpu(hardware_enable_nolock, NULL, 1);
2570 2571 2572 2573 2574 2575 2576

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

2577
	raw_spin_unlock(&kvm_lock);
2578 2579 2580 2581

	return r;
}

A
Avi Kivity 已提交
2582 2583 2584 2585 2586
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

2587 2588 2589
	if (!kvm_usage_count)
		return NOTIFY_OK;

2590
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
2591
	switch (val) {
2592
	case CPU_DYING:
2593 2594 2595 2596
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
2597
	case CPU_STARTING:
2598 2599
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
2600
		hardware_enable(NULL);
A
Avi Kivity 已提交
2601 2602 2603 2604 2605
		break;
	}
	return NOTIFY_OK;
}

2606

2607
asmlinkage void kvm_spurious_fault(void)
2608 2609 2610 2611
{
	/* Fault while not rebooting.  We want the trace. */
	BUG();
}
2612
EXPORT_SYMBOL_GPL(kvm_spurious_fault);
2613

2614
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
2615
		      void *v)
2616
{
2617 2618 2619 2620 2621 2622 2623 2624
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
	kvm_rebooting = true;
2625
	on_each_cpu(hardware_disable_nolock, NULL, 1);
2626 2627 2628 2629 2630 2631 2632 2633
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
2634
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2635 2636 2637 2638
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
2639
		struct kvm_io_device *pos = bus->range[i].dev;
2640 2641 2642

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
2643
	kfree(bus);
2644 2645
}

2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696
int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
{
	const struct kvm_io_range *r1 = p1;
	const struct kvm_io_range *r2 = p2;

	if (r1->addr < r2->addr)
		return -1;
	if (r1->addr + r1->len > r2->addr + r2->len)
		return 1;
	return 0;
}

int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
			  gpa_t addr, int len)
{
	bus->range[bus->dev_count++] = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
		.dev = dev,
	};

	sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
		kvm_io_bus_sort_cmp, NULL);

	return 0;
}

int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
			     gpa_t addr, int len)
{
	struct kvm_io_range *range, key;
	int off;

	key = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	range = bsearch(&key, bus->range, bus->dev_count,
			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
	if (range == NULL)
		return -ENOENT;

	off = range - bus->range;

	while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0)
		off--;

	return off;
}

2697
/* kvm_io_bus_write - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2698
int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2699
		     int len, const void *val)
2700
{
2701
	int idx;
2702
	struct kvm_io_bus *bus;
2703 2704 2705 2706 2707 2708
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
2709 2710

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2711 2712 2713 2714 2715 2716 2717
	idx = kvm_io_bus_get_first_dev(bus, addr, len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
		kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
		if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val))
2718
			return 0;
2719 2720 2721
		idx++;
	}

2722 2723
	return -EOPNOTSUPP;
}
2724

2725
/* kvm_io_bus_read - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2726 2727
int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
		    int len, void *val)
2728
{
2729
	int idx;
2730
	struct kvm_io_bus *bus;
2731 2732 2733 2734 2735 2736
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
M
Marcelo Tosatti 已提交
2737

2738
	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2739 2740 2741 2742 2743 2744 2745
	idx = kvm_io_bus_get_first_dev(bus, addr, len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
		kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
		if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val))
2746
			return 0;
2747 2748 2749
		idx++;
	}

2750
	return -EOPNOTSUPP;
2751 2752
}

2753
/* Caller must hold slots_lock. */
2754 2755
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, struct kvm_io_device *dev)
2756
{
M
Marcelo Tosatti 已提交
2757
	struct kvm_io_bus *new_bus, *bus;
2758

M
Marcelo Tosatti 已提交
2759
	bus = kvm->buses[bus_idx];
2760
	if (bus->dev_count > NR_IOBUS_DEVS - 1)
2761
		return -ENOSPC;
2762

2763 2764
	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
M
Marcelo Tosatti 已提交
2765 2766
	if (!new_bus)
		return -ENOMEM;
2767 2768
	memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
	       sizeof(struct kvm_io_range)));
2769
	kvm_io_bus_insert_dev(new_bus, dev, addr, len);
M
Marcelo Tosatti 已提交
2770 2771 2772
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
2773 2774 2775 2776

	return 0;
}

2777
/* Caller must hold slots_lock. */
M
Marcelo Tosatti 已提交
2778 2779
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			      struct kvm_io_device *dev)
2780
{
M
Marcelo Tosatti 已提交
2781 2782
	int i, r;
	struct kvm_io_bus *new_bus, *bus;
2783

2784
	bus = kvm->buses[bus_idx];
M
Marcelo Tosatti 已提交
2785
	r = -ENOENT;
2786 2787
	for (i = 0; i < bus->dev_count; i++)
		if (bus->range[i].dev == dev) {
M
Marcelo Tosatti 已提交
2788
			r = 0;
2789 2790
			break;
		}
M
Marcelo Tosatti 已提交
2791

2792
	if (r)
M
Marcelo Tosatti 已提交
2793
		return r;
2794 2795 2796 2797 2798 2799 2800 2801 2802 2803

	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
	if (!new_bus)
		return -ENOMEM;

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count--;
	memcpy(new_bus->range + i, bus->range + i + 1,
	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
2804 2805 2806 2807 2808

	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
	return r;
2809 2810
}

A
Avi Kivity 已提交
2811 2812 2813 2814
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
};

2815
static int vm_stat_get(void *_offset, u64 *val)
2816 2817 2818 2819
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

2820
	*val = 0;
2821
	raw_spin_lock(&kvm_lock);
2822
	list_for_each_entry(kvm, &vm_list, vm_list)
2823
		*val += *(u32 *)((void *)kvm + offset);
2824
	raw_spin_unlock(&kvm_lock);
2825
	return 0;
2826 2827 2828 2829
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

2830
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
2831 2832 2833 2834 2835 2836
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

2837
	*val = 0;
2838
	raw_spin_lock(&kvm_lock);
A
Avi Kivity 已提交
2839
	list_for_each_entry(kvm, &vm_list, vm_list)
2840 2841 2842
		kvm_for_each_vcpu(i, vcpu, kvm)
			*val += *(u32 *)((void *)vcpu + offset);

2843
	raw_spin_unlock(&kvm_lock);
2844
	return 0;
A
Avi Kivity 已提交
2845 2846
}

2847 2848
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

2849
static const struct file_operations *stat_fops[] = {
2850 2851 2852
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
2853

2854
static int kvm_init_debug(void)
A
Avi Kivity 已提交
2855
{
2856
	int r = -EFAULT;
A
Avi Kivity 已提交
2857 2858
	struct kvm_stats_debugfs_item *p;

2859
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
2860 2861 2862 2863
	if (kvm_debugfs_dir == NULL)
		goto out;

	for (p = debugfs_entries; p->name; ++p) {
2864
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
2865
						(void *)(long)p->offset,
2866
						stat_fops[p->kind]);
2867 2868 2869 2870 2871 2872 2873 2874 2875 2876
		if (p->dentry == NULL)
			goto out_dir;
	}

	return 0;

out_dir:
	debugfs_remove_recursive(kvm_debugfs_dir);
out:
	return r;
A
Avi Kivity 已提交
2877 2878 2879 2880 2881 2882 2883 2884
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
2885
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
2886 2887
}

2888
static int kvm_suspend(void)
2889
{
2890
	if (kvm_usage_count)
2891
		hardware_disable_nolock(NULL);
2892 2893 2894
	return 0;
}

2895
static void kvm_resume(void)
2896
{
2897
	if (kvm_usage_count) {
2898
		WARN_ON(raw_spin_is_locked(&kvm_lock));
2899
		hardware_enable_nolock(NULL);
2900
	}
2901 2902
}

2903
static struct syscore_ops kvm_syscore_ops = {
2904 2905 2906 2907
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

2908 2909 2910 2911 2912 2913 2914 2915 2916 2917
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2918
	kvm_arch_vcpu_load(vcpu, cpu);
2919 2920 2921 2922 2923 2924 2925
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2926
	kvm_arch_vcpu_put(vcpu);
2927 2928
}

2929
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
2930
		  struct module *module)
A
Avi Kivity 已提交
2931 2932
{
	int r;
Y
Yang, Sheng 已提交
2933
	int cpu;
A
Avi Kivity 已提交
2934

2935 2936
	r = kvm_arch_init(opaque);
	if (r)
2937
		goto out_fail;
2938

2939
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2940 2941 2942 2943
		r = -ENOMEM;
		goto out_free_0;
	}

2944
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
2945
	if (r < 0)
2946
		goto out_free_0a;
A
Avi Kivity 已提交
2947

Y
Yang, Sheng 已提交
2948 2949
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
2950
				kvm_arch_check_processor_compat,
2951
				&r, 1);
Y
Yang, Sheng 已提交
2952
		if (r < 0)
2953
			goto out_free_1;
Y
Yang, Sheng 已提交
2954 2955
	}

A
Avi Kivity 已提交
2956 2957
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
2958
		goto out_free_2;
A
Avi Kivity 已提交
2959 2960
	register_reboot_notifier(&kvm_reboot_notifier);

2961
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
2962 2963 2964
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
J
Joe Perches 已提交
2965
					   0, NULL);
2966 2967
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
2968
		goto out_free_3;
2969 2970
	}

2971 2972 2973 2974
	r = kvm_async_pf_init();
	if (r)
		goto out_free;

A
Avi Kivity 已提交
2975
	kvm_chardev_ops.owner = module;
2976 2977
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
2978 2979 2980

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
2981
		printk(KERN_ERR "kvm: misc device register failed\n");
2982
		goto out_unreg;
A
Avi Kivity 已提交
2983 2984
	}

2985 2986
	register_syscore_ops(&kvm_syscore_ops);

2987 2988 2989
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

2990 2991 2992 2993 2994
	r = kvm_init_debug();
	if (r) {
		printk(KERN_ERR "kvm: create debugfs files failed\n");
		goto out_undebugfs;
	}
2995

2996
	return 0;
A
Avi Kivity 已提交
2997

2998 2999
out_undebugfs:
	unregister_syscore_ops(&kvm_syscore_ops);
3000 3001
out_unreg:
	kvm_async_pf_deinit();
A
Avi Kivity 已提交
3002
out_free:
3003
	kmem_cache_destroy(kvm_vcpu_cache);
3004
out_free_3:
A
Avi Kivity 已提交
3005
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
3006
	unregister_cpu_notifier(&kvm_cpu_notifier);
3007 3008
out_free_2:
out_free_1:
3009
	kvm_arch_hardware_unsetup();
3010 3011
out_free_0a:
	free_cpumask_var(cpus_hardware_enabled);
3012
out_free_0:
3013
	kvm_arch_exit();
3014
out_fail:
A
Avi Kivity 已提交
3015 3016
	return r;
}
3017
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
3018

3019
void kvm_exit(void)
A
Avi Kivity 已提交
3020
{
3021
	kvm_exit_debug();
A
Avi Kivity 已提交
3022
	misc_deregister(&kvm_dev);
3023
	kmem_cache_destroy(kvm_vcpu_cache);
3024
	kvm_async_pf_deinit();
3025
	unregister_syscore_ops(&kvm_syscore_ops);
A
Avi Kivity 已提交
3026
	unregister_reboot_notifier(&kvm_reboot_notifier);
3027
	unregister_cpu_notifier(&kvm_cpu_notifier);
3028
	on_each_cpu(hardware_disable_nolock, NULL, 1);
3029
	kvm_arch_hardware_unsetup();
3030
	kvm_arch_exit();
3031
	free_cpumask_var(cpus_hardware_enabled);
A
Avi Kivity 已提交
3032
}
3033
EXPORT_SYMBOL_GPL(kvm_exit);