kvm_main.c 56.6 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "iodev.h"
A
Avi Kivity 已提交
20

21
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/sysdev.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
35
#include <linux/sched.h>
36 37
#include <linux/cpumask.h>
#include <linux/smp.h>
38
#include <linux/anon_inodes.h>
39
#include <linux/profile.h>
40
#include <linux/kvm_para.h>
41
#include <linux/pagemap.h>
42
#include <linux/mman.h>
43
#include <linux/swap.h>
44
#include <linux/bitops.h>
45
#include <linux/spinlock.h>
46
#include <linux/compat.h>
47
#include <linux/srcu.h>
J
Joerg Roedel 已提交
48
#include <linux/hugetlb.h>
49
#include <linux/slab.h>
A
Avi Kivity 已提交
50

A
Avi Kivity 已提交
51 52 53
#include <asm/processor.h>
#include <asm/io.h>
#include <asm/uaccess.h>
54
#include <asm/pgtable.h>
55
#include <asm-generic/bitops/le.h>
A
Avi Kivity 已提交
56

57
#include "coalesced_mmio.h"
58
#include "async_pf.h"
59

60 61 62
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

A
Avi Kivity 已提交
63 64 65
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

66 67 68
/*
 * Ordering of locks:
 *
69
 * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
70 71
 */

72 73
DEFINE_SPINLOCK(kvm_lock);
LIST_HEAD(vm_list);
74

75
static cpumask_var_t cpus_hardware_enabled;
76 77
static int kvm_usage_count = 0;
static atomic_t hardware_enable_failed;
78

79 80
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
81

82 83
static __read_mostly struct preempt_ops kvm_preempt_ops;

84
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
85

A
Avi Kivity 已提交
86 87
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
88 89
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
90

M
Marcelo Tosatti 已提交
91 92
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);

93 94
bool kvm_rebooting;
EXPORT_SYMBOL_GPL(kvm_rebooting);
95

96 97
static bool largepages_enabled = true;

98 99
static struct page *hwpoison_page;
static pfn_t hwpoison_pfn;
100

101 102 103
static struct page *fault_page;
static pfn_t fault_pfn;

104
inline int kvm_is_mmio_pfn(pfn_t pfn)
B
Ben-Ami Yassour 已提交
105
{
106
	if (pfn_valid(pfn)) {
107
		int reserved;
108
		struct page *tail = pfn_to_page(pfn);
109 110
		struct page *head = compound_trans_head(tail);
		reserved = PageReserved(head);
111 112
		if (head != tail) {
			/*
113 114 115 116 117 118 119 120
			 * "head" is not a dangling pointer
			 * (compound_trans_head takes care of that)
			 * but the hugepage may have been splitted
			 * from under us (and we may not hold a
			 * reference count on the head page so it can
			 * be reused before we run PageReferenced), so
			 * we've to check PageTail before returning
			 * what we just read.
121
			 */
122 123 124
			smp_rmb();
			if (PageTail(tail))
				return reserved;
125 126
		}
		return PageReserved(tail);
127
	}
B
Ben-Ami Yassour 已提交
128 129 130 131

	return true;
}

A
Avi Kivity 已提交
132 133 134
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
135
void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
136
{
137 138
	int cpu;

A
Avi Kivity 已提交
139
	mutex_lock(&vcpu->mutex);
140 141
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
142
	kvm_arch_vcpu_load(vcpu, cpu);
143
	put_cpu();
A
Avi Kivity 已提交
144 145
}

146
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
147
{
148
	preempt_disable();
149
	kvm_arch_vcpu_put(vcpu);
150 151
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
152 153 154
	mutex_unlock(&vcpu->mutex);
}

155 156 157 158
static void ack_flush(void *_completed)
{
}

159
static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
160
{
161
	int i, cpu, me;
162 163
	cpumask_var_t cpus;
	bool called = true;
164 165
	struct kvm_vcpu *vcpu;

166
	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
167

168
	me = get_cpu();
169
	kvm_for_each_vcpu(i, vcpu, kvm) {
170
		kvm_make_request(req, vcpu);
171
		cpu = vcpu->cpu;
172 173 174 175 176 177

		/* Set ->requests bit before we read ->mode */
		smp_mb();

		if (cpus != NULL && cpu != -1 && cpu != me &&
		      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
178
			cpumask_set_cpu(cpu, cpus);
179
	}
180 181 182 183 184 185
	if (unlikely(cpus == NULL))
		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
	else if (!cpumask_empty(cpus))
		smp_call_function_many(cpus, ack_flush, NULL, 1);
	else
		called = false;
186
	put_cpu();
187
	free_cpumask_var(cpus);
188
	return called;
189 190
}

191
void kvm_flush_remote_tlbs(struct kvm *kvm)
192
{
193 194 195
	int dirty_count = kvm->tlbs_dirty;

	smp_mb();
196 197
	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
		++kvm->stat.remote_tlb_flush;
198
	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
199 200
}

201 202 203 204
void kvm_reload_remote_mmus(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
}
205

R
Rusty Russell 已提交
206 207 208 209 210 211 212 213 214
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
E
Eddie Dong 已提交
215
	init_waitqueue_head(&vcpu->wq);
216
	kvm_async_pf_vcpu_init(vcpu);
R
Rusty Russell 已提交
217 218 219 220 221 222 223 224

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

225
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
226
	if (r < 0)
227
		goto fail_free_run;
R
Rusty Russell 已提交
228 229 230 231 232
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
233
	return r;
R
Rusty Russell 已提交
234 235 236 237 238
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
239
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
240 241 242 243
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

244 245 246 247 248 249 250 251 252 253 254
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
255
	int need_tlb_flush, idx;
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
275
	idx = srcu_read_lock(&kvm->srcu);
276 277
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
278
	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
279
	spin_unlock(&kvm->mmu_lock);
280
	srcu_read_unlock(&kvm->srcu, idx);
281 282 283 284 285 286 287

	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

}

288 289 290 291 292 293
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
294
	int idx;
295

296
	idx = srcu_read_lock(&kvm->srcu);
297 298 299 300
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	kvm_set_spte_hva(kvm, address, pte);
	spin_unlock(&kvm->mmu_lock);
301
	srcu_read_unlock(&kvm->srcu, idx);
302 303
}

304 305 306 307 308 309
static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
310
	int need_tlb_flush = 0, idx;
311

312
	idx = srcu_read_lock(&kvm->srcu);
313 314 315 316 317 318 319 320 321
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
	for (; start < end; start += PAGE_SIZE)
		need_tlb_flush |= kvm_unmap_hva(kvm, start);
322
	need_tlb_flush |= kvm->tlbs_dirty;
323
	spin_unlock(&kvm->mmu_lock);
324
	srcu_read_unlock(&kvm->srcu, idx);
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361

	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
	/*
	 * The above sequence increase must be visible before the
	 * below count decrease but both values are read by the kvm
	 * page fault under mmu_lock spinlock so we don't need to add
	 * a smb_wmb() here in between the two.
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
					      unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
362
	int young, idx;
363

364
	idx = srcu_read_lock(&kvm->srcu);
365 366 367
	spin_lock(&kvm->mmu_lock);
	young = kvm_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
368
	srcu_read_unlock(&kvm->srcu, idx);
369 370 371 372 373 374 375

	if (young)
		kvm_flush_remote_tlbs(kvm);

	return young;
}

A
Andrea Arcangeli 已提交
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
				       struct mm_struct *mm,
				       unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	young = kvm_test_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

392 393 394 395
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
396 397 398
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
399
	kvm_arch_flush_shadow(kvm);
400
	srcu_read_unlock(&kvm->srcu, idx);
401 402
}

403 404 405 406 407
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
A
Andrea Arcangeli 已提交
408
	.test_young		= kvm_mmu_notifier_test_young,
409
	.change_pte		= kvm_mmu_notifier_change_pte,
410
	.release		= kvm_mmu_notifier_release,
411
};
412 413 414 415 416 417 418 419 420 421 422 423 424 425

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

426 427
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

428
static struct kvm *kvm_create_vm(void)
A
Avi Kivity 已提交
429
{
430 431
	int r, i;
	struct kvm *kvm = kvm_arch_alloc_vm();
A
Avi Kivity 已提交
432

433 434 435 436 437 438
	if (!kvm)
		return ERR_PTR(-ENOMEM);

	r = kvm_arch_init_vm(kvm);
	if (r)
		goto out_err_nodisable;
439 440 441 442 443

	r = hardware_enable_all();
	if (r)
		goto out_err_nodisable;

444 445
#ifdef CONFIG_HAVE_KVM_IRQCHIP
	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
446
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
447
#endif
A
Avi Kivity 已提交
448

449 450 451
	r = -ENOMEM;
	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
	if (!kvm->memslots)
452
		goto out_err_nosrcu;
453
	if (init_srcu_struct(&kvm->srcu))
454
		goto out_err_nosrcu;
M
Marcelo Tosatti 已提交
455 456 457
	for (i = 0; i < KVM_NR_BUSES; i++) {
		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
					GFP_KERNEL);
458
		if (!kvm->buses[i])
M
Marcelo Tosatti 已提交
459 460
			goto out_err;
	}
461

462
	r = kvm_init_mmu_notifier(kvm);
463
	if (r)
464
		goto out_err;
465

466 467
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
468
	spin_lock_init(&kvm->mmu_lock);
G
Gregory Haskins 已提交
469
	kvm_eventfd_init(kvm);
S
Shaohua Li 已提交
470
	mutex_init(&kvm->lock);
471
	mutex_init(&kvm->irq_lock);
472
	mutex_init(&kvm->slots_lock);
I
Izik Eidus 已提交
473
	atomic_set(&kvm->users_count, 1);
474 475 476
	spin_lock(&kvm_lock);
	list_add(&kvm->vm_list, &vm_list);
	spin_unlock(&kvm_lock);
477

478
	return kvm;
479 480

out_err:
481 482
	cleanup_srcu_struct(&kvm->srcu);
out_err_nosrcu:
483 484
	hardware_disable_all();
out_err_nodisable:
M
Marcelo Tosatti 已提交
485 486
	for (i = 0; i < KVM_NR_BUSES; i++)
		kfree(kvm->buses[i]);
487
	kfree(kvm->memslots);
488
	kvm_arch_free_vm(kvm);
489
	return ERR_PTR(r);
490 491
}

492 493 494 495 496
static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
	if (!memslot->dirty_bitmap)
		return;

497 498 499 500 501
	if (2 * kvm_dirty_bitmap_bytes(memslot) > PAGE_SIZE)
		vfree(memslot->dirty_bitmap_head);
	else
		kfree(memslot->dirty_bitmap_head);

502
	memslot->dirty_bitmap = NULL;
503
	memslot->dirty_bitmap_head = NULL;
504 505
}

A
Avi Kivity 已提交
506 507 508 509 510 511
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
512 513
	int i;

514 515
	if (!dont || free->rmap != dont->rmap)
		vfree(free->rmap);
A
Avi Kivity 已提交
516 517

	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
518
		kvm_destroy_dirty_bitmap(free);
A
Avi Kivity 已提交
519

520 521 522 523 524 525 526

	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
		if (!dont || free->lpage_info[i] != dont->lpage_info[i]) {
			vfree(free->lpage_info[i]);
			free->lpage_info[i] = NULL;
		}
	}
M
Marcelo Tosatti 已提交
527

A
Avi Kivity 已提交
528
	free->npages = 0;
529
	free->rmap = NULL;
A
Avi Kivity 已提交
530 531
}

532
void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
533 534
{
	int i;
535 536 537 538
	struct kvm_memslots *slots = kvm->memslots;

	for (i = 0; i < slots->nmemslots; ++i)
		kvm_free_physmem_slot(&slots->memslots[i], NULL);
A
Avi Kivity 已提交
539

540
	kfree(kvm->memslots);
A
Avi Kivity 已提交
541 542
}

543 544
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
545
	int i;
546 547
	struct mm_struct *mm = kvm->mm;

548
	kvm_arch_sync_events(kvm);
549 550 551
	spin_lock(&kvm_lock);
	list_del(&kvm->vm_list);
	spin_unlock(&kvm_lock);
552
	kvm_free_irq_routing(kvm);
M
Marcelo Tosatti 已提交
553 554
	for (i = 0; i < KVM_NR_BUSES; i++)
		kvm_io_bus_destroy(kvm->buses[i]);
555
	kvm_coalesced_mmio_free(kvm);
556 557
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
558 559
#else
	kvm_arch_flush_shadow(kvm);
560
#endif
561
	kvm_arch_destroy_vm(kvm);
562 563 564
	kvm_free_physmem(kvm);
	cleanup_srcu_struct(&kvm->srcu);
	kvm_arch_free_vm(kvm);
565
	hardware_disable_all();
566
	mmdrop(mm);
567 568
}

I
Izik Eidus 已提交
569 570 571 572 573 574 575 576 577 578 579 580 581 582
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


583 584 585 586
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
587 588
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
589
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
590 591 592
	return 0;
}

593
#ifndef CONFIG_S390
594 595 596 597 598
/*
 * Allocation size is twice as large as the actual dirty bitmap size.
 * This makes it possible to do double buffering: see x86's
 * kvm_vm_ioctl_get_dirty_log().
 */
599 600
static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
{
601
	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
602

603 604 605 606 607
	if (dirty_bytes > PAGE_SIZE)
		memslot->dirty_bitmap = vzalloc(dirty_bytes);
	else
		memslot->dirty_bitmap = kzalloc(dirty_bytes, GFP_KERNEL);

608 609 610
	if (!memslot->dirty_bitmap)
		return -ENOMEM;

611
	memslot->dirty_bitmap_head = memslot->dirty_bitmap;
612 613
	return 0;
}
614
#endif /* !CONFIG_S390 */
615

A
Avi Kivity 已提交
616 617 618 619 620
/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
621
 *
622
 * Must be called holding mmap_sem for write.
A
Avi Kivity 已提交
623
 */
624 625 626
int __kvm_set_memory_region(struct kvm *kvm,
			    struct kvm_userspace_memory_region *mem,
			    int user_alloc)
A
Avi Kivity 已提交
627
{
628
	int r;
A
Avi Kivity 已提交
629
	gfn_t base_gfn;
630 631
	unsigned long npages;
	unsigned long i;
A
Avi Kivity 已提交
632 633
	struct kvm_memory_slot *memslot;
	struct kvm_memory_slot old, new;
634
	struct kvm_memslots *slots, *old_memslots;
A
Avi Kivity 已提交
635 636 637 638 639 640 641

	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
S
Sheng Yang 已提交
642
	if (user_alloc && (mem->userspace_addr & (PAGE_SIZE - 1)))
643
		goto out;
644
	if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
A
Avi Kivity 已提交
645 646 647 648
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

649
	memslot = &kvm->memslots->memslots[mem->slot];
A
Avi Kivity 已提交
650 651 652
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

653 654 655 656
	r = -EINVAL;
	if (npages > KVM_MEM_MAX_NR_PAGES)
		goto out;

A
Avi Kivity 已提交
657 658 659 660 661
	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

	new = old = *memslot;

662
	new.id = mem->slot;
A
Avi Kivity 已提交
663 664 665 666 667 668 669
	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	/* Disallow changing a memory slot's size. */
	r = -EINVAL;
	if (npages && old.npages && npages != old.npages)
670
		goto out_free;
A
Avi Kivity 已提交
671 672 673 674

	/* Check for overlaps */
	r = -EEXIST;
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
675
		struct kvm_memory_slot *s = &kvm->memslots->memslots[i];
A
Avi Kivity 已提交
676

677
		if (s == memslot || !s->npages)
A
Avi Kivity 已提交
678 679 680
			continue;
		if (!((base_gfn + npages <= s->base_gfn) ||
		      (base_gfn >= s->base_gfn + s->npages)))
681
			goto out_free;
A
Avi Kivity 已提交
682 683 684 685
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
686
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
687 688 689 690

	r = -ENOMEM;

	/* Allocate if a slot is being created */
691
#ifndef CONFIG_S390
692
	if (npages && !new.rmap) {
693
		new.rmap = vzalloc(npages * sizeof(*new.rmap));
694 695

		if (!new.rmap)
696
			goto out_free;
697

698
		new.user_alloc = user_alloc;
699
		new.userspace_addr = mem->userspace_addr;
A
Avi Kivity 已提交
700
	}
701 702
	if (!npages)
		goto skip_lpage;
M
Marcelo Tosatti 已提交
703

704
	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
705 706 707
		unsigned long ugfn;
		unsigned long j;
		int lpages;
708
		int level = i + 2;
M
Marcelo Tosatti 已提交
709

710 711 712 713 714 715
		/* Avoid unused variable warning if no large pages */
		(void)level;

		if (new.lpage_info[i])
			continue;

716 717 718
		lpages = 1 + ((base_gfn + npages - 1)
			     >> KVM_HPAGE_GFN_SHIFT(level));
		lpages -= base_gfn >> KVM_HPAGE_GFN_SHIFT(level);
719

720
		new.lpage_info[i] = vzalloc(lpages * sizeof(*new.lpage_info[i]));
721 722

		if (!new.lpage_info[i])
M
Marcelo Tosatti 已提交
723 724
			goto out_free;

725
		if (base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
726
			new.lpage_info[i][0].write_count = 1;
727
		if ((base_gfn+npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
728
			new.lpage_info[i][lpages - 1].write_count = 1;
729 730 731
		ugfn = new.userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
732 733
		 * other, or if explicitly asked to, disable large page
		 * support for this slot
734
		 */
735
		if ((base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
736
		    !largepages_enabled)
737 738
			for (j = 0; j < lpages; ++j)
				new.lpage_info[i][j].write_count = 1;
M
Marcelo Tosatti 已提交
739
	}
A
Avi Kivity 已提交
740

741 742
skip_lpage:

A
Avi Kivity 已提交
743 744
	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
745
		if (kvm_create_dirty_bitmap(&new) < 0)
746
			goto out_free;
747
		/* destroy any largepage mappings for dirty tracking */
A
Avi Kivity 已提交
748
	}
749 750 751 752
#else  /* not defined CONFIG_S390 */
	new.user_alloc = user_alloc;
	if (user_alloc)
		new.userspace_addr = mem->userspace_addr;
753
#endif /* not defined CONFIG_S390 */
A
Avi Kivity 已提交
754

755 756 757 758 759 760 761 762
	if (!npages) {
		r = -ENOMEM;
		slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
		if (!slots)
			goto out_free;
		memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
		if (mem->slot >= slots->nmemslots)
			slots->nmemslots = mem->slot + 1;
763
		slots->generation++;
764 765 766 767 768 769 770 771 772 773 774 775
		slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID;

		old_memslots = kvm->memslots;
		rcu_assign_pointer(kvm->memslots, slots);
		synchronize_srcu_expedited(&kvm->srcu);
		/* From this point no new shadow pages pointing to a deleted
		 * memslot will be created.
		 *
		 * validation of sp->gfn happens in:
		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
		 * 	- kvm_is_visible_gfn (mmu_check_roots)
		 */
776
		kvm_arch_flush_shadow(kvm);
777 778
		kfree(old_memslots);
	}
779

780 781 782 783
	r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
	if (r)
		goto out_free;

784 785 786 787 788 789
	/* map the pages in iommu page table */
	if (npages) {
		r = kvm_iommu_map_pages(kvm, &new);
		if (r)
			goto out_free;
	}
790

791 792 793 794 795 796 797
	r = -ENOMEM;
	slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
	if (!slots)
		goto out_free;
	memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
	if (mem->slot >= slots->nmemslots)
		slots->nmemslots = mem->slot + 1;
798
	slots->generation++;
799 800 801 802 803 804 805 806 807 808 809 810 811

	/* actual memory is freed via old in kvm_free_physmem_slot below */
	if (!npages) {
		new.rmap = NULL;
		new.dirty_bitmap = NULL;
		for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i)
			new.lpage_info[i] = NULL;
	}

	slots->memslots[mem->slot] = new;
	old_memslots = kvm->memslots;
	rcu_assign_pointer(kvm->memslots, slots);
	synchronize_srcu_expedited(&kvm->srcu);
812

813
	kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
814

815 816 817
	kvm_free_physmem_slot(&old, &new);
	kfree(old_memslots);

A
Avi Kivity 已提交
818 819
	return 0;

820
out_free:
A
Avi Kivity 已提交
821 822 823
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
824 825

}
826 827 828 829 830 831 832 833
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
			  struct kvm_userspace_memory_region *mem,
			  int user_alloc)
{
	int r;

834
	mutex_lock(&kvm->slots_lock);
835
	r = __kvm_set_memory_region(kvm, mem, user_alloc);
836
	mutex_unlock(&kvm->slots_lock);
837 838
	return r;
}
839 840
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

841 842 843 844
int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
				   struct
				   kvm_userspace_memory_region *mem,
				   int user_alloc)
845
{
846 847
	if (mem->slot >= KVM_MEMORY_SLOTS)
		return -EINVAL;
848
	return kvm_set_memory_region(kvm, mem, user_alloc);
A
Avi Kivity 已提交
849 850
}

851 852
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
853 854 855
{
	struct kvm_memory_slot *memslot;
	int r, i;
856
	unsigned long n;
A
Avi Kivity 已提交
857 858 859 860 861 862
	unsigned long any = 0;

	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

863
	memslot = &kvm->memslots->memslots[log->slot];
A
Avi Kivity 已提交
864 865 866 867
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

868
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
869

870
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
871 872 873 874 875 876
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

877 878
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
879 880 881 882 883 884

	r = 0;
out:
	return r;
}

885 886 887 888 889 890
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

891 892
int is_error_page(struct page *page)
{
893
	return page == bad_page || page == hwpoison_page || page == fault_page;
894 895 896
}
EXPORT_SYMBOL_GPL(is_error_page);

897 898
int is_error_pfn(pfn_t pfn)
{
899
	return pfn == bad_pfn || pfn == hwpoison_pfn || pfn == fault_pfn;
900 901 902
}
EXPORT_SYMBOL_GPL(is_error_pfn);

903 904 905 906 907 908
int is_hwpoison_pfn(pfn_t pfn)
{
	return pfn == hwpoison_pfn;
}
EXPORT_SYMBOL_GPL(is_hwpoison_pfn);

909 910 911 912 913 914
int is_fault_pfn(pfn_t pfn)
{
	return pfn == fault_pfn;
}
EXPORT_SYMBOL_GPL(is_fault_pfn);

I
Izik Eidus 已提交
915 916 917 918 919 920 921 922 923 924 925
static inline unsigned long bad_hva(void)
{
	return PAGE_OFFSET;
}

int kvm_is_error_hva(unsigned long addr)
{
	return addr == bad_hva();
}
EXPORT_SYMBOL_GPL(kvm_is_error_hva);

926 927
static struct kvm_memory_slot *__gfn_to_memslot(struct kvm_memslots *slots,
						gfn_t gfn)
A
Avi Kivity 已提交
928 929 930
{
	int i;

931 932
	for (i = 0; i < slots->nmemslots; ++i) {
		struct kvm_memory_slot *memslot = &slots->memslots[i];
A
Avi Kivity 已提交
933 934 935 936 937

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return memslot;
	}
A
Al Viro 已提交
938
	return NULL;
A
Avi Kivity 已提交
939
}
940 941 942 943 944

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
A
Avi Kivity 已提交
945
EXPORT_SYMBOL_GPL(gfn_to_memslot);
A
Avi Kivity 已提交
946

947 948 949
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
	int i;
950
	struct kvm_memslots *slots = kvm_memslots(kvm);
951 952

	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
953
		struct kvm_memory_slot *memslot = &slots->memslots[i];
954

955 956 957
		if (memslot->flags & KVM_MEMSLOT_INVALID)
			continue;

958 959 960 961 962 963 964 965
		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

J
Joerg Roedel 已提交
966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

990 991 992
int memslot_id(struct kvm *kvm, gfn_t gfn)
{
	int i;
993
	struct kvm_memslots *slots = kvm_memslots(kvm);
994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006
	struct kvm_memory_slot *memslot = NULL;

	for (i = 0; i < slots->nmemslots; ++i) {
		memslot = &slots->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			break;
	}

	return memslot - slots->memslots;
}

1007
static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1008
				     gfn_t *nr_pages)
I
Izik Eidus 已提交
1009
{
1010
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
I
Izik Eidus 已提交
1011
		return bad_hva();
1012 1013 1014 1015

	if (nr_pages)
		*nr_pages = slot->npages - (gfn - slot->base_gfn);

1016
	return gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
1017
}
1018 1019 1020

unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
1021
	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1022
}
1023
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1024

1025 1026 1027 1028 1029 1030
static pfn_t get_fault_pfn(void)
{
	get_page(fault_page);
	return fault_pfn;
}

1031 1032 1033 1034 1035 1036 1037 1038 1039
static inline int check_user_page_hwpoison(unsigned long addr)
{
	int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;

	rc = __get_user_pages(current, current->mm, addr, 1,
			      flags, NULL, NULL, NULL);
	return rc == -EHWPOISON;
}

1040
static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr, bool atomic,
1041
			bool *async, bool write_fault, bool *writable)
A
Avi Kivity 已提交
1042
{
1043
	struct page *page[1];
1044
	int npages = 0;
1045
	pfn_t pfn;
A
Avi Kivity 已提交
1046

1047 1048 1049
	/* we can do it either atomically or asynchronously, not both */
	BUG_ON(atomic && async);

1050 1051 1052 1053 1054
	BUG_ON(!write_fault && !writable);

	if (writable)
		*writable = true;

1055
	if (atomic || async)
1056
		npages = __get_user_pages_fast(addr, 1, 1, page);
1057 1058

	if (unlikely(npages != 1) && !atomic) {
1059
		might_sleep();
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077

		if (writable)
			*writable = write_fault;

		npages = get_user_pages_fast(addr, 1, write_fault, page);

		/* map read fault as writable if possible */
		if (unlikely(!write_fault) && npages == 1) {
			struct page *wpage[1];

			npages = __get_user_pages_fast(addr, 1, 1, wpage);
			if (npages == 1) {
				*writable = true;
				put_page(page[0]);
				page[0] = wpage[0];
			}
			npages = 1;
		}
1078
	}
I
Izik Eidus 已提交
1079

1080 1081 1082
	if (unlikely(npages != 1)) {
		struct vm_area_struct *vma;

1083
		if (atomic)
1084
			return get_fault_pfn();
1085

1086
		down_read(&current->mm->mmap_sem);
1087
		if (check_user_page_hwpoison(addr)) {
1088
			up_read(&current->mm->mmap_sem);
1089 1090 1091 1092
			get_page(hwpoison_page);
			return page_to_pfn(hwpoison_page);
		}

1093
		vma = find_vma_intersection(current->mm, addr, addr+1);
1094

1095 1096 1097 1098 1099 1100 1101 1102
		if (vma == NULL)
			pfn = get_fault_pfn();
		else if ((vma->vm_flags & VM_PFNMAP)) {
			pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
				vma->vm_pgoff;
			BUG_ON(!kvm_is_mmio_pfn(pfn));
		} else {
			if (async && (vma->vm_flags & VM_WRITE))
1103
				*async = true;
1104
			pfn = get_fault_pfn();
1105
		}
1106
		up_read(&current->mm->mmap_sem);
1107 1108
	} else
		pfn = page_to_pfn(page[0]);
1109

1110
	return pfn;
1111 1112
}

1113 1114
pfn_t hva_to_pfn_atomic(struct kvm *kvm, unsigned long addr)
{
1115
	return hva_to_pfn(kvm, addr, true, NULL, true, NULL);
1116 1117 1118
}
EXPORT_SYMBOL_GPL(hva_to_pfn_atomic);

1119 1120
static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
			  bool write_fault, bool *writable)
1121 1122 1123
{
	unsigned long addr;

1124 1125 1126
	if (async)
		*async = false;

1127 1128 1129 1130 1131 1132
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr)) {
		get_page(bad_page);
		return page_to_pfn(bad_page);
	}

1133
	return hva_to_pfn(kvm, addr, atomic, async, write_fault, writable);
1134 1135 1136 1137
}

pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
{
1138
	return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1139 1140 1141
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);

1142 1143
pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
		       bool write_fault, bool *writable)
1144
{
1145
	return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1146 1147 1148
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_async);

1149 1150
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
{
1151
	return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1152
}
1153 1154
EXPORT_SYMBOL_GPL(gfn_to_pfn);

1155 1156 1157 1158 1159 1160 1161
pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
		      bool *writable)
{
	return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);

1162 1163 1164 1165
pfn_t gfn_to_pfn_memslot(struct kvm *kvm,
			 struct kvm_memory_slot *slot, gfn_t gfn)
{
	unsigned long addr = gfn_to_hva_memslot(slot, gfn);
1166
	return hva_to_pfn(kvm, addr, false, NULL, true, NULL);
1167 1168
}

1169 1170 1171 1172 1173 1174
int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
								  int nr_pages)
{
	unsigned long addr;
	gfn_t entry;

1175
	addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
	if (kvm_is_error_hva(addr))
		return -1;

	if (entry < nr_pages)
		return 0;

	return __get_user_pages_fast(addr, nr_pages, 1, pages);
}
EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);

1186 1187
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
1188 1189 1190
	pfn_t pfn;

	pfn = gfn_to_pfn(kvm, gfn);
1191
	if (!kvm_is_mmio_pfn(pfn))
1192 1193
		return pfn_to_page(pfn);

1194
	WARN_ON(kvm_is_mmio_pfn(pfn));
1195 1196 1197

	get_page(bad_page);
	return bad_page;
A
Avi Kivity 已提交
1198
}
1199

A
Avi Kivity 已提交
1200 1201
EXPORT_SYMBOL_GPL(gfn_to_page);

1202 1203
void kvm_release_page_clean(struct page *page)
{
1204
	kvm_release_pfn_clean(page_to_pfn(page));
1205 1206 1207
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

1208 1209
void kvm_release_pfn_clean(pfn_t pfn)
{
1210
	if (!kvm_is_mmio_pfn(pfn))
1211
		put_page(pfn_to_page(pfn));
1212 1213 1214
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1215
void kvm_release_page_dirty(struct page *page)
1216
{
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

void kvm_release_pfn_dirty(pfn_t pfn)
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);

void kvm_set_page_dirty(struct page *page)
{
	kvm_set_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_set_page_dirty);

void kvm_set_pfn_dirty(pfn_t pfn)
{
1236
	if (!kvm_is_mmio_pfn(pfn)) {
1237 1238 1239 1240
		struct page *page = pfn_to_page(pfn);
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1241
}
1242 1243 1244 1245
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
1246
	if (!kvm_is_mmio_pfn(pfn))
1247
		mark_page_accessed(pfn_to_page(pfn));
1248 1249 1250 1251 1252
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
1253
	if (!kvm_is_mmio_pfn(pfn))
1254
		get_page(pfn_to_page(pfn));
1255 1256
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1257

1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
1269 1270
	int r;
	unsigned long addr;
1271

1272 1273 1274 1275 1276
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_from_user(data, (void __user *)addr + offset, len);
	if (r)
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1313
	pagefault_disable();
1314
	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1315
	pagefault_enable();
1316 1317 1318 1319 1320 1321
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

1322 1323 1324
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
1325 1326
	int r;
	unsigned long addr;
1327

1328 1329 1330 1331 1332
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_to_user((void __user *)addr + offset, data, len);
	if (r)
1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			      gpa_t gpa)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int offset = offset_in_page(gpa);
	gfn_t gfn = gpa >> PAGE_SHIFT;

	ghc->gpa = gpa;
	ghc->generation = slots->generation;
	ghc->memslot = __gfn_to_memslot(slots, gfn);
	ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
	if (!kvm_is_error_hva(ghc->hva))
		ghc->hva += offset;
	else
		return -EFAULT;

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);

int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

	if (slots->generation != ghc->generation)
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

	r = copy_to_user((void __user *)ghc->hva, data, len);
	if (r)
		return -EFAULT;
	mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);

1400 1401
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
1402 1403
	return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
				    offset, len);
1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

1426 1427
void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
			     gfn_t gfn)
A
Avi Kivity 已提交
1428
{
R
Rusty Russell 已提交
1429 1430
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1431

1432
		generic___set_le_bit(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1433 1434 1435
	}
}

1436 1437 1438 1439 1440 1441 1442 1443
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = gfn_to_memslot(kvm, gfn);
	mark_page_dirty_in_slot(kvm, memslot, gfn);
}

E
Eddie Dong 已提交
1444 1445 1446
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1447
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1448
{
1449 1450 1451 1452 1453
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

1454
		if (kvm_arch_vcpu_runnable(vcpu)) {
1455
			kvm_make_request(KVM_REQ_UNHALT, vcpu);
1456
			break;
1457
		}
1458 1459
		if (kvm_cpu_has_pending_timer(vcpu))
			break;
1460 1461 1462
		if (signal_pending(current))
			break;

E
Eddie Dong 已提交
1463 1464
		schedule();
	}
1465

1466
	finish_wait(&vcpu->wq, &wait);
E
Eddie Dong 已提交
1467 1468
}

A
Avi Kivity 已提交
1469 1470
void kvm_resched(struct kvm_vcpu *vcpu)
{
1471 1472
	if (!need_resched())
		return;
A
Avi Kivity 已提交
1473 1474 1475 1476
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

Z
Zhai, Edwin 已提交
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu)
{
	ktime_t expires;
	DEFINE_WAIT(wait);

	prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

	/* Sleep for 100 us, and hope lock-holder got scheduled */
	expires = ktime_add_ns(ktime_get(), 100000UL);
	schedule_hrtimeout(&expires, HRTIMER_MODE_ABS);

	finish_wait(&vcpu->wq, &wait);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

1492
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1493 1494 1495 1496
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

1497
	if (vmf->pgoff == 0)
1498
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
1499
#ifdef CONFIG_X86
1500
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1501
		page = virt_to_page(vcpu->arch.pio_data);
1502 1503 1504 1505
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
1506
#endif
1507
	else
1508
		return VM_FAULT_SIGBUS;
1509
	get_page(page);
1510 1511
	vmf->page = page;
	return 0;
1512 1513
}

1514
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1515
	.fault = kvm_vcpu_fault,
1516 1517 1518 1519 1520 1521 1522 1523
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
1524 1525 1526 1527
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
1528
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1529 1530 1531
	return 0;
}

1532
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
1533 1534 1535
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
	.compat_ioctl   = kvm_vcpu_ioctl,
1536
	.mmap           = kvm_vcpu_mmap,
1537
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
1538 1539 1540 1541 1542 1543 1544
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
1545
	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
A
Avi Kivity 已提交
1546 1547
}

1548 1549 1550
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
1551
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1552 1553
{
	int r;
1554
	struct kvm_vcpu *vcpu, *v;
1555

1556
	vcpu = kvm_arch_vcpu_create(kvm, id);
R
Rusty Russell 已提交
1557 1558
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
1559

1560 1561
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

1562 1563
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
1564
		return r;
1565

S
Shaohua Li 已提交
1566
	mutex_lock(&kvm->lock);
1567 1568
	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
		r = -EINVAL;
1569
		goto vcpu_destroy;
R
Rusty Russell 已提交
1570
	}
1571

1572 1573
	kvm_for_each_vcpu(r, v, kvm)
		if (v->vcpu_id == id) {
1574 1575 1576 1577 1578
			r = -EEXIST;
			goto vcpu_destroy;
		}

	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1579

R
Rusty Russell 已提交
1580
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
1581
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
1582
	r = create_vcpu_fd(vcpu);
1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596
	if (r < 0) {
		kvm_put_kvm(kvm);
		goto vcpu_destroy;
	}

	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	if (kvm->bsp_vcpu_id == id)
		kvm->bsp_vcpu = vcpu;
#endif
	mutex_unlock(&kvm->lock);
R
Rusty Russell 已提交
1597
	return r;
1598

1599
vcpu_destroy:
1600
	mutex_unlock(&kvm->lock);
1601
	kvm_arch_vcpu_destroy(vcpu);
1602 1603 1604
	return r;
}

A
Avi Kivity 已提交
1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
1616 1617
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
1618
{
A
Avi Kivity 已提交
1619
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
1620
	void __user *argp = (void __user *)arg;
1621
	int r;
1622 1623
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
1624

1625 1626
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638

#if defined(CONFIG_S390) || defined(CONFIG_PPC)
	/*
	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
	 * so vcpu_load() would break it.
	 */
	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
#endif


	vcpu_load(vcpu);
A
Avi Kivity 已提交
1639
	switch (ioctl) {
1640
	case KVM_RUN:
1641 1642 1643
		r = -EINVAL;
		if (arg)
			goto out;
1644
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1645
		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
A
Avi Kivity 已提交
1646 1647
		break;
	case KVM_GET_REGS: {
1648
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1649

1650 1651 1652
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1653
			goto out;
1654 1655 1656
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
1657
		r = -EFAULT;
1658 1659
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
1660
		r = 0;
1661 1662
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
1663 1664 1665
		break;
	}
	case KVM_SET_REGS: {
1666
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1667

1668 1669 1670
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1671
			goto out;
1672 1673 1674 1675
		r = -EFAULT;
		if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
			goto out_free2;
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
A
Avi Kivity 已提交
1676
		if (r)
1677
			goto out_free2;
A
Avi Kivity 已提交
1678
		r = 0;
1679 1680
out_free2:
		kfree(kvm_regs);
A
Avi Kivity 已提交
1681 1682 1683
		break;
	}
	case KVM_GET_SREGS: {
1684 1685 1686 1687 1688
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
1689 1690 1691
		if (r)
			goto out;
		r = -EFAULT;
1692
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
1693 1694 1695 1696 1697
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
1698 1699 1700 1701
		kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
A
Avi Kivity 已提交
1702
		r = -EFAULT;
1703
		if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
1704
			goto out;
1705
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
1706 1707 1708 1709 1710
		if (r)
			goto out;
		r = 0;
		break;
	}
1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1735 1736 1737 1738
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
1739
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
1740
			goto out;
1741
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
1742 1743 1744
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
1745
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
1746 1747 1748 1749
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
1750 1751
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
1752 1753

		r = -EFAULT;
A
Al Viro 已提交
1754
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
1755
			goto out;
J
Jan Kiszka 已提交
1756
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
1757 1758 1759 1760 1761
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
1782
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
A
Avi Kivity 已提交
1783 1784
		break;
	}
A
Avi Kivity 已提交
1785
	case KVM_GET_FPU: {
1786 1787 1788 1789 1790
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
1791 1792 1793
		if (r)
			goto out;
		r = -EFAULT;
1794
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
1795 1796 1797 1798 1799
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
1800 1801 1802 1803
		fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
A
Avi Kivity 已提交
1804
		r = -EFAULT;
1805
		if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
1806
			goto out;
1807
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
1808 1809 1810 1811 1812
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1813
	default:
1814
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1815 1816
	}
out:
1817
	vcpu_put(vcpu);
1818 1819
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
1820 1821 1822 1823 1824 1825 1826 1827
	return r;
}

static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
1828
	int r;
A
Avi Kivity 已提交
1829

1830 1831
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
1832 1833 1834 1835 1836 1837
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		if (r < 0)
			goto out;
		break;
1838 1839 1840 1841 1842 1843 1844 1845 1846
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
A
Avi Kivity 已提交
1847 1848 1849 1850 1851 1852 1853 1854
		if (r)
			goto out;
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
1855
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
1856
			goto out;
1857
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
1858 1859 1860 1861
		if (r)
			goto out;
		break;
	}
1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		if (r)
			goto out;
		r = 0;
		break;
	}
#endif
G
Gregory Haskins 已提交
1886 1887 1888 1889 1890 1891 1892 1893 1894
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
		r = kvm_irqfd(kvm, data.fd, data.gsi, data.flags);
		break;
	}
G
Gregory Haskins 已提交
1895 1896 1897 1898 1899 1900 1901 1902 1903
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
1904 1905 1906
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
1907
		mutex_lock(&kvm->lock);
1908 1909 1910 1911
		if (atomic_read(&kvm->online_vcpus) != 0)
			r = -EBUSY;
		else
			kvm->bsp_vcpu_id = arg;
1912
		mutex_unlock(&kvm->lock);
1913 1914
		break;
#endif
1915
	default:
1916
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1917 1918
		if (r == -ENOTTY)
			r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
1919 1920 1921 1922 1923
	}
out:
	return r;
}

1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
#ifdef CONFIG_COMPAT
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		r = -EFAULT;
		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
			goto out;
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		if (r)
			goto out;
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

1970
static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1971
{
1972 1973 1974 1975
	struct page *page[1];
	unsigned long addr;
	int npages;
	gfn_t gfn = vmf->pgoff;
1976 1977
	struct kvm *kvm = vma->vm_file->private_data;

1978 1979
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
1980
		return VM_FAULT_SIGBUS;
1981 1982 1983 1984

	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
				NULL);
	if (unlikely(npages != 1))
1985
		return VM_FAULT_SIGBUS;
1986 1987

	vmf->page = page[0];
1988
	return 0;
1989 1990
}

1991
static const struct vm_operations_struct kvm_vm_vm_ops = {
1992
	.fault = kvm_vm_fault,
1993 1994 1995 1996 1997 1998 1999 2000
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

2001
static struct file_operations kvm_vm_fops = {
2002 2003
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
2004 2005 2006
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vm_compat_ioctl,
#endif
2007
	.mmap           = kvm_vm_mmap,
2008
	.llseek		= noop_llseek,
2009 2010 2011 2012
};

static int kvm_dev_ioctl_create_vm(void)
{
2013
	int r;
2014 2015 2016
	struct kvm *kvm;

	kvm = kvm_create_vm();
2017 2018
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
2019 2020 2021 2022 2023 2024 2025
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	r = kvm_coalesced_mmio_init(kvm);
	if (r < 0) {
		kvm_put_kvm(kvm);
		return r;
	}
#endif
2026 2027
	r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
	if (r < 0)
A
Al Viro 已提交
2028
		kvm_put_kvm(kvm);
2029

2030
	return r;
2031 2032
}

2033 2034 2035
static long kvm_dev_ioctl_check_extension_generic(long arg)
{
	switch (arg) {
2036
	case KVM_CAP_USER_MEMORY:
2037
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2038
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2039 2040 2041
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_CAP_SET_BOOT_CPU_ID:
#endif
2042
	case KVM_CAP_INTERNAL_ERROR_DATA:
2043
		return 1;
2044 2045
#ifdef CONFIG_HAVE_KVM_IRQCHIP
	case KVM_CAP_IRQ_ROUTING:
2046
		return KVM_MAX_IRQ_ROUTES;
2047
#endif
2048 2049 2050 2051 2052 2053
	default:
		break;
	}
	return kvm_dev_ioctl_check_extension(arg);
}

2054 2055 2056
static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
2057
	long r = -EINVAL;
2058 2059 2060

	switch (ioctl) {
	case KVM_GET_API_VERSION:
2061 2062 2063
		r = -EINVAL;
		if (arg)
			goto out;
2064 2065 2066
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
2067 2068 2069
		r = -EINVAL;
		if (arg)
			goto out;
2070 2071
		r = kvm_dev_ioctl_create_vm();
		break;
2072
	case KVM_CHECK_EXTENSION:
2073
		r = kvm_dev_ioctl_check_extension_generic(arg);
2074
		break;
2075 2076 2077 2078
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
2079 2080 2081
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
2082 2083 2084
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
		r += PAGE_SIZE;    /* coalesced mmio ring page */
2085
#endif
2086
		break;
2087 2088 2089
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
2090
		r = -EOPNOTSUPP;
2091
		break;
A
Avi Kivity 已提交
2092
	default:
2093
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2094 2095 2096 2097 2098 2099 2100 2101
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
2102
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
2103 2104 2105
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
2106
	KVM_MINOR,
A
Avi Kivity 已提交
2107 2108 2109 2110
	"kvm",
	&kvm_chardev_ops,
};

2111
static void hardware_enable_nolock(void *junk)
2112 2113
{
	int cpu = raw_smp_processor_id();
2114
	int r;
2115

2116
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2117
		return;
2118

2119
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
2120 2121 2122 2123 2124 2125 2126 2127 2128

	r = kvm_arch_hardware_enable(NULL);

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
		printk(KERN_INFO "kvm: enabling virtualization on "
				 "CPU%d failed\n", cpu);
	}
2129 2130
}

2131 2132 2133 2134 2135 2136 2137 2138
static void hardware_enable(void *junk)
{
	spin_lock(&kvm_lock);
	hardware_enable_nolock(junk);
	spin_unlock(&kvm_lock);
}

static void hardware_disable_nolock(void *junk)
2139 2140 2141
{
	int cpu = raw_smp_processor_id();

2142
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2143
		return;
2144
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2145
	kvm_arch_hardware_disable(NULL);
2146 2147
}

2148 2149 2150 2151 2152 2153 2154
static void hardware_disable(void *junk)
{
	spin_lock(&kvm_lock);
	hardware_disable_nolock(junk);
	spin_unlock(&kvm_lock);
}

2155 2156 2157 2158 2159 2160
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
2161
		on_each_cpu(hardware_disable_nolock, NULL, 1);
2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
}

static void hardware_disable_all(void)
{
	spin_lock(&kvm_lock);
	hardware_disable_all_nolock();
	spin_unlock(&kvm_lock);
}

static int hardware_enable_all(void)
{
	int r = 0;

	spin_lock(&kvm_lock);

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
2180
		on_each_cpu(hardware_enable_nolock, NULL, 1);
2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

	spin_unlock(&kvm_lock);

	return r;
}

A
Avi Kivity 已提交
2193 2194 2195 2196 2197
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

2198 2199 2200
	if (!kvm_usage_count)
		return NOTIFY_OK;

2201
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
2202
	switch (val) {
2203
	case CPU_DYING:
2204 2205 2206 2207
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
2208
	case CPU_STARTING:
2209 2210
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
2211
		hardware_enable(NULL);
A
Avi Kivity 已提交
2212 2213 2214 2215 2216
		break;
	}
	return NOTIFY_OK;
}

2217

2218
asmlinkage void kvm_spurious_fault(void)
2219 2220 2221 2222
{
	/* Fault while not rebooting.  We want the trace. */
	BUG();
}
2223
EXPORT_SYMBOL_GPL(kvm_spurious_fault);
2224

2225
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
2226
		      void *v)
2227
{
2228 2229 2230 2231 2232 2233 2234 2235
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
	kvm_rebooting = true;
2236
	on_each_cpu(hardware_disable_nolock, NULL, 1);
2237 2238 2239 2240 2241 2242 2243 2244
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
2245
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2246 2247 2248 2249 2250 2251 2252 2253
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
2254
	kfree(bus);
2255 2256
}

2257
/* kvm_io_bus_write - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2258
int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2259
		     int len, const void *val)
2260 2261
{
	int i;
2262 2263 2264
	struct kvm_io_bus *bus;

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2265 2266 2267 2268 2269
	for (i = 0; i < bus->dev_count; i++)
		if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
			return 0;
	return -EOPNOTSUPP;
}
2270

2271
/* kvm_io_bus_read - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2272 2273
int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
		    int len, void *val)
2274 2275
{
	int i;
2276
	struct kvm_io_bus *bus;
M
Marcelo Tosatti 已提交
2277

2278
	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2279 2280 2281 2282
	for (i = 0; i < bus->dev_count; i++)
		if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
			return 0;
	return -EOPNOTSUPP;
2283 2284
}

2285
/* Caller must hold slots_lock. */
M
Marcelo Tosatti 已提交
2286 2287
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			    struct kvm_io_device *dev)
2288
{
M
Marcelo Tosatti 已提交
2289
	struct kvm_io_bus *new_bus, *bus;
2290

M
Marcelo Tosatti 已提交
2291
	bus = kvm->buses[bus_idx];
2292 2293
	if (bus->dev_count > NR_IOBUS_DEVS-1)
		return -ENOSPC;
2294

M
Marcelo Tosatti 已提交
2295 2296 2297 2298 2299 2300 2301 2302
	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
	if (!new_bus)
		return -ENOMEM;
	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
	new_bus->devs[new_bus->dev_count++] = dev;
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
2303 2304 2305 2306

	return 0;
}

2307
/* Caller must hold slots_lock. */
M
Marcelo Tosatti 已提交
2308 2309
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			      struct kvm_io_device *dev)
2310
{
M
Marcelo Tosatti 已提交
2311 2312
	int i, r;
	struct kvm_io_bus *new_bus, *bus;
2313

M
Marcelo Tosatti 已提交
2314 2315 2316
	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
	if (!new_bus)
		return -ENOMEM;
2317

M
Marcelo Tosatti 已提交
2318 2319 2320 2321 2322 2323 2324 2325
	bus = kvm->buses[bus_idx];
	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));

	r = -ENOENT;
	for (i = 0; i < new_bus->dev_count; i++)
		if (new_bus->devs[i] == dev) {
			r = 0;
			new_bus->devs[i] = new_bus->devs[--new_bus->dev_count];
2326 2327
			break;
		}
M
Marcelo Tosatti 已提交
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337

	if (r) {
		kfree(new_bus);
		return r;
	}

	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
	return r;
2338 2339
}

A
Avi Kivity 已提交
2340 2341 2342 2343
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
};

2344
static int vm_stat_get(void *_offset, u64 *val)
2345 2346 2347 2348
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

2349
	*val = 0;
2350 2351
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
2352
		*val += *(u32 *)((void *)kvm + offset);
2353
	spin_unlock(&kvm_lock);
2354
	return 0;
2355 2356 2357 2358
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

2359
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
2360 2361 2362 2363 2364 2365
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

2366
	*val = 0;
A
Avi Kivity 已提交
2367 2368
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
2369 2370 2371
		kvm_for_each_vcpu(i, vcpu, kvm)
			*val += *(u32 *)((void *)vcpu + offset);

A
Avi Kivity 已提交
2372
	spin_unlock(&kvm_lock);
2373
	return 0;
A
Avi Kivity 已提交
2374 2375
}

2376 2377
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

2378
static const struct file_operations *stat_fops[] = {
2379 2380 2381
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
2382

2383
static void kvm_init_debug(void)
A
Avi Kivity 已提交
2384 2385 2386
{
	struct kvm_stats_debugfs_item *p;

2387
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
A
Avi Kivity 已提交
2388
	for (p = debugfs_entries; p->name; ++p)
2389
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
2390
						(void *)(long)p->offset,
2391
						stat_fops[p->kind]);
A
Avi Kivity 已提交
2392 2393 2394 2395 2396 2397 2398 2399
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
2400
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
2401 2402
}

2403 2404
static int kvm_suspend(struct sys_device *dev, pm_message_t state)
{
2405
	if (kvm_usage_count)
2406
		hardware_disable_nolock(NULL);
2407 2408 2409 2410 2411
	return 0;
}

static int kvm_resume(struct sys_device *dev)
{
2412 2413
	if (kvm_usage_count) {
		WARN_ON(spin_is_locked(&kvm_lock));
2414
		hardware_enable_nolock(NULL);
2415
	}
2416 2417 2418 2419
	return 0;
}

static struct sysdev_class kvm_sysdev_class = {
2420
	.name = "kvm",
2421 2422 2423 2424 2425 2426 2427 2428 2429
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

static struct sys_device kvm_sysdev = {
	.id = 0,
	.cls = &kvm_sysdev_class,
};

2430
struct page *bad_page;
2431
pfn_t bad_pfn;
A
Avi Kivity 已提交
2432

2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2443
	kvm_arch_vcpu_load(vcpu, cpu);
2444 2445 2446 2447 2448 2449 2450
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2451
	kvm_arch_vcpu_put(vcpu);
2452 2453
}

2454
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
2455
		  struct module *module)
A
Avi Kivity 已提交
2456 2457
{
	int r;
Y
Yang, Sheng 已提交
2458
	int cpu;
A
Avi Kivity 已提交
2459

2460 2461
	r = kvm_arch_init(opaque);
	if (r)
2462
		goto out_fail;
2463 2464 2465 2466 2467 2468 2469 2470

	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

	if (bad_page == NULL) {
		r = -ENOMEM;
		goto out;
	}

2471 2472
	bad_pfn = page_to_pfn(bad_page);

2473 2474 2475 2476 2477 2478 2479 2480 2481
	hwpoison_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

	if (hwpoison_page == NULL) {
		r = -ENOMEM;
		goto out_free_0;
	}

	hwpoison_pfn = page_to_pfn(hwpoison_page);

2482 2483 2484 2485 2486 2487 2488 2489 2490
	fault_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

	if (fault_page == NULL) {
		r = -ENOMEM;
		goto out_free_0;
	}

	fault_pfn = page_to_pfn(fault_page);

2491
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2492 2493 2494 2495
		r = -ENOMEM;
		goto out_free_0;
	}

2496
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
2497
	if (r < 0)
2498
		goto out_free_0a;
A
Avi Kivity 已提交
2499

Y
Yang, Sheng 已提交
2500 2501
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
2502
				kvm_arch_check_processor_compat,
2503
				&r, 1);
Y
Yang, Sheng 已提交
2504
		if (r < 0)
2505
			goto out_free_1;
Y
Yang, Sheng 已提交
2506 2507
	}

A
Avi Kivity 已提交
2508 2509
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
2510
		goto out_free_2;
A
Avi Kivity 已提交
2511 2512
	register_reboot_notifier(&kvm_reboot_notifier);

2513 2514
	r = sysdev_class_register(&kvm_sysdev_class);
	if (r)
2515
		goto out_free_3;
2516 2517 2518

	r = sysdev_register(&kvm_sysdev);
	if (r)
2519
		goto out_free_4;
2520

2521
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
2522 2523 2524
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
J
Joe Perches 已提交
2525
					   0, NULL);
2526 2527
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
2528
		goto out_free_5;
2529 2530
	}

2531 2532 2533 2534
	r = kvm_async_pf_init();
	if (r)
		goto out_free;

A
Avi Kivity 已提交
2535
	kvm_chardev_ops.owner = module;
2536 2537
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
2538 2539 2540

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
2541
		printk(KERN_ERR "kvm: misc device register failed\n");
2542
		goto out_unreg;
A
Avi Kivity 已提交
2543 2544
	}

2545 2546 2547
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

2548 2549
	kvm_init_debug();

2550
	return 0;
A
Avi Kivity 已提交
2551

2552 2553
out_unreg:
	kvm_async_pf_deinit();
A
Avi Kivity 已提交
2554
out_free:
2555
	kmem_cache_destroy(kvm_vcpu_cache);
2556
out_free_5:
2557
	sysdev_unregister(&kvm_sysdev);
2558
out_free_4:
2559
	sysdev_class_unregister(&kvm_sysdev_class);
2560
out_free_3:
A
Avi Kivity 已提交
2561
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
2562
	unregister_cpu_notifier(&kvm_cpu_notifier);
2563 2564
out_free_2:
out_free_1:
2565
	kvm_arch_hardware_unsetup();
2566 2567
out_free_0a:
	free_cpumask_var(cpus_hardware_enabled);
2568
out_free_0:
2569 2570
	if (fault_page)
		__free_page(fault_page);
2571 2572
	if (hwpoison_page)
		__free_page(hwpoison_page);
2573
	__free_page(bad_page);
2574
out:
2575
	kvm_arch_exit();
2576
out_fail:
A
Avi Kivity 已提交
2577 2578
	return r;
}
2579
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
2580

2581
void kvm_exit(void)
A
Avi Kivity 已提交
2582
{
2583
	kvm_exit_debug();
A
Avi Kivity 已提交
2584
	misc_deregister(&kvm_dev);
2585
	kmem_cache_destroy(kvm_vcpu_cache);
2586
	kvm_async_pf_deinit();
2587 2588
	sysdev_unregister(&kvm_sysdev);
	sysdev_class_unregister(&kvm_sysdev_class);
A
Avi Kivity 已提交
2589
	unregister_reboot_notifier(&kvm_reboot_notifier);
2590
	unregister_cpu_notifier(&kvm_cpu_notifier);
2591
	on_each_cpu(hardware_disable_nolock, NULL, 1);
2592
	kvm_arch_hardware_unsetup();
2593
	kvm_arch_exit();
2594
	free_cpumask_var(cpus_hardware_enabled);
2595
	__free_page(hwpoison_page);
2596
	__free_page(bad_page);
A
Avi Kivity 已提交
2597
}
2598
EXPORT_SYMBOL_GPL(kvm_exit);