kvm_main.c 75.7 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "iodev.h"
A
Avi Kivity 已提交
20

21
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/syscore_ops.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
35
#include <linux/sched.h>
36 37
#include <linux/cpumask.h>
#include <linux/smp.h>
38
#include <linux/anon_inodes.h>
39
#include <linux/profile.h>
40
#include <linux/kvm_para.h>
41
#include <linux/pagemap.h>
42
#include <linux/mman.h>
43
#include <linux/swap.h>
44
#include <linux/bitops.h>
45
#include <linux/spinlock.h>
46
#include <linux/compat.h>
47
#include <linux/srcu.h>
J
Joerg Roedel 已提交
48
#include <linux/hugetlb.h>
49
#include <linux/slab.h>
50 51
#include <linux/sort.h>
#include <linux/bsearch.h>
A
Avi Kivity 已提交
52

A
Avi Kivity 已提交
53 54
#include <asm/processor.h>
#include <asm/io.h>
55
#include <asm/ioctl.h>
A
Avi Kivity 已提交
56
#include <asm/uaccess.h>
57
#include <asm/pgtable.h>
A
Avi Kivity 已提交
58

59
#include "coalesced_mmio.h"
60
#include "async_pf.h"
P
Paolo Bonzini 已提交
61
#include "vfio.h"
62

63 64 65
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

A
Avi Kivity 已提交
66 67 68
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

69 70 71
/*
 * Ordering of locks:
 *
72
 * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
73 74
 */

75
DEFINE_SPINLOCK(kvm_lock);
76
static DEFINE_RAW_SPINLOCK(kvm_count_lock);
77
LIST_HEAD(vm_list);
78

79
static cpumask_var_t cpus_hardware_enabled;
80 81
static int kvm_usage_count = 0;
static atomic_t hardware_enable_failed;
82

83 84
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
85

86 87
static __read_mostly struct preempt_ops kvm_preempt_ops;

88
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
89

A
Avi Kivity 已提交
90 91
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
92 93 94 95
#ifdef CONFIG_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
				  unsigned long arg);
#endif
96 97
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
98

M
Marcelo Tosatti 已提交
99
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
100 101 102 103

static void kvm_release_pfn_dirty(pfn_t pfn);
static void mark_page_dirty_in_slot(struct kvm *kvm,
				    struct kvm_memory_slot *memslot, gfn_t gfn);
M
Marcelo Tosatti 已提交
104

105
__visible bool kvm_rebooting;
106
EXPORT_SYMBOL_GPL(kvm_rebooting);
107

108 109
static bool largepages_enabled = true;

X
Xiao Guangrong 已提交
110
bool kvm_is_mmio_pfn(pfn_t pfn)
B
Ben-Ami Yassour 已提交
111
{
112
	if (pfn_valid(pfn))
113
		return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn));
B
Ben-Ami Yassour 已提交
114 115 116 117

	return true;
}

A
Avi Kivity 已提交
118 119 120
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
121
int vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
122
{
123 124
	int cpu;

125 126
	if (mutex_lock_killable(&vcpu->mutex))
		return -EINTR;
127 128 129 130 131
	if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
		/* The thread running this VCPU changed. */
		struct pid *oldpid = vcpu->pid;
		struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
		rcu_assign_pointer(vcpu->pid, newpid);
132 133
		if (oldpid)
			synchronize_rcu();
134 135
		put_pid(oldpid);
	}
136 137
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
138
	kvm_arch_vcpu_load(vcpu, cpu);
139
	put_cpu();
140
	return 0;
A
Avi Kivity 已提交
141 142
}

143
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
144
{
145
	preempt_disable();
146
	kvm_arch_vcpu_put(vcpu);
147 148
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
149 150 151
	mutex_unlock(&vcpu->mutex);
}

152 153 154 155
static void ack_flush(void *_completed)
{
}

156
bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
157
{
158
	int i, cpu, me;
159 160
	cpumask_var_t cpus;
	bool called = true;
161 162
	struct kvm_vcpu *vcpu;

163
	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
164

165
	me = get_cpu();
166
	kvm_for_each_vcpu(i, vcpu, kvm) {
167
		kvm_make_request(req, vcpu);
168
		cpu = vcpu->cpu;
169 170 171 172 173 174

		/* Set ->requests bit before we read ->mode */
		smp_mb();

		if (cpus != NULL && cpu != -1 && cpu != me &&
		      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
175
			cpumask_set_cpu(cpu, cpus);
176
	}
177 178 179 180 181 182
	if (unlikely(cpus == NULL))
		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
	else if (!cpumask_empty(cpus))
		smp_call_function_many(cpus, ack_flush, NULL, 1);
	else
		called = false;
183
	put_cpu();
184
	free_cpumask_var(cpus);
185
	return called;
186 187
}

188
void kvm_flush_remote_tlbs(struct kvm *kvm)
189
{
190 191 192
	long dirty_count = kvm->tlbs_dirty;

	smp_mb();
193
	if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
194
		++kvm->stat.remote_tlb_flush;
195
	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
196
}
197
EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
198

199 200
void kvm_reload_remote_mmus(struct kvm *kvm)
{
201
	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
202
}
203

204 205
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
206
	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
207 208
}

209
void kvm_make_scan_ioapic_request(struct kvm *kvm)
210
{
211
	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
212 213
}

R
Rusty Russell 已提交
214 215 216 217 218 219 220 221 222
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
223
	vcpu->pid = NULL;
E
Eddie Dong 已提交
224
	init_waitqueue_head(&vcpu->wq);
225
	kvm_async_pf_vcpu_init(vcpu);
R
Rusty Russell 已提交
226 227 228 229 230 231 232 233

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

234 235
	kvm_vcpu_set_in_spin_loop(vcpu, false);
	kvm_vcpu_set_dy_eligible(vcpu, false);
236
	vcpu->preempted = false;
237

238
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
239
	if (r < 0)
240
		goto fail_free_run;
R
Rusty Russell 已提交
241 242 243 244 245
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
246
	return r;
R
Rusty Russell 已提交
247 248 249 250 251
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
252
	put_pid(vcpu->pid);
253
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
254 255 256 257
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

258 259 260 261 262 263 264 265 266 267 268
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
269
	int need_tlb_flush, idx;
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
289
	idx = srcu_read_lock(&kvm->srcu);
290
	spin_lock(&kvm->mmu_lock);
291

292
	kvm->mmu_notifier_seq++;
293
	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
294 295 296 297
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

298
	spin_unlock(&kvm->mmu_lock);
299 300 301

	kvm_arch_mmu_notifier_invalidate_page(kvm, address);

302
	srcu_read_unlock(&kvm->srcu, idx);
303 304
}

305 306 307 308 309 310
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
311
	int idx;
312

313
	idx = srcu_read_lock(&kvm->srcu);
314 315 316 317
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	kvm_set_spte_hva(kvm, address, pte);
	spin_unlock(&kvm->mmu_lock);
318
	srcu_read_unlock(&kvm->srcu, idx);
319 320
}

321 322 323 324 325 326
static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
327
	int need_tlb_flush = 0, idx;
328

329
	idx = srcu_read_lock(&kvm->srcu);
330 331 332 333 334 335 336
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
337
	need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
338
	need_tlb_flush |= kvm->tlbs_dirty;
339 340 341
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
342 343 344

	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
361
	smp_wmb();
362 363
	/*
	 * The above sequence increase must be visible before the
364 365
	 * below count decrease, which is ensured by the smp_wmb above
	 * in conjunction with the smp_rmb in mmu_notifier_retry().
366 367 368 369 370 371 372 373 374
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
A
Andres Lagar-Cavilla 已提交
375 376
					      unsigned long start,
					      unsigned long end)
377 378
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
379
	int young, idx;
380

381
	idx = srcu_read_lock(&kvm->srcu);
382 383
	spin_lock(&kvm->mmu_lock);

A
Andres Lagar-Cavilla 已提交
384
	young = kvm_age_hva(kvm, start, end);
385 386 387
	if (young)
		kvm_flush_remote_tlbs(kvm);

388 389 390
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

391 392 393
	return young;
}

A
Andrea Arcangeli 已提交
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
				       struct mm_struct *mm,
				       unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	young = kvm_test_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

410 411 412 413
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
414 415 416
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
417
	kvm_arch_flush_shadow_all(kvm);
418
	srcu_read_unlock(&kvm->srcu, idx);
419 420
}

421 422 423 424 425
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
A
Andrea Arcangeli 已提交
426
	.test_young		= kvm_mmu_notifier_test_young,
427
	.change_pte		= kvm_mmu_notifier_change_pte,
428
	.release		= kvm_mmu_notifier_release,
429
};
430 431 432 433 434 435 436 437 438 439 440 441 442 443

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

444 445
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

446 447 448 449 450 451
static void kvm_init_memslots_id(struct kvm *kvm)
{
	int i;
	struct kvm_memslots *slots = kvm->memslots;

	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
452
		slots->id_to_index[i] = slots->memslots[i].id = i;
453 454
}

455
static struct kvm *kvm_create_vm(unsigned long type)
A
Avi Kivity 已提交
456
{
457 458
	int r, i;
	struct kvm *kvm = kvm_arch_alloc_vm();
A
Avi Kivity 已提交
459

460 461 462
	if (!kvm)
		return ERR_PTR(-ENOMEM);

463
	r = kvm_arch_init_vm(kvm, type);
464
	if (r)
465
		goto out_err_no_disable;
466 467 468

	r = hardware_enable_all();
	if (r)
469
		goto out_err_no_disable;
470

471 472
#ifdef CONFIG_HAVE_KVM_IRQCHIP
	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
473 474
#endif
#ifdef CONFIG_HAVE_KVM_IRQFD
475
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
476
#endif
A
Avi Kivity 已提交
477

478 479
	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);

480 481 482
	r = -ENOMEM;
	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
	if (!kvm->memslots)
483
		goto out_err_no_srcu;
484 485 486 487 488 489 490

	/*
	 * Init kvm generation close to the maximum to easily test the
	 * code of handling generation number wrap-around.
	 */
	kvm->memslots->generation = -150;

491
	kvm_init_memslots_id(kvm);
492
	if (init_srcu_struct(&kvm->srcu))
493 494 495
		goto out_err_no_srcu;
	if (init_srcu_struct(&kvm->irq_srcu))
		goto out_err_no_irq_srcu;
M
Marcelo Tosatti 已提交
496 497 498
	for (i = 0; i < KVM_NR_BUSES; i++) {
		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
					GFP_KERNEL);
499
		if (!kvm->buses[i])
M
Marcelo Tosatti 已提交
500 501
			goto out_err;
	}
502

503
	spin_lock_init(&kvm->mmu_lock);
504 505
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
G
Gregory Haskins 已提交
506
	kvm_eventfd_init(kvm);
S
Shaohua Li 已提交
507
	mutex_init(&kvm->lock);
508
	mutex_init(&kvm->irq_lock);
509
	mutex_init(&kvm->slots_lock);
I
Izik Eidus 已提交
510
	atomic_set(&kvm->users_count, 1);
511
	INIT_LIST_HEAD(&kvm->devices);
512 513 514 515 516

	r = kvm_init_mmu_notifier(kvm);
	if (r)
		goto out_err;

517
	spin_lock(&kvm_lock);
518
	list_add(&kvm->vm_list, &vm_list);
519
	spin_unlock(&kvm_lock);
520

521
	return kvm;
522 523

out_err:
524 525
	cleanup_srcu_struct(&kvm->irq_srcu);
out_err_no_irq_srcu:
526
	cleanup_srcu_struct(&kvm->srcu);
527
out_err_no_srcu:
528
	hardware_disable_all();
529
out_err_no_disable:
M
Marcelo Tosatti 已提交
530 531
	for (i = 0; i < KVM_NR_BUSES; i++)
		kfree(kvm->buses[i]);
532
	kfree(kvm->memslots);
533
	kvm_arch_free_vm(kvm);
534
	return ERR_PTR(r);
535 536
}

537 538 539 540
/*
 * Avoid using vmalloc for a small buffer.
 * Should not be used when the size is statically known.
 */
541
void *kvm_kvzalloc(unsigned long size)
542 543 544 545 546 547 548
{
	if (size > PAGE_SIZE)
		return vzalloc(size);
	else
		return kzalloc(size, GFP_KERNEL);
}

549
void kvm_kvfree(const void *addr)
550 551 552 553 554 555 556
{
	if (is_vmalloc_addr(addr))
		vfree(addr);
	else
		kfree(addr);
}

557 558 559 560 561
static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
	if (!memslot->dirty_bitmap)
		return;

562
	kvm_kvfree(memslot->dirty_bitmap);
563 564 565
	memslot->dirty_bitmap = NULL;
}

A
Avi Kivity 已提交
566 567 568
/*
 * Free any memory in @free but not in @dont.
 */
569
static void kvm_free_physmem_slot(struct kvm *kvm, struct kvm_memory_slot *free,
A
Avi Kivity 已提交
570 571 572
				  struct kvm_memory_slot *dont)
{
	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
573
		kvm_destroy_dirty_bitmap(free);
A
Avi Kivity 已提交
574

575
	kvm_arch_free_memslot(kvm, free, dont);
M
Marcelo Tosatti 已提交
576

A
Avi Kivity 已提交
577 578 579
	free->npages = 0;
}

580
static void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
581
{
582
	struct kvm_memslots *slots = kvm->memslots;
583
	struct kvm_memory_slot *memslot;
584

585
	kvm_for_each_memslot(memslot, slots)
586
		kvm_free_physmem_slot(kvm, memslot, NULL);
A
Avi Kivity 已提交
587

588
	kfree(kvm->memslots);
A
Avi Kivity 已提交
589 590
}

591 592 593 594 595 596 597 598 599 600 601 602 603
static void kvm_destroy_devices(struct kvm *kvm)
{
	struct list_head *node, *tmp;

	list_for_each_safe(node, tmp, &kvm->devices) {
		struct kvm_device *dev =
			list_entry(node, struct kvm_device, vm_node);

		list_del(node);
		dev->ops->destroy(dev);
	}
}

604 605
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
606
	int i;
607 608
	struct mm_struct *mm = kvm->mm;

609
	kvm_arch_sync_events(kvm);
610
	spin_lock(&kvm_lock);
611
	list_del(&kvm->vm_list);
612
	spin_unlock(&kvm_lock);
613
	kvm_free_irq_routing(kvm);
M
Marcelo Tosatti 已提交
614 615
	for (i = 0; i < KVM_NR_BUSES; i++)
		kvm_io_bus_destroy(kvm->buses[i]);
616
	kvm_coalesced_mmio_free(kvm);
617 618
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
619
#else
620
	kvm_arch_flush_shadow_all(kvm);
621
#endif
622
	kvm_arch_destroy_vm(kvm);
623
	kvm_destroy_devices(kvm);
624
	kvm_free_physmem(kvm);
625
	cleanup_srcu_struct(&kvm->irq_srcu);
626 627
	cleanup_srcu_struct(&kvm->srcu);
	kvm_arch_free_vm(kvm);
628
	hardware_disable_all();
629
	mmdrop(mm);
630 631
}

I
Izik Eidus 已提交
632 633 634 635 636 637 638 639 640 641 642 643 644 645
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


646 647 648 649
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
650 651
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
652
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
653 654 655
	return 0;
}

656 657
/*
 * Allocation size is twice as large as the actual dirty bitmap size.
658
 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
659
 */
660 661
static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
{
662
	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
663

664
	memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
665 666 667 668 669 670
	if (!memslot->dirty_bitmap)
		return -ENOMEM;

	return 0;
}

671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691
static int cmp_memslot(const void *slot1, const void *slot2)
{
	struct kvm_memory_slot *s1, *s2;

	s1 = (struct kvm_memory_slot *)slot1;
	s2 = (struct kvm_memory_slot *)slot2;

	if (s1->npages < s2->npages)
		return 1;
	if (s1->npages > s2->npages)
		return -1;

	return 0;
}

/*
 * Sort the memslots base on its size, so the larger slots
 * will get better fit.
 */
static void sort_memslots(struct kvm_memslots *slots)
{
692 693
	int i;

694 695
	sort(slots->memslots, KVM_MEM_SLOTS_NUM,
	      sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
696 697 698

	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
		slots->id_to_index[slots->memslots[i].id] = i;
699 700
}

701
static void update_memslots(struct kvm_memslots *slots,
702
			    struct kvm_memory_slot *new)
703 704 705
{
	if (new) {
		int id = new->id;
706
		struct kvm_memory_slot *old = id_to_memslot(slots, id);
707
		unsigned long npages = old->npages;
708

709
		*old = *new;
710 711
		if (new->npages != npages)
			sort_memslots(slots);
712 713 714
	}
}

715 716
static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
{
X
Xiao Guangrong 已提交
717 718
	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;

719
#ifdef __KVM_HAVE_READONLY_MEM
X
Xiao Guangrong 已提交
720 721 722 723
	valid_flags |= KVM_MEM_READONLY;
#endif

	if (mem->flags & ~valid_flags)
724 725 726 727 728
		return -EINVAL;

	return 0;
}

729 730 731 732 733
static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
		struct kvm_memslots *slots, struct kvm_memory_slot *new)
{
	struct kvm_memslots *old_memslots = kvm->memslots;

734 735 736 737 738 739 740 741
	/*
	 * Set the low bit in the generation, which disables SPTE caching
	 * until the end of synchronize_srcu_expedited.
	 */
	WARN_ON(old_memslots->generation & 1);
	slots->generation = old_memslots->generation + 1;

	update_memslots(slots, new);
742 743
	rcu_assign_pointer(kvm->memslots, slots);
	synchronize_srcu_expedited(&kvm->srcu);
744

745 746 747 748 749 750 751
	/*
	 * Increment the new memslot generation a second time. This prevents
	 * vm exits that race with memslot updates from caching a memslot
	 * generation that will (potentially) be valid forever.
	 */
	slots->generation++;

752 753 754
	kvm_arch_memslots_updated(kvm);

	return old_memslots;
755 756
}

A
Avi Kivity 已提交
757 758 759 760 761
/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
762
 *
763
 * Must be called holding mmap_sem for write.
A
Avi Kivity 已提交
764
 */
765
int __kvm_set_memory_region(struct kvm *kvm,
766
			    struct kvm_userspace_memory_region *mem)
A
Avi Kivity 已提交
767
{
768
	int r;
A
Avi Kivity 已提交
769
	gfn_t base_gfn;
770
	unsigned long npages;
771
	struct kvm_memory_slot *slot;
A
Avi Kivity 已提交
772
	struct kvm_memory_slot old, new;
773
	struct kvm_memslots *slots = NULL, *old_memslots;
774
	enum kvm_mr_change change;
A
Avi Kivity 已提交
775

776 777 778 779
	r = check_memory_region_flags(mem);
	if (r)
		goto out;

A
Avi Kivity 已提交
780 781 782 783 784 785
	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
786
	/* We can read the guest memory with __xxx_user() later on. */
787
	if ((mem->slot < KVM_USER_MEM_SLOTS) &&
788
	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
789 790 791
	     !access_ok(VERIFY_WRITE,
			(void __user *)(unsigned long)mem->userspace_addr,
			mem->memory_size)))
792
		goto out;
793
	if (mem->slot >= KVM_MEM_SLOTS_NUM)
A
Avi Kivity 已提交
794 795 796 797
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

798
	slot = id_to_memslot(kvm->memslots, mem->slot);
A
Avi Kivity 已提交
799 800 801
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

802 803 804
	if (npages > KVM_MEM_MAX_NR_PAGES)
		goto out;

A
Avi Kivity 已提交
805 806 807
	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

808
	new = old = *slot;
A
Avi Kivity 已提交
809

810
	new.id = mem->slot;
A
Avi Kivity 已提交
811 812 813 814
	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

815 816 817 818 819
	if (npages) {
		if (!old.npages)
			change = KVM_MR_CREATE;
		else { /* Modify an existing slot. */
			if ((mem->userspace_addr != old.userspace_addr) ||
820 821
			    (npages != old.npages) ||
			    ((new.flags ^ old.flags) & KVM_MEM_READONLY))
822 823 824 825 826 827 828 829 830 831 832 833 834 835
				goto out;

			if (base_gfn != old.base_gfn)
				change = KVM_MR_MOVE;
			else if (new.flags != old.flags)
				change = KVM_MR_FLAGS_ONLY;
			else { /* Nothing to change. */
				r = 0;
				goto out;
			}
		}
	} else if (old.npages) {
		change = KVM_MR_DELETE;
	} else /* Modify a non-existent slot: disallowed. */
836
		goto out;
A
Avi Kivity 已提交
837

838
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
839 840 841
		/* Check for overlaps */
		r = -EEXIST;
		kvm_for_each_memslot(slot, kvm->memslots) {
842 843
			if ((slot->id >= KVM_USER_MEM_SLOTS) ||
			    (slot->id == mem->slot))
844 845 846 847 848
				continue;
			if (!((base_gfn + npages <= slot->base_gfn) ||
			      (base_gfn >= slot->base_gfn + slot->npages)))
				goto out;
		}
A
Avi Kivity 已提交
849 850 851 852
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
853
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
854 855

	r = -ENOMEM;
856
	if (change == KVM_MR_CREATE) {
857
		new.userspace_addr = mem->userspace_addr;
858

859
		if (kvm_arch_create_memslot(kvm, &new, npages))
860
			goto out_free;
A
Avi Kivity 已提交
861
	}
862

A
Avi Kivity 已提交
863 864
	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
865
		if (kvm_create_dirty_bitmap(&new) < 0)
866
			goto out_free;
A
Avi Kivity 已提交
867 868
	}

869
	if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
870 871
		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
				GFP_KERNEL);
872 873
		if (!slots)
			goto out_free;
874 875 876
		slot = id_to_memslot(slots, mem->slot);
		slot->flags |= KVM_MEMSLOT_INVALID;

877
		old_memslots = install_new_memslots(kvm, slots, NULL);
878

879 880
		/* slot was deleted or moved, clear iommu mapping */
		kvm_iommu_unmap_pages(kvm, &old);
881 882
		/* From this point no new shadow pages pointing to a deleted,
		 * or moved, memslot will be created.
883 884 885 886 887
		 *
		 * validation of sp->gfn happens in:
		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
		 * 	- kvm_is_visible_gfn (mmu_check_roots)
		 */
888
		kvm_arch_flush_shadow_memslot(kvm, slot);
889
		slots = old_memslots;
890
	}
891

892
	r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
893
	if (r)
894
		goto out_slots;
895

896
	r = -ENOMEM;
897 898 899 900 901 902 903 904 905 906 907
	/*
	 * We can re-use the old_memslots from above, the only difference
	 * from the currently installed memslots is the invalid flag.  This
	 * will get overwritten by update_memslots anyway.
	 */
	if (!slots) {
		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
				GFP_KERNEL);
		if (!slots)
			goto out_free;
	}
908 909

	/* actual memory is freed via old in kvm_free_physmem_slot below */
910
	if (change == KVM_MR_DELETE) {
911
		new.dirty_bitmap = NULL;
912
		memset(&new.arch, 0, sizeof(new.arch));
913 914
	}

915
	old_memslots = install_new_memslots(kvm, slots, &new);
916

917
	kvm_arch_commit_memory_region(kvm, mem, &old, change);
918

919
	kvm_free_physmem_slot(kvm, &old, &new);
920 921
	kfree(old_memslots);

922 923
	/*
	 * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
924 925 926 927 928 929
	 * un-mapped and re-mapped if their base changes.  Since base change
	 * unmapping is handled above with slot deletion, mapping alone is
	 * needed here.  Anything else the iommu might care about for existing
	 * slots (size changes, userspace addr changes and read-only flag
	 * changes) is disallowed above, so any other attribute changes getting
	 * here can be skipped.
930
	 */
931 932
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
		r = kvm_iommu_map_pages(kvm, &new);
933
		return r;
934 935
	}

A
Avi Kivity 已提交
936 937
	return 0;

938 939
out_slots:
	kfree(slots);
940
out_free:
941
	kvm_free_physmem_slot(kvm, &new, &old);
A
Avi Kivity 已提交
942 943
out:
	return r;
944
}
945 946 947
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
948
			  struct kvm_userspace_memory_region *mem)
949 950 951
{
	int r;

952
	mutex_lock(&kvm->slots_lock);
953
	r = __kvm_set_memory_region(kvm, mem);
954
	mutex_unlock(&kvm->slots_lock);
955 956
	return r;
}
957 958
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

959 960
static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
					  struct kvm_userspace_memory_region *mem)
961
{
962
	if (mem->slot >= KVM_USER_MEM_SLOTS)
963
		return -EINVAL;
964
	return kvm_set_memory_region(kvm, mem);
A
Avi Kivity 已提交
965 966
}

967 968
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
969 970 971
{
	struct kvm_memory_slot *memslot;
	int r, i;
972
	unsigned long n;
A
Avi Kivity 已提交
973 974 975
	unsigned long any = 0;

	r = -EINVAL;
976
	if (log->slot >= KVM_USER_MEM_SLOTS)
A
Avi Kivity 已提交
977 978
		goto out;

979
	memslot = id_to_memslot(kvm->memslots, log->slot);
A
Avi Kivity 已提交
980 981 982 983
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

984
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
985

986
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
987 988 989 990 991 992
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

993 994
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
995 996 997 998 999

	r = 0;
out:
	return r;
}
1000
EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
A
Avi Kivity 已提交
1001

1002 1003 1004 1005 1006
bool kvm_largepages_enabled(void)
{
	return largepages_enabled;
}

1007 1008 1009 1010 1011 1012
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

1013 1014 1015 1016
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
A
Avi Kivity 已提交
1017
EXPORT_SYMBOL_GPL(gfn_to_memslot);
A
Avi Kivity 已提交
1018

1019 1020
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
1021
	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1022

1023
	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1024 1025
	      memslot->flags & KVM_MEMSLOT_INVALID)
		return 0;
1026

1027
	return 1;
1028 1029 1030
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

J
Joerg Roedel 已提交
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

X
Xiao Guangrong 已提交
1055 1056 1057 1058 1059 1060 1061
static bool memslot_is_readonly(struct kvm_memory_slot *slot)
{
	return slot->flags & KVM_MEM_READONLY;
}

static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				       gfn_t *nr_pages, bool write)
I
Izik Eidus 已提交
1062
{
1063
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
X
Xiao Guangrong 已提交
1064
		return KVM_HVA_ERR_BAD;
1065

X
Xiao Guangrong 已提交
1066 1067
	if (memslot_is_readonly(slot) && write)
		return KVM_HVA_ERR_RO_BAD;
1068 1069 1070 1071

	if (nr_pages)
		*nr_pages = slot->npages - (gfn - slot->base_gfn);

X
Xiao Guangrong 已提交
1072
	return __gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
1073
}
1074

X
Xiao Guangrong 已提交
1075 1076 1077 1078
static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				     gfn_t *nr_pages)
{
	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
I
Izik Eidus 已提交
1079
}
1080

X
Xiao Guangrong 已提交
1081
unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1082
					gfn_t gfn)
X
Xiao Guangrong 已提交
1083 1084 1085 1086 1087
{
	return gfn_to_hva_many(slot, gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);

1088 1089
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
1090
	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1091
}
1092
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1093

1094
/*
1095 1096
 * If writable is set to false, the hva returned by this function is only
 * allowed to be read.
1097
 */
1098 1099
unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
				      gfn_t gfn, bool *writable)
1100
{
1101 1102 1103
	unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);

	if (!kvm_is_error_hva(hva) && writable)
1104 1105
		*writable = !memslot_is_readonly(slot);

1106
	return hva;
1107 1108
}

1109 1110 1111 1112 1113 1114 1115
unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

1116
static int kvm_read_hva(void *data, void __user *hva, int len)
1117
{
1118 1119 1120 1121 1122 1123
	return __copy_from_user(data, hva, len);
}

static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
{
	return __copy_from_user_inatomic(data, hva, len);
1124 1125
}

G
Geoff Levand 已提交
1126
static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
	unsigned long start, int write, struct page **page)
{
	int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;

	if (write)
		flags |= FOLL_WRITE;

	return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
}

1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
int kvm_get_user_page_io(struct task_struct *tsk, struct mm_struct *mm,
			 unsigned long addr, bool write_fault,
			 struct page **pagep)
{
	int npages;
	int locked = 1;
	int flags = FOLL_TOUCH | FOLL_HWPOISON |
		    (pagep ? FOLL_GET : 0) |
		    (write_fault ? FOLL_WRITE : 0);

	/*
	 * If retrying the fault, we get here *not* having allowed the filemap
	 * to wait on the page lock. We should now allow waiting on the IO with
	 * the mmap semaphore released.
	 */
	down_read(&mm->mmap_sem);
	npages = __get_user_pages(tsk, mm, addr, 1, flags, pagep, NULL,
				  &locked);
	if (!locked) {
1156
		VM_BUG_ON(npages);
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173

		if (!pagep)
			return 0;

		/*
		 * The previous call has now waited on the IO. Now we can
		 * retry and complete. Pass TRIED to ensure we do not re
		 * schedule async IO (see e.g. filemap_fault).
		 */
		down_read(&mm->mmap_sem);
		npages = __get_user_pages(tsk, mm, addr, 1, flags | FOLL_TRIED,
					  pagep, NULL, NULL);
	}
	up_read(&mm->mmap_sem);
	return npages;
}

1174 1175 1176 1177 1178 1179 1180 1181 1182
static inline int check_user_page_hwpoison(unsigned long addr)
{
	int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;

	rc = __get_user_pages(current, current->mm, addr, 1,
			      flags, NULL, NULL, NULL);
	return rc == -EHWPOISON;
}

X
Xiao Guangrong 已提交
1183 1184 1185 1186 1187 1188
/*
 * The atomic path to get the writable pfn which will be stored in @pfn,
 * true indicates success, otherwise false is returned.
 */
static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
			    bool write_fault, bool *writable, pfn_t *pfn)
A
Avi Kivity 已提交
1189
{
1190
	struct page *page[1];
X
Xiao Guangrong 已提交
1191
	int npages;
A
Avi Kivity 已提交
1192

X
Xiao Guangrong 已提交
1193 1194
	if (!(async || atomic))
		return false;
1195

1196 1197 1198 1199 1200 1201 1202
	/*
	 * Fast pin a writable pfn only if it is a write fault request
	 * or the caller allows to map a writable pfn for a read fault
	 * request.
	 */
	if (!(write_fault || writable))
		return false;
1203

X
Xiao Guangrong 已提交
1204 1205 1206
	npages = __get_user_pages_fast(addr, 1, 1, page);
	if (npages == 1) {
		*pfn = page_to_pfn(page[0]);
1207

X
Xiao Guangrong 已提交
1208 1209 1210 1211
		if (writable)
			*writable = true;
		return true;
	}
1212

X
Xiao Guangrong 已提交
1213 1214
	return false;
}
1215

X
Xiao Guangrong 已提交
1216 1217 1218 1219 1220 1221 1222 1223 1224
/*
 * The slow path to get the pfn of the specified host virtual address,
 * 1 indicates success, -errno is returned if error is detected.
 */
static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
			   bool *writable, pfn_t *pfn)
{
	struct page *page[1];
	int npages = 0;
1225

X
Xiao Guangrong 已提交
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
	might_sleep();

	if (writable)
		*writable = write_fault;

	if (async) {
		down_read(&current->mm->mmap_sem);
		npages = get_user_page_nowait(current, current->mm,
					      addr, write_fault, page);
		up_read(&current->mm->mmap_sem);
1236 1237 1238 1239 1240 1241 1242 1243 1244
	} else {
		/*
		 * By now we have tried gup_fast, and possibly async_pf, and we
		 * are certainly not atomic. Time to retry the gup, allowing
		 * mmap semaphore to be relinquished in the case of IO.
		 */
		npages = kvm_get_user_page_io(current, current->mm, addr,
					      write_fault, page);
	}
X
Xiao Guangrong 已提交
1245 1246 1247 1248
	if (npages != 1)
		return npages;

	/* map read fault as writable if possible */
1249
	if (unlikely(!write_fault) && writable) {
X
Xiao Guangrong 已提交
1250 1251 1252 1253 1254 1255 1256
		struct page *wpage[1];

		npages = __get_user_pages_fast(addr, 1, 1, wpage);
		if (npages == 1) {
			*writable = true;
			put_page(page[0]);
			page[0] = wpage[0];
1257
		}
X
Xiao Guangrong 已提交
1258 1259

		npages = 1;
1260
	}
X
Xiao Guangrong 已提交
1261 1262 1263
	*pfn = page_to_pfn(page[0]);
	return npages;
}
I
Izik Eidus 已提交
1264

X
Xiao Guangrong 已提交
1265 1266 1267 1268
static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
{
	if (unlikely(!(vma->vm_flags & VM_READ)))
		return false;
1269

X
Xiao Guangrong 已提交
1270 1271
	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
		return false;
1272

X
Xiao Guangrong 已提交
1273 1274
	return true;
}
1275

1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289
/*
 * Pin guest page in memory and return its pfn.
 * @addr: host virtual address which maps memory to the guest
 * @atomic: whether this function can sleep
 * @async: whether this function need to wait IO complete if the
 *         host page is not in the memory
 * @write_fault: whether we should get a writable host page
 * @writable: whether it allows to map a writable host page for !@write_fault
 *
 * The function will map a writable host page for these two cases:
 * 1): @write_fault = true
 * 2): @write_fault = false && @writable, @writable will tell the caller
 *     whether the mapping is writable.
 */
X
Xiao Guangrong 已提交
1290 1291 1292 1293 1294 1295
static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
			bool write_fault, bool *writable)
{
	struct vm_area_struct *vma;
	pfn_t pfn = 0;
	int npages;
1296

X
Xiao Guangrong 已提交
1297 1298
	/* we can do it either atomically or asynchronously, not both */
	BUG_ON(atomic && async);
1299

X
Xiao Guangrong 已提交
1300 1301 1302 1303 1304 1305 1306 1307 1308
	if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
		return pfn;

	if (atomic)
		return KVM_PFN_ERR_FAULT;

	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
	if (npages == 1)
		return pfn;
1309

X
Xiao Guangrong 已提交
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
	down_read(&current->mm->mmap_sem);
	if (npages == -EHWPOISON ||
	      (!async && check_user_page_hwpoison(addr))) {
		pfn = KVM_PFN_ERR_HWPOISON;
		goto exit;
	}

	vma = find_vma_intersection(current->mm, addr, addr + 1);

	if (vma == NULL)
		pfn = KVM_PFN_ERR_FAULT;
	else if ((vma->vm_flags & VM_PFNMAP)) {
		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
			vma->vm_pgoff;
		BUG_ON(!kvm_is_mmio_pfn(pfn));
	} else {
X
Xiao Guangrong 已提交
1326
		if (async && vma_is_valid(vma, write_fault))
X
Xiao Guangrong 已提交
1327 1328 1329 1330 1331
			*async = true;
		pfn = KVM_PFN_ERR_FAULT;
	}
exit:
	up_read(&current->mm->mmap_sem);
1332
	return pfn;
1333 1334
}

X
Xiao Guangrong 已提交
1335 1336 1337
static pfn_t
__gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
		     bool *async, bool write_fault, bool *writable)
1338
{
X
Xiao Guangrong 已提交
1339 1340 1341 1342 1343 1344
	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);

	if (addr == KVM_HVA_ERR_RO_BAD)
		return KVM_PFN_ERR_RO_FAULT;

	if (kvm_is_error_hva(addr))
1345
		return KVM_PFN_NOSLOT;
X
Xiao Guangrong 已提交
1346 1347 1348 1349 1350 1351 1352 1353 1354

	/* Do not map writable pfn in the readonly memslot. */
	if (writable && memslot_is_readonly(slot)) {
		*writable = false;
		writable = NULL;
	}

	return hva_to_pfn(addr, atomic, async, write_fault,
			  writable);
1355 1356
}

1357 1358
static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
			  bool write_fault, bool *writable)
1359
{
X
Xiao Guangrong 已提交
1360
	struct kvm_memory_slot *slot;
1361

1362 1363 1364
	if (async)
		*async = false;

X
Xiao Guangrong 已提交
1365
	slot = gfn_to_memslot(kvm, gfn);
1366

X
Xiao Guangrong 已提交
1367 1368
	return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
				    writable);
1369 1370 1371 1372
}

pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
{
1373
	return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1374 1375 1376
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);

1377 1378
pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
		       bool write_fault, bool *writable)
1379
{
1380
	return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1381 1382 1383
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_async);

1384 1385
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
{
1386
	return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1387
}
1388 1389
EXPORT_SYMBOL_GPL(gfn_to_pfn);

1390 1391 1392 1393 1394 1395 1396
pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
		      bool *writable)
{
	return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);

1397
pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1398
{
X
Xiao Guangrong 已提交
1399
	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1400 1401
}

1402
pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1403
{
X
Xiao Guangrong 已提交
1404
	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1405
}
1406
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1407

1408 1409 1410 1411 1412 1413
int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
								  int nr_pages)
{
	unsigned long addr;
	gfn_t entry;

1414
	addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1415 1416 1417 1418 1419 1420 1421 1422 1423 1424
	if (kvm_is_error_hva(addr))
		return -1;

	if (entry < nr_pages)
		return 0;

	return __get_user_pages_fast(addr, nr_pages, 1, pages);
}
EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);

X
Xiao Guangrong 已提交
1425 1426
static struct page *kvm_pfn_to_page(pfn_t pfn)
{
1427
	if (is_error_noslot_pfn(pfn))
1428
		return KVM_ERR_PTR_BAD_PAGE;
X
Xiao Guangrong 已提交
1429

1430 1431
	if (kvm_is_mmio_pfn(pfn)) {
		WARN_ON(1);
1432
		return KVM_ERR_PTR_BAD_PAGE;
1433
	}
X
Xiao Guangrong 已提交
1434 1435 1436 1437

	return pfn_to_page(pfn);
}

1438 1439
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
1440 1441 1442 1443
	pfn_t pfn;

	pfn = gfn_to_pfn(kvm, gfn);

X
Xiao Guangrong 已提交
1444
	return kvm_pfn_to_page(pfn);
A
Avi Kivity 已提交
1445
}
1446

A
Avi Kivity 已提交
1447 1448
EXPORT_SYMBOL_GPL(gfn_to_page);

1449 1450
void kvm_release_page_clean(struct page *page)
{
1451 1452
	WARN_ON(is_error_page(page));

1453
	kvm_release_pfn_clean(page_to_pfn(page));
1454 1455 1456
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

1457 1458
void kvm_release_pfn_clean(pfn_t pfn)
{
1459
	if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn))
1460
		put_page(pfn_to_page(pfn));
1461 1462 1463
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1464
void kvm_release_page_dirty(struct page *page)
1465
{
X
Xiao Guangrong 已提交
1466 1467
	WARN_ON(is_error_page(page));

1468 1469 1470 1471
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

1472
static void kvm_release_pfn_dirty(pfn_t pfn)
1473 1474 1475 1476 1477 1478 1479
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}

void kvm_set_pfn_dirty(pfn_t pfn)
{
1480
	if (!kvm_is_mmio_pfn(pfn)) {
1481 1482 1483 1484
		struct page *page = pfn_to_page(pfn);
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1485
}
1486 1487 1488 1489
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
1490
	if (!kvm_is_mmio_pfn(pfn))
1491
		mark_page_accessed(pfn_to_page(pfn));
1492 1493 1494 1495 1496
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
1497
	if (!kvm_is_mmio_pfn(pfn))
1498
		get_page(pfn_to_page(pfn));
1499 1500
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1501

1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
1513 1514
	int r;
	unsigned long addr;
1515

1516
	addr = gfn_to_hva_prot(kvm, gfn, NULL);
1517 1518
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1519
	r = kvm_read_hva(data, (void __user *)addr + offset, len);
1520
	if (r)
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1546 1547 1548 1549 1550 1551 1552 1553
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

1554
	addr = gfn_to_hva_prot(kvm, gfn, NULL);
1555 1556
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1557
	pagefault_disable();
1558
	r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
1559
	pagefault_enable();
1560 1561 1562 1563 1564 1565
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

1566 1567 1568
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
1569 1570
	int r;
	unsigned long addr;
1571

1572 1573 1574
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1575
	r = __copy_to_user((void __user *)addr + offset, data, len);
1576
	if (r)
1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

1603
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1604
			      gpa_t gpa, unsigned long len)
1605 1606 1607
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int offset = offset_in_page(gpa);
1608 1609 1610 1611
	gfn_t start_gfn = gpa >> PAGE_SHIFT;
	gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
	gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
	gfn_t nr_pages_avail;
1612 1613 1614

	ghc->gpa = gpa;
	ghc->generation = slots->generation;
1615 1616 1617 1618
	ghc->len = len;
	ghc->memslot = gfn_to_memslot(kvm, start_gfn);
	ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, &nr_pages_avail);
	if (!kvm_is_error_hva(ghc->hva) && nr_pages_avail >= nr_pages_needed) {
1619
		ghc->hva += offset;
1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635
	} else {
		/*
		 * If the requested region crosses two memslots, we still
		 * verify that the entire region is valid here.
		 */
		while (start_gfn <= end_gfn) {
			ghc->memslot = gfn_to_memslot(kvm, start_gfn);
			ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
						   &nr_pages_avail);
			if (kvm_is_error_hva(ghc->hva))
				return -EFAULT;
			start_gfn += nr_pages_avail;
		}
		/* Use the slow path for cross page reads and writes. */
		ghc->memslot = NULL;
	}
1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);

int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

1646 1647
	BUG_ON(len > ghc->len);

1648
	if (slots->generation != ghc->generation)
1649 1650 1651 1652
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);

	if (unlikely(!ghc->memslot))
		return kvm_write_guest(kvm, ghc->gpa, data, len);
1653 1654 1655 1656

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

1657
	r = __copy_to_user((void __user *)ghc->hva, data, len);
1658 1659 1660 1661 1662 1663 1664 1665
	if (r)
		return -EFAULT;
	mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);

1666 1667 1668 1669 1670 1671
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

1672 1673
	BUG_ON(len > ghc->len);

1674
	if (slots->generation != ghc->generation)
1675 1676 1677 1678
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);

	if (unlikely(!ghc->memslot))
		return kvm_read_guest(kvm, ghc->gpa, data, len);
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

	r = __copy_from_user(data, (void __user *)ghc->hva, len);
	if (r)
		return -EFAULT;

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_cached);

1691 1692
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
1693 1694 1695
	const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));

	return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

1718 1719 1720
static void mark_page_dirty_in_slot(struct kvm *kvm,
				    struct kvm_memory_slot *memslot,
				    gfn_t gfn)
A
Avi Kivity 已提交
1721
{
R
Rusty Russell 已提交
1722 1723
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1724

1725
		set_bit_le(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1726 1727 1728
	}
}

1729 1730 1731 1732 1733 1734 1735
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = gfn_to_memslot(kvm, gfn);
	mark_page_dirty_in_slot(kvm, memslot, gfn);
}
1736
EXPORT_SYMBOL_GPL(mark_page_dirty);
1737

E
Eddie Dong 已提交
1738 1739 1740
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1741
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1742
{
1743 1744 1745 1746 1747
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

1748
		if (kvm_arch_vcpu_runnable(vcpu)) {
1749
			kvm_make_request(KVM_REQ_UNHALT, vcpu);
1750
			break;
1751
		}
1752 1753
		if (kvm_cpu_has_pending_timer(vcpu))
			break;
1754 1755 1756
		if (signal_pending(current))
			break;

E
Eddie Dong 已提交
1757 1758
		schedule();
	}
1759

1760
	finish_wait(&vcpu->wq, &wait);
E
Eddie Dong 已提交
1761
}
1762
EXPORT_SYMBOL_GPL(kvm_vcpu_block);
E
Eddie Dong 已提交
1763

1764
#ifndef CONFIG_S390
1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785
/*
 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
 */
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
	int me;
	int cpu = vcpu->cpu;
	wait_queue_head_t *wqp;

	wqp = kvm_arch_vcpu_wq(vcpu);
	if (waitqueue_active(wqp)) {
		wake_up_interruptible(wqp);
		++vcpu->stat.halt_wakeup;
	}

	me = get_cpu();
	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
		if (kvm_arch_vcpu_should_kick(vcpu))
			smp_send_reschedule(cpu);
	put_cpu();
}
1786
EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
1787
#endif /* !CONFIG_S390 */
1788

1789
int kvm_vcpu_yield_to(struct kvm_vcpu *target)
1790 1791 1792
{
	struct pid *pid;
	struct task_struct *task = NULL;
1793
	int ret = 0;
1794 1795 1796 1797

	rcu_read_lock();
	pid = rcu_dereference(target->pid);
	if (pid)
1798
		task = get_pid_task(pid, PIDTYPE_PID);
1799 1800
	rcu_read_unlock();
	if (!task)
1801
		return ret;
1802 1803
	if (task->flags & PF_VCPU) {
		put_task_struct(task);
1804
		return ret;
1805
	}
1806
	ret = yield_to(task, 1);
1807
	put_task_struct(task);
1808 1809

	return ret;
1810 1811 1812
}
EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);

1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834
/*
 * Helper that checks whether a VCPU is eligible for directed yield.
 * Most eligible candidate to yield is decided by following heuristics:
 *
 *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
 *  (preempted lock holder), indicated by @in_spin_loop.
 *  Set at the beiginning and cleared at the end of interception/PLE handler.
 *
 *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
 *  chance last time (mostly it has become eligible now since we have probably
 *  yielded to lockholder in last iteration. This is done by toggling
 *  @dy_eligible each time a VCPU checked for eligibility.)
 *
 *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
 *  to preempted lock-holder could result in wrong VCPU selection and CPU
 *  burning. Giving priority for a potential lock-holder increases lock
 *  progress.
 *
 *  Since algorithm is based on heuristics, accessing another VCPU data without
 *  locking does not harm. It may result in trying to yield to  same VCPU, fail
 *  and continue with next VCPU and so on.
 */
1835
static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
1836
{
1837
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
1838 1839 1840
	bool eligible;

	eligible = !vcpu->spin_loop.in_spin_loop ||
1841
		    vcpu->spin_loop.dy_eligible;
1842 1843 1844 1845 1846

	if (vcpu->spin_loop.in_spin_loop)
		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);

	return eligible;
1847 1848
#else
	return true;
1849
#endif
1850
}
1851

1852
void kvm_vcpu_on_spin(struct kvm_vcpu *me)
Z
Zhai, Edwin 已提交
1853
{
1854 1855 1856 1857
	struct kvm *kvm = me->kvm;
	struct kvm_vcpu *vcpu;
	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
	int yielded = 0;
1858
	int try = 3;
1859 1860
	int pass;
	int i;
Z
Zhai, Edwin 已提交
1861

1862
	kvm_vcpu_set_in_spin_loop(me, true);
1863 1864 1865 1866 1867 1868 1869
	/*
	 * We boost the priority of a VCPU that is runnable but not
	 * currently running, because it got preempted by something
	 * else and called schedule in __vcpu_run.  Hopefully that
	 * VCPU is holding the lock that we need and will release it.
	 * We approximate round-robin by starting at the last boosted VCPU.
	 */
1870
	for (pass = 0; pass < 2 && !yielded && try; pass++) {
1871
		kvm_for_each_vcpu(i, vcpu, kvm) {
1872
			if (!pass && i <= last_boosted_vcpu) {
1873 1874 1875 1876
				i = last_boosted_vcpu;
				continue;
			} else if (pass && i > last_boosted_vcpu)
				break;
1877 1878
			if (!ACCESS_ONCE(vcpu->preempted))
				continue;
1879 1880
			if (vcpu == me)
				continue;
1881
			if (waitqueue_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
1882
				continue;
1883 1884
			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
				continue;
1885 1886 1887

			yielded = kvm_vcpu_yield_to(vcpu);
			if (yielded > 0) {
1888 1889
				kvm->last_boosted_vcpu = i;
				break;
1890 1891 1892 1893
			} else if (yielded < 0) {
				try--;
				if (!try)
					break;
1894 1895 1896
			}
		}
	}
1897
	kvm_vcpu_set_in_spin_loop(me, false);
1898 1899 1900

	/* Ensure vcpu is not eligible during next spinloop */
	kvm_vcpu_set_dy_eligible(me, false);
Z
Zhai, Edwin 已提交
1901 1902 1903
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

1904
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1905 1906 1907 1908
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

1909
	if (vmf->pgoff == 0)
1910
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
1911
#ifdef CONFIG_X86
1912
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1913
		page = virt_to_page(vcpu->arch.pio_data);
1914 1915 1916 1917
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
1918
#endif
1919
	else
1920
		return kvm_arch_vcpu_fault(vcpu, vmf);
1921
	get_page(page);
1922 1923
	vmf->page = page;
	return 0;
1924 1925
}

1926
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1927
	.fault = kvm_vcpu_fault,
1928 1929 1930 1931 1932 1933 1934 1935
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
1936 1937 1938 1939
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
1940
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1941 1942 1943
	return 0;
}

1944
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
1945 1946
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
1947 1948 1949
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vcpu_compat_ioctl,
#endif
1950
	.mmap           = kvm_vcpu_mmap,
1951
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
1952 1953 1954 1955 1956 1957 1958
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
1959
	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
A
Avi Kivity 已提交
1960 1961
}

1962 1963 1964
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
1965
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1966 1967
{
	int r;
1968
	struct kvm_vcpu *vcpu, *v;
1969

1970 1971 1972
	if (id >= KVM_MAX_VCPUS)
		return -EINVAL;

1973
	vcpu = kvm_arch_vcpu_create(kvm, id);
R
Rusty Russell 已提交
1974 1975
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
1976

1977 1978
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

1979 1980
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
1981
		goto vcpu_destroy;
1982

S
Shaohua Li 已提交
1983
	mutex_lock(&kvm->lock);
1984 1985 1986 1987
	if (!kvm_vcpu_compatible(vcpu)) {
		r = -EINVAL;
		goto unlock_vcpu_destroy;
	}
1988 1989
	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
		r = -EINVAL;
1990
		goto unlock_vcpu_destroy;
R
Rusty Russell 已提交
1991
	}
1992

1993 1994
	kvm_for_each_vcpu(r, v, kvm)
		if (v->vcpu_id == id) {
1995
			r = -EEXIST;
1996
			goto unlock_vcpu_destroy;
1997 1998 1999
		}

	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2000

R
Rusty Russell 已提交
2001
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
2002
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
2003
	r = create_vcpu_fd(vcpu);
2004 2005
	if (r < 0) {
		kvm_put_kvm(kvm);
2006
		goto unlock_vcpu_destroy;
2007 2008 2009 2010 2011 2012 2013
	}

	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

	mutex_unlock(&kvm->lock);
2014
	kvm_arch_vcpu_postcreate(vcpu);
R
Rusty Russell 已提交
2015
	return r;
2016

2017
unlock_vcpu_destroy:
2018
	mutex_unlock(&kvm->lock);
2019
vcpu_destroy:
2020
	kvm_arch_vcpu_destroy(vcpu);
2021 2022 2023
	return r;
}

A
Avi Kivity 已提交
2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
2035 2036
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
2037
{
A
Avi Kivity 已提交
2038
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
2039
	void __user *argp = (void __user *)arg;
2040
	int r;
2041 2042
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
2043

2044 2045
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
2046

2047 2048 2049
	if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
		return -EINVAL;

2050
#if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
2051 2052 2053 2054 2055 2056 2057 2058 2059
	/*
	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
	 * so vcpu_load() would break it.
	 */
	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
#endif


2060 2061 2062
	r = vcpu_load(vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
2063
	switch (ioctl) {
2064
	case KVM_RUN:
2065 2066 2067
		r = -EINVAL;
		if (arg)
			goto out;
2068
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2069
		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
A
Avi Kivity 已提交
2070 2071
		break;
	case KVM_GET_REGS: {
2072
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2073

2074 2075 2076
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
2077
			goto out;
2078 2079 2080
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
2081
		r = -EFAULT;
2082 2083
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
2084
		r = 0;
2085 2086
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
2087 2088 2089
		break;
	}
	case KVM_SET_REGS: {
2090
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2091

2092
		r = -ENOMEM;
2093 2094 2095
		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
		if (IS_ERR(kvm_regs)) {
			r = PTR_ERR(kvm_regs);
A
Avi Kivity 已提交
2096
			goto out;
2097
		}
2098 2099
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
		kfree(kvm_regs);
A
Avi Kivity 已提交
2100 2101 2102
		break;
	}
	case KVM_GET_SREGS: {
2103 2104 2105 2106 2107
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2108 2109 2110
		if (r)
			goto out;
		r = -EFAULT;
2111
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
2112 2113 2114 2115 2116
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
2117 2118 2119
		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
		if (IS_ERR(kvm_sregs)) {
			r = PTR_ERR(kvm_sregs);
G
Guo Chao 已提交
2120
			kvm_sregs = NULL;
A
Avi Kivity 已提交
2121
			goto out;
2122
		}
2123
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2124 2125
		break;
	}
2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		break;
	}
A
Avi Kivity 已提交
2147 2148 2149 2150
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
2151
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
2152
			goto out;
2153
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2154 2155 2156
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2157
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
2158 2159 2160 2161
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
2162 2163
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
2164 2165

		r = -EFAULT;
A
Al Viro 已提交
2166
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
2167
			goto out;
J
Jan Kiszka 已提交
2168
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
2169 2170
		break;
	}
A
Avi Kivity 已提交
2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
2191
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
A
Avi Kivity 已提交
2192 2193
		break;
	}
A
Avi Kivity 已提交
2194
	case KVM_GET_FPU: {
2195 2196 2197 2198 2199
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2200 2201 2202
		if (r)
			goto out;
		r = -EFAULT;
2203
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
2204 2205 2206 2207 2208
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
2209 2210 2211
		fpu = memdup_user(argp, sizeof(*fpu));
		if (IS_ERR(fpu)) {
			r = PTR_ERR(fpu);
G
Guo Chao 已提交
2212
			fpu = NULL;
A
Avi Kivity 已提交
2213
			goto out;
2214
		}
2215
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2216 2217
		break;
	}
A
Avi Kivity 已提交
2218
	default:
2219
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2220 2221
	}
out:
2222
	vcpu_put(vcpu);
2223 2224
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
2225 2226 2227
	return r;
}

2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257
#ifdef CONFIG_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *filp,
				  unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = compat_ptr(arg);
	int r;

	if (vcpu->kvm->mm != current->mm)
		return -EIO;

	switch (ioctl) {
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		compat_sigset_t csigset;
		sigset_t sigset;

		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof csigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&csigset, sigmask_arg->sigset,
					   sizeof csigset))
				goto out;
2258 2259 2260 2261
			sigset_from_compat(&sigset, &csigset);
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		} else
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272
		break;
	}
	default:
		r = kvm_vcpu_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

S
Scott Wood 已提交
2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319
static int kvm_device_ioctl_attr(struct kvm_device *dev,
				 int (*accessor)(struct kvm_device *dev,
						 struct kvm_device_attr *attr),
				 unsigned long arg)
{
	struct kvm_device_attr attr;

	if (!accessor)
		return -EPERM;

	if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
		return -EFAULT;

	return accessor(dev, &attr);
}

static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
			     unsigned long arg)
{
	struct kvm_device *dev = filp->private_data;

	switch (ioctl) {
	case KVM_SET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
	case KVM_GET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
	case KVM_HAS_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
	default:
		if (dev->ops->ioctl)
			return dev->ops->ioctl(dev, ioctl, arg);

		return -ENOTTY;
	}
}

static int kvm_device_release(struct inode *inode, struct file *filp)
{
	struct kvm_device *dev = filp->private_data;
	struct kvm *kvm = dev->kvm;

	kvm_put_kvm(kvm);
	return 0;
}

static const struct file_operations kvm_device_fops = {
	.unlocked_ioctl = kvm_device_ioctl,
2320 2321 2322
#ifdef CONFIG_COMPAT
	.compat_ioctl = kvm_device_ioctl,
#endif
S
Scott Wood 已提交
2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333
	.release = kvm_device_release,
};

struct kvm_device *kvm_device_from_filp(struct file *filp)
{
	if (filp->f_op != &kvm_device_fops)
		return NULL;

	return filp->private_data;
}

2334
static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2335
#ifdef CONFIG_KVM_MPIC
2336 2337
	[KVM_DEV_TYPE_FSL_MPIC_20]	= &kvm_mpic_ops,
	[KVM_DEV_TYPE_FSL_MPIC_42]	= &kvm_mpic_ops,
2338
#endif
2339

2340
#ifdef CONFIG_KVM_XICS
2341
	[KVM_DEV_TYPE_XICS]		= &kvm_xics_ops,
A
Alex Williamson 已提交
2342
#endif
2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356
};

int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
{
	if (type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENOSPC;

	if (kvm_device_ops_table[type] != NULL)
		return -EEXIST;

	kvm_device_ops_table[type] = ops;
	return 0;
}

2357 2358 2359 2360 2361 2362
void kvm_unregister_device_ops(u32 type)
{
	if (kvm_device_ops_table[type] != NULL)
		kvm_device_ops_table[type] = NULL;
}

S
Scott Wood 已提交
2363 2364 2365 2366 2367 2368 2369 2370
static int kvm_ioctl_create_device(struct kvm *kvm,
				   struct kvm_create_device *cd)
{
	struct kvm_device_ops *ops = NULL;
	struct kvm_device *dev;
	bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
	int ret;

2371 2372 2373 2374 2375
	if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENODEV;

	ops = kvm_device_ops_table[cd->type];
	if (ops == NULL)
S
Scott Wood 已提交
2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393
		return -ENODEV;

	if (test)
		return 0;

	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
	if (!dev)
		return -ENOMEM;

	dev->ops = ops;
	dev->kvm = kvm;

	ret = ops->create(dev, cd->type);
	if (ret < 0) {
		kfree(dev);
		return ret;
	}

2394
	ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
S
Scott Wood 已提交
2395 2396 2397 2398 2399
	if (ret < 0) {
		ops->destroy(dev);
		return ret;
	}

2400
	list_add(&dev->vm_node, &kvm->devices);
S
Scott Wood 已提交
2401 2402 2403 2404 2405
	kvm_get_kvm(kvm);
	cd->fd = ret;
	return 0;
}

2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418
static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
{
	switch (arg) {
	case KVM_CAP_USER_MEMORY:
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_CAP_SET_BOOT_CPU_ID:
#endif
	case KVM_CAP_INTERNAL_ERROR_DATA:
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_CAP_SIGNAL_MSI:
#endif
2419
#ifdef CONFIG_HAVE_KVM_IRQFD
2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433
	case KVM_CAP_IRQFD_RESAMPLE:
#endif
	case KVM_CAP_CHECK_EXTENSION_VM:
		return 1;
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_CAP_IRQ_ROUTING:
		return KVM_MAX_IRQ_ROUTES;
#endif
	default:
		break;
	}
	return kvm_vm_ioctl_check_extension(kvm, arg);
}

A
Avi Kivity 已提交
2434 2435 2436 2437 2438
static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
2439
	int r;
A
Avi Kivity 已提交
2440

2441 2442
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
2443 2444 2445 2446
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		break;
2447 2448 2449 2450 2451 2452 2453 2454
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

2455
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
A
Avi Kivity 已提交
2456 2457 2458 2459 2460 2461
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
2462
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
2463
			goto out;
2464
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
2465 2466
		break;
	}
2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		break;
	}
#endif
G
Gregory Haskins 已提交
2485 2486 2487 2488 2489 2490
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
2491
		r = kvm_irqfd(kvm, &data);
G
Gregory Haskins 已提交
2492 2493
		break;
	}
G
Gregory Haskins 已提交
2494 2495 2496 2497 2498 2499 2500 2501 2502
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
2503 2504 2505
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
2506
		mutex_lock(&kvm->lock);
2507 2508 2509 2510
		if (atomic_read(&kvm->online_vcpus) != 0)
			r = -EBUSY;
		else
			kvm->bsp_vcpu_id = arg;
2511
		mutex_unlock(&kvm->lock);
2512
		break;
2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523
#endif
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_SIGNAL_MSI: {
		struct kvm_msi msi;

		r = -EFAULT;
		if (copy_from_user(&msi, argp, sizeof msi))
			goto out;
		r = kvm_send_userspace_msi(kvm, &msi);
		break;
	}
2524 2525 2526 2527 2528 2529 2530 2531 2532 2533
#endif
#ifdef __KVM_HAVE_IRQ_LINE
	case KVM_IRQ_LINE_STATUS:
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
		if (copy_from_user(&irq_event, argp, sizeof irq_event))
			goto out;

2534 2535
		r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
					ioctl == KVM_IRQ_LINE_STATUS);
2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547
		if (r)
			goto out;

		r = -EFAULT;
		if (ioctl == KVM_IRQ_LINE_STATUS) {
			if (copy_to_user(argp, &irq_event, sizeof irq_event))
				goto out;
		}

		r = 0;
		break;
	}
2548
#endif
2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_SET_GSI_ROUTING: {
		struct kvm_irq_routing routing;
		struct kvm_irq_routing __user *urouting;
		struct kvm_irq_routing_entry *entries;

		r = -EFAULT;
		if (copy_from_user(&routing, argp, sizeof(routing)))
			goto out;
		r = -EINVAL;
		if (routing.nr >= KVM_MAX_IRQ_ROUTES)
			goto out;
		if (routing.flags)
			goto out;
		r = -ENOMEM;
		entries = vmalloc(routing.nr * sizeof(*entries));
		if (!entries)
			goto out;
		r = -EFAULT;
		urouting = argp;
		if (copy_from_user(entries, urouting->entries,
				   routing.nr * sizeof(*entries)))
			goto out_free_irq_routing;
		r = kvm_set_irq_routing(kvm, entries, routing.nr,
					routing.flags);
	out_free_irq_routing:
		vfree(entries);
		break;
	}
#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
S
Scott Wood 已提交
2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
	case KVM_CREATE_DEVICE: {
		struct kvm_create_device cd;

		r = -EFAULT;
		if (copy_from_user(&cd, argp, sizeof(cd)))
			goto out;

		r = kvm_ioctl_create_device(kvm, &cd);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(argp, &cd, sizeof(cd)))
			goto out;

		r = 0;
		break;
	}
2597 2598 2599
	case KVM_CHECK_EXTENSION:
		r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
		break;
2600
	default:
2601
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2602 2603
		if (r == -ENOTTY)
			r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2604 2605 2606 2607 2608
	}
out:
	return r;
}

2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652
#ifdef CONFIG_COMPAT
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		r = -EFAULT;
		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
			goto out;
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

2653
static struct file_operations kvm_vm_fops = {
2654 2655
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
2656 2657 2658
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vm_compat_ioctl,
#endif
2659
	.llseek		= noop_llseek,
2660 2661
};

2662
static int kvm_dev_ioctl_create_vm(unsigned long type)
2663
{
2664
	int r;
2665 2666
	struct kvm *kvm;

2667
	kvm = kvm_create_vm(type);
2668 2669
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
2670 2671 2672 2673 2674 2675 2676
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	r = kvm_coalesced_mmio_init(kvm);
	if (r < 0) {
		kvm_put_kvm(kvm);
		return r;
	}
#endif
2677
	r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC);
2678
	if (r < 0)
A
Al Viro 已提交
2679
		kvm_put_kvm(kvm);
2680

2681
	return r;
2682 2683 2684 2685 2686
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
2687
	long r = -EINVAL;
2688 2689 2690

	switch (ioctl) {
	case KVM_GET_API_VERSION:
2691 2692
		if (arg)
			goto out;
2693 2694 2695
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
2696
		r = kvm_dev_ioctl_create_vm(arg);
2697
		break;
2698
	case KVM_CHECK_EXTENSION:
2699
		r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
2700
		break;
2701 2702 2703
	case KVM_GET_VCPU_MMAP_SIZE:
		if (arg)
			goto out;
2704 2705 2706
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
2707 2708 2709
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
		r += PAGE_SIZE;    /* coalesced mmio ring page */
2710
#endif
2711
		break;
2712 2713 2714
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
2715
		r = -EOPNOTSUPP;
2716
		break;
A
Avi Kivity 已提交
2717
	default:
2718
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2719 2720 2721 2722 2723 2724 2725 2726
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
2727
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
2728 2729 2730
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
2731
	KVM_MINOR,
A
Avi Kivity 已提交
2732 2733 2734 2735
	"kvm",
	&kvm_chardev_ops,
};

2736
static void hardware_enable_nolock(void *junk)
2737 2738
{
	int cpu = raw_smp_processor_id();
2739
	int r;
2740

2741
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2742
		return;
2743

2744
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
2745

2746
	r = kvm_arch_hardware_enable();
2747 2748 2749 2750 2751 2752 2753

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
		printk(KERN_INFO "kvm: enabling virtualization on "
				 "CPU%d failed\n", cpu);
	}
2754 2755
}

2756
static void hardware_enable(void)
2757
{
2758
	raw_spin_lock(&kvm_count_lock);
2759 2760
	if (kvm_usage_count)
		hardware_enable_nolock(NULL);
2761
	raw_spin_unlock(&kvm_count_lock);
2762 2763 2764
}

static void hardware_disable_nolock(void *junk)
2765 2766 2767
{
	int cpu = raw_smp_processor_id();

2768
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2769
		return;
2770
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2771
	kvm_arch_hardware_disable();
2772 2773
}

2774
static void hardware_disable(void)
2775
{
2776
	raw_spin_lock(&kvm_count_lock);
2777 2778
	if (kvm_usage_count)
		hardware_disable_nolock(NULL);
2779
	raw_spin_unlock(&kvm_count_lock);
2780 2781
}

2782 2783 2784 2785 2786 2787
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
2788
		on_each_cpu(hardware_disable_nolock, NULL, 1);
2789 2790 2791 2792
}

static void hardware_disable_all(void)
{
2793
	raw_spin_lock(&kvm_count_lock);
2794
	hardware_disable_all_nolock();
2795
	raw_spin_unlock(&kvm_count_lock);
2796 2797 2798 2799 2800 2801
}

static int hardware_enable_all(void)
{
	int r = 0;

2802
	raw_spin_lock(&kvm_count_lock);
2803 2804 2805 2806

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
2807
		on_each_cpu(hardware_enable_nolock, NULL, 1);
2808 2809 2810 2811 2812 2813 2814

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

2815
	raw_spin_unlock(&kvm_count_lock);
2816 2817 2818 2819

	return r;
}

A
Avi Kivity 已提交
2820 2821 2822 2823 2824
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

2825
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
2826
	switch (val) {
2827
	case CPU_DYING:
2828 2829
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
2830
		hardware_disable();
2831
		break;
2832
	case CPU_STARTING:
2833 2834
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
2835
		hardware_enable();
A
Avi Kivity 已提交
2836 2837 2838 2839 2840
		break;
	}
	return NOTIFY_OK;
}

2841
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
2842
		      void *v)
2843
{
2844 2845 2846 2847 2848 2849 2850 2851
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
	kvm_rebooting = true;
2852
	on_each_cpu(hardware_disable_nolock, NULL, 1);
2853 2854 2855 2856 2857 2858 2859 2860
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
2861
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2862 2863 2864 2865
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
2866
		struct kvm_io_device *pos = bus->range[i].dev;
2867 2868 2869

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
2870
	kfree(bus);
2871 2872
}

2873 2874
static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
                                 const struct kvm_io_range *r2)
2875 2876 2877 2878 2879 2880 2881 2882
{
	if (r1->addr < r2->addr)
		return -1;
	if (r1->addr + r1->len > r2->addr + r2->len)
		return 1;
	return 0;
}

2883 2884
static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
{
2885
	return kvm_io_bus_cmp(p1, p2);
2886 2887
}

G
Geoff Levand 已提交
2888
static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902
			  gpa_t addr, int len)
{
	bus->range[bus->dev_count++] = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
		.dev = dev,
	};

	sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
		kvm_io_bus_sort_cmp, NULL);

	return 0;
}

G
Geoff Levand 已提交
2903
static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920
			     gpa_t addr, int len)
{
	struct kvm_io_range *range, key;
	int off;

	key = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	range = bsearch(&key, bus->range, bus->dev_count,
			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
	if (range == NULL)
		return -ENOENT;

	off = range - bus->range;

2921
	while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
2922 2923 2924 2925 2926
		off--;

	return off;
}

C
Cornelia Huck 已提交
2927 2928 2929 2930 2931 2932 2933 2934 2935 2936
static int __kvm_io_bus_write(struct kvm_io_bus *bus,
			      struct kvm_io_range *range, const void *val)
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
2937
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
C
Cornelia Huck 已提交
2938 2939 2940 2941 2942 2943 2944 2945 2946
		if (!kvm_iodevice_write(bus->range[idx].dev, range->addr,
					range->len, val))
			return idx;
		idx++;
	}

	return -EOPNOTSUPP;
}

2947
/* kvm_io_bus_write - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2948
int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2949
		     int len, const void *val)
2950
{
2951
	struct kvm_io_bus *bus;
2952
	struct kvm_io_range range;
C
Cornelia Huck 已提交
2953
	int r;
2954 2955 2956 2957 2958

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
2959 2960

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
C
Cornelia Huck 已提交
2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980
	r = __kvm_io_bus_write(bus, &range, val);
	return r < 0 ? r : 0;
}

/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, const void *val, long cookie)
{
	struct kvm_io_bus *bus;
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);

	/* First try the device referenced by cookie. */
	if ((cookie >= 0) && (cookie < bus->dev_count) &&
2981
	    (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
C
Cornelia Huck 已提交
2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998
		if (!kvm_iodevice_write(bus->range[cookie].dev, addr, len,
					val))
			return cookie;

	/*
	 * cookie contained garbage; fall back to search and return the
	 * correct cookie value.
	 */
	return __kvm_io_bus_write(bus, &range, val);
}

static int __kvm_io_bus_read(struct kvm_io_bus *bus, struct kvm_io_range *range,
			     void *val)
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
2999 3000 3001 3002
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
3003
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
C
Cornelia Huck 已提交
3004 3005 3006
		if (!kvm_iodevice_read(bus->range[idx].dev, range->addr,
				       range->len, val))
			return idx;
3007 3008 3009
		idx++;
	}

3010 3011
	return -EOPNOTSUPP;
}
3012
EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3013

3014
/* kvm_io_bus_read - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
3015 3016
int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
		    int len, void *val)
3017
{
3018
	struct kvm_io_bus *bus;
3019
	struct kvm_io_range range;
C
Cornelia Huck 已提交
3020
	int r;
3021 3022 3023 3024 3025

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
M
Marcelo Tosatti 已提交
3026

3027
	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
C
Cornelia Huck 已提交
3028 3029 3030
	r = __kvm_io_bus_read(bus, &range, val);
	return r < 0 ? r : 0;
}
3031

3032

3033
/* Caller must hold slots_lock. */
3034 3035
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, struct kvm_io_device *dev)
3036
{
M
Marcelo Tosatti 已提交
3037
	struct kvm_io_bus *new_bus, *bus;
3038

M
Marcelo Tosatti 已提交
3039
	bus = kvm->buses[bus_idx];
3040 3041
	/* exclude ioeventfd which is limited by maximum fd */
	if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3042
		return -ENOSPC;
3043

3044 3045
	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
M
Marcelo Tosatti 已提交
3046 3047
	if (!new_bus)
		return -ENOMEM;
3048 3049
	memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
	       sizeof(struct kvm_io_range)));
3050
	kvm_io_bus_insert_dev(new_bus, dev, addr, len);
M
Marcelo Tosatti 已提交
3051 3052 3053
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
3054 3055 3056 3057

	return 0;
}

3058
/* Caller must hold slots_lock. */
M
Marcelo Tosatti 已提交
3059 3060
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			      struct kvm_io_device *dev)
3061
{
M
Marcelo Tosatti 已提交
3062 3063
	int i, r;
	struct kvm_io_bus *new_bus, *bus;
3064

3065
	bus = kvm->buses[bus_idx];
M
Marcelo Tosatti 已提交
3066
	r = -ENOENT;
3067 3068
	for (i = 0; i < bus->dev_count; i++)
		if (bus->range[i].dev == dev) {
M
Marcelo Tosatti 已提交
3069
			r = 0;
3070 3071
			break;
		}
M
Marcelo Tosatti 已提交
3072

3073
	if (r)
M
Marcelo Tosatti 已提交
3074
		return r;
3075 3076 3077 3078 3079 3080 3081 3082 3083 3084

	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
	if (!new_bus)
		return -ENOMEM;

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count--;
	memcpy(new_bus->range + i, bus->range + i + 1,
	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
3085 3086 3087 3088 3089

	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
	return r;
3090 3091
}

A
Avi Kivity 已提交
3092 3093 3094 3095
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
};

3096
static int vm_stat_get(void *_offset, u64 *val)
3097 3098 3099 3100
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

3101
	*val = 0;
3102
	spin_lock(&kvm_lock);
3103
	list_for_each_entry(kvm, &vm_list, vm_list)
3104
		*val += *(u32 *)((void *)kvm + offset);
3105
	spin_unlock(&kvm_lock);
3106
	return 0;
3107 3108 3109 3110
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

3111
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
3112 3113 3114 3115 3116 3117
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

3118
	*val = 0;
3119
	spin_lock(&kvm_lock);
A
Avi Kivity 已提交
3120
	list_for_each_entry(kvm, &vm_list, vm_list)
3121 3122 3123
		kvm_for_each_vcpu(i, vcpu, kvm)
			*val += *(u32 *)((void *)vcpu + offset);

3124
	spin_unlock(&kvm_lock);
3125
	return 0;
A
Avi Kivity 已提交
3126 3127
}

3128 3129
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

3130
static const struct file_operations *stat_fops[] = {
3131 3132 3133
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
3134

3135
static int kvm_init_debug(void)
A
Avi Kivity 已提交
3136
{
3137
	int r = -EEXIST;
A
Avi Kivity 已提交
3138 3139
	struct kvm_stats_debugfs_item *p;

3140
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3141 3142 3143 3144
	if (kvm_debugfs_dir == NULL)
		goto out;

	for (p = debugfs_entries; p->name; ++p) {
3145
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
3146
						(void *)(long)p->offset,
3147
						stat_fops[p->kind]);
3148 3149 3150 3151 3152 3153 3154 3155 3156 3157
		if (p->dentry == NULL)
			goto out_dir;
	}

	return 0;

out_dir:
	debugfs_remove_recursive(kvm_debugfs_dir);
out:
	return r;
A
Avi Kivity 已提交
3158 3159 3160 3161 3162 3163 3164 3165
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
3166
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
3167 3168
}

3169
static int kvm_suspend(void)
3170
{
3171
	if (kvm_usage_count)
3172
		hardware_disable_nolock(NULL);
3173 3174 3175
	return 0;
}

3176
static void kvm_resume(void)
3177
{
3178
	if (kvm_usage_count) {
3179
		WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3180
		hardware_enable_nolock(NULL);
3181
	}
3182 3183
}

3184
static struct syscore_ops kvm_syscore_ops = {
3185 3186 3187 3188
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

3189 3190 3191 3192 3193 3194 3195 3196 3197
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3198 3199
	if (vcpu->preempted)
		vcpu->preempted = false;
3200

R
Radim Krčmář 已提交
3201 3202
	kvm_arch_sched_in(vcpu, cpu);

3203
	kvm_arch_vcpu_load(vcpu, cpu);
3204 3205 3206 3207 3208 3209 3210
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

3211 3212
	if (current->state == TASK_RUNNING)
		vcpu->preempted = true;
3213
	kvm_arch_vcpu_put(vcpu);
3214 3215
}

3216
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3217
		  struct module *module)
A
Avi Kivity 已提交
3218 3219
{
	int r;
Y
Yang, Sheng 已提交
3220
	int cpu;
A
Avi Kivity 已提交
3221

3222 3223
	r = kvm_arch_init(opaque);
	if (r)
3224
		goto out_fail;
3225

3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236
	/*
	 * kvm_arch_init makes sure there's at most one caller
	 * for architectures that support multiple implementations,
	 * like intel and amd on x86.
	 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
	 * conflicts in case kvm is already setup for another implementation.
	 */
	r = kvm_irqfd_init();
	if (r)
		goto out_irqfd;

3237
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
3238 3239 3240 3241
		r = -ENOMEM;
		goto out_free_0;
	}

3242
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
3243
	if (r < 0)
3244
		goto out_free_0a;
A
Avi Kivity 已提交
3245

Y
Yang, Sheng 已提交
3246 3247
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
3248
				kvm_arch_check_processor_compat,
3249
				&r, 1);
Y
Yang, Sheng 已提交
3250
		if (r < 0)
3251
			goto out_free_1;
Y
Yang, Sheng 已提交
3252 3253
	}

A
Avi Kivity 已提交
3254 3255
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
3256
		goto out_free_2;
A
Avi Kivity 已提交
3257 3258
	register_reboot_notifier(&kvm_reboot_notifier);

3259
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
3260 3261 3262
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
J
Joe Perches 已提交
3263
					   0, NULL);
3264 3265
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
3266
		goto out_free_3;
3267 3268
	}

3269 3270 3271 3272
	r = kvm_async_pf_init();
	if (r)
		goto out_free;

A
Avi Kivity 已提交
3273
	kvm_chardev_ops.owner = module;
3274 3275
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
3276 3277 3278

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
3279
		printk(KERN_ERR "kvm: misc device register failed\n");
3280
		goto out_unreg;
A
Avi Kivity 已提交
3281 3282
	}

3283 3284
	register_syscore_ops(&kvm_syscore_ops);

3285 3286 3287
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

3288 3289 3290 3291 3292
	r = kvm_init_debug();
	if (r) {
		printk(KERN_ERR "kvm: create debugfs files failed\n");
		goto out_undebugfs;
	}
3293

P
Paolo Bonzini 已提交
3294 3295 3296
	r = kvm_vfio_ops_init();
	WARN_ON(r);

3297
	return 0;
A
Avi Kivity 已提交
3298

3299 3300
out_undebugfs:
	unregister_syscore_ops(&kvm_syscore_ops);
3301
	misc_deregister(&kvm_dev);
3302 3303
out_unreg:
	kvm_async_pf_deinit();
A
Avi Kivity 已提交
3304
out_free:
3305
	kmem_cache_destroy(kvm_vcpu_cache);
3306
out_free_3:
A
Avi Kivity 已提交
3307
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
3308
	unregister_cpu_notifier(&kvm_cpu_notifier);
3309 3310
out_free_2:
out_free_1:
3311
	kvm_arch_hardware_unsetup();
3312 3313
out_free_0a:
	free_cpumask_var(cpus_hardware_enabled);
3314
out_free_0:
3315 3316
	kvm_irqfd_exit();
out_irqfd:
3317 3318
	kvm_arch_exit();
out_fail:
A
Avi Kivity 已提交
3319 3320
	return r;
}
3321
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
3322

3323
void kvm_exit(void)
A
Avi Kivity 已提交
3324
{
3325
	kvm_exit_debug();
A
Avi Kivity 已提交
3326
	misc_deregister(&kvm_dev);
3327
	kmem_cache_destroy(kvm_vcpu_cache);
3328
	kvm_async_pf_deinit();
3329
	unregister_syscore_ops(&kvm_syscore_ops);
A
Avi Kivity 已提交
3330
	unregister_reboot_notifier(&kvm_reboot_notifier);
3331
	unregister_cpu_notifier(&kvm_cpu_notifier);
3332
	on_each_cpu(hardware_disable_nolock, NULL, 1);
3333
	kvm_arch_hardware_unsetup();
3334
	kvm_arch_exit();
3335
	kvm_irqfd_exit();
3336
	free_cpumask_var(cpus_hardware_enabled);
3337
	kvm_vfio_ops_exit();
A
Avi Kivity 已提交
3338
}
3339
EXPORT_SYMBOL_GPL(kvm_exit);