kvm_main.c 96.3 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include <kvm/iodev.h>
A
Avi Kivity 已提交
20

21
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/syscore_ops.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
35
#include <linux/sched/signal.h>
36
#include <linux/sched/mm.h>
37
#include <linux/sched/stat.h>
38 39
#include <linux/cpumask.h>
#include <linux/smp.h>
40
#include <linux/anon_inodes.h>
41
#include <linux/profile.h>
42
#include <linux/kvm_para.h>
43
#include <linux/pagemap.h>
44
#include <linux/mman.h>
45
#include <linux/swap.h>
46
#include <linux/bitops.h>
47
#include <linux/spinlock.h>
48
#include <linux/compat.h>
49
#include <linux/srcu.h>
J
Joerg Roedel 已提交
50
#include <linux/hugetlb.h>
51
#include <linux/slab.h>
52 53
#include <linux/sort.h>
#include <linux/bsearch.h>
A
Avi Kivity 已提交
54

A
Avi Kivity 已提交
55 56
#include <asm/processor.h>
#include <asm/io.h>
57
#include <asm/ioctl.h>
58
#include <linux/uaccess.h>
59
#include <asm/pgtable.h>
A
Avi Kivity 已提交
60

61
#include "coalesced_mmio.h"
62
#include "async_pf.h"
P
Paolo Bonzini 已提交
63
#include "vfio.h"
64

65 66 67
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

68 69 70
/* Worst case buffer size needed for holding an integer. */
#define ITOA_MAX_LEN 12

A
Avi Kivity 已提交
71 72 73
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

74
/* Architectures should define their poll value according to the halt latency */
75
unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
76
module_param(halt_poll_ns, uint, 0644);
77
EXPORT_SYMBOL_GPL(halt_poll_ns);
78

W
Wanpeng Li 已提交
79
/* Default doubles per-vcpu halt_poll_ns. */
80
unsigned int halt_poll_ns_grow = 2;
81
module_param(halt_poll_ns_grow, uint, 0644);
82
EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
W
Wanpeng Li 已提交
83 84

/* Default resets per-vcpu halt_poll_ns . */
85
unsigned int halt_poll_ns_shrink;
86
module_param(halt_poll_ns_shrink, uint, 0644);
87
EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
W
Wanpeng Li 已提交
88

89 90 91
/*
 * Ordering of locks:
 *
92
 *	kvm->lock --> kvm->slots_lock --> kvm->irq_lock
93 94
 */

95
DEFINE_SPINLOCK(kvm_lock);
96
static DEFINE_RAW_SPINLOCK(kvm_count_lock);
97
LIST_HEAD(vm_list);
98

99
static cpumask_var_t cpus_hardware_enabled;
100
static int kvm_usage_count;
101
static atomic_t hardware_enable_failed;
102

103 104
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
105

106 107
static __read_mostly struct preempt_ops kvm_preempt_ops;

108
struct dentry *kvm_debugfs_dir;
109
EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
A
Avi Kivity 已提交
110

111 112 113
static int kvm_debugfs_num_entries;
static const struct file_operations *stat_fops_per_vm[];

A
Avi Kivity 已提交
114 115
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
116
#ifdef CONFIG_KVM_COMPAT
117 118 119
static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
				  unsigned long arg);
#endif
120 121
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
122

M
Marcelo Tosatti 已提交
123
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
124

D
Dan Williams 已提交
125
static void kvm_release_pfn_dirty(kvm_pfn_t pfn);
126
static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
M
Marcelo Tosatti 已提交
127

128
__visible bool kvm_rebooting;
129
EXPORT_SYMBOL_GPL(kvm_rebooting);
130

131 132
static bool largepages_enabled = true;

133 134 135 136 137 138
#define KVM_EVENT_CREATE_VM 0
#define KVM_EVENT_DESTROY_VM 1
static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
static unsigned long long kvm_createvm_count;
static unsigned long long kvm_active_vms;

D
Dan Williams 已提交
139
bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
B
Ben-Ami Yassour 已提交
140
{
141
	if (pfn_valid(pfn))
142
		return PageReserved(pfn_to_page(pfn));
B
Ben-Ami Yassour 已提交
143 144 145 146

	return true;
}

A
Avi Kivity 已提交
147 148 149
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
150
int vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
151
{
152 153
	int cpu;

154 155
	if (mutex_lock_killable(&vcpu->mutex))
		return -EINTR;
156 157
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
158
	kvm_arch_vcpu_load(vcpu, cpu);
159
	put_cpu();
160
	return 0;
A
Avi Kivity 已提交
161
}
162
EXPORT_SYMBOL_GPL(vcpu_load);
A
Avi Kivity 已提交
163

164
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
165
{
166
	preempt_disable();
167
	kvm_arch_vcpu_put(vcpu);
168 169
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
170 171
	mutex_unlock(&vcpu->mutex);
}
172
EXPORT_SYMBOL_GPL(vcpu_put);
A
Avi Kivity 已提交
173

174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
/* TODO: merge with kvm_arch_vcpu_should_kick */
static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
{
	int mode = kvm_vcpu_exiting_guest_mode(vcpu);

	/*
	 * We need to wait for the VCPU to reenable interrupts and get out of
	 * READING_SHADOW_PAGE_TABLES mode.
	 */
	if (req & KVM_REQUEST_WAIT)
		return mode != OUTSIDE_GUEST_MODE;

	/*
	 * Need to kick a running VCPU, but otherwise there is nothing to do.
	 */
	return mode == IN_GUEST_MODE;
}

192 193 194 195
static void ack_flush(void *_completed)
{
}

196 197 198 199 200 201 202 203 204 205 206 207
static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
{
	if (unlikely(!cpus))
		cpus = cpu_online_mask;

	if (cpumask_empty(cpus))
		return false;

	smp_call_function_many(cpus, ack_flush, NULL, wait);
	return true;
}

208
bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
209
{
210
	int i, cpu, me;
211
	cpumask_var_t cpus;
212
	bool called;
213 214
	struct kvm_vcpu *vcpu;

215
	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
216

217
	me = get_cpu();
218
	kvm_for_each_vcpu(i, vcpu, kvm) {
219
		kvm_make_request(req, vcpu);
220
		cpu = vcpu->cpu;
221

222 223
		if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
			continue;
224

225
		if (cpus != NULL && cpu != -1 && cpu != me &&
226
		    kvm_request_needs_ipi(vcpu, req))
227
			__cpumask_set_cpu(cpu, cpus);
228
	}
229
	called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
230
	put_cpu();
231
	free_cpumask_var(cpus);
232
	return called;
233 234
}

235
#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
236
void kvm_flush_remote_tlbs(struct kvm *kvm)
237
{
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
	/*
	 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
	 * kvm_make_all_cpus_request.
	 */
	long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);

	/*
	 * We want to publish modifications to the page tables before reading
	 * mode. Pairs with a memory barrier in arch-specific code.
	 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
	 * and smp_mb in walk_shadow_page_lockless_begin/end.
	 * - powerpc: smp_mb in kvmppc_prepare_to_enter.
	 *
	 * There is already an smp_mb__after_atomic() before
	 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
	 * barrier here.
	 */
255
	if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
256
		++kvm->stat.remote_tlb_flush;
257
	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
258
}
259
EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
260
#endif
261

262 263
void kvm_reload_remote_mmus(struct kvm *kvm)
{
264
	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
265
}
266

R
Rusty Russell 已提交
267 268 269 270 271 272 273 274 275
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
276
	vcpu->pid = NULL;
277
	init_swait_queue_head(&vcpu->wq);
278
	kvm_async_pf_vcpu_init(vcpu);
R
Rusty Russell 已提交
279

280 281 282
	vcpu->pre_pcpu = -1;
	INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);

R
Rusty Russell 已提交
283 284 285 286 287 288 289
	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

290 291
	kvm_vcpu_set_in_spin_loop(vcpu, false);
	kvm_vcpu_set_dy_eligible(vcpu, false);
292
	vcpu->preempted = false;
293

294
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
295
	if (r < 0)
296
		goto fail_free_run;
R
Rusty Russell 已提交
297 298 299 300 301
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
302
	return r;
R
Rusty Russell 已提交
303 304 305 306 307
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
308 309 310 311 312 313
	/*
	 * no need for rcu_read_lock as VCPU_RUN is the only place that
	 * will change the vcpu->pid pointer and on uninit all file
	 * descriptors are already gone.
	 */
	put_pid(rcu_dereference_protected(vcpu->pid, 1));
314
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
315 316 317 318
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

319 320 321 322 323 324 325 326 327 328 329
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
330
	int need_tlb_flush, idx;
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
350
	idx = srcu_read_lock(&kvm->srcu);
351
	spin_lock(&kvm->mmu_lock);
352

353
	kvm->mmu_notifier_seq++;
354
	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
355 356 357 358
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

359
	spin_unlock(&kvm->mmu_lock);
360 361 362

	kvm_arch_mmu_notifier_invalidate_page(kvm, address);

363
	srcu_read_unlock(&kvm->srcu, idx);
364 365
}

366 367 368 369 370 371
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
372
	int idx;
373

374
	idx = srcu_read_lock(&kvm->srcu);
375 376 377 378
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	kvm_set_spte_hva(kvm, address, pte);
	spin_unlock(&kvm->mmu_lock);
379
	srcu_read_unlock(&kvm->srcu, idx);
380 381
}

382 383 384 385 386 387
static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
388
	int need_tlb_flush = 0, idx;
389

390
	idx = srcu_read_lock(&kvm->srcu);
391 392 393 394 395 396 397
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
398
	need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
399
	need_tlb_flush |= kvm->tlbs_dirty;
400 401 402
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
403 404 405

	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
422
	smp_wmb();
423 424
	/*
	 * The above sequence increase must be visible before the
425 426
	 * below count decrease, which is ensured by the smp_wmb above
	 * in conjunction with the smp_rmb in mmu_notifier_retry().
427 428 429 430 431 432 433 434 435
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
A
Andres Lagar-Cavilla 已提交
436 437
					      unsigned long start,
					      unsigned long end)
438 439
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
440
	int young, idx;
441

442
	idx = srcu_read_lock(&kvm->srcu);
443 444
	spin_lock(&kvm->mmu_lock);

A
Andres Lagar-Cavilla 已提交
445
	young = kvm_age_hva(kvm, start, end);
446 447 448
	if (young)
		kvm_flush_remote_tlbs(kvm);

449 450 451
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

452 453 454
	return young;
}

455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long start,
					unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	/*
	 * Even though we do not flush TLB, this will still adversely
	 * affect performance on pre-Haswell Intel EPT, where there is
	 * no EPT Access Bit to clear so that we have to tear down EPT
	 * tables instead. If we find this unacceptable, we can always
	 * add a parameter to kvm_age_hva so that it effectively doesn't
	 * do anything on clear_young.
	 *
	 * Also note that currently we never issue secondary TLB flushes
	 * from clear_young, leaving this job up to the regular system
	 * cadence. If we find this inaccurate, we might come up with a
	 * more sophisticated heuristic later.
	 */
	young = kvm_age_hva(kvm, start, end);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

A
Andrea Arcangeli 已提交
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
				       struct mm_struct *mm,
				       unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	young = kvm_test_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

501 502 503 504
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
505 506 507
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
508
	kvm_arch_flush_shadow_all(kvm);
509
	srcu_read_unlock(&kvm->srcu, idx);
510 511
}

512 513 514 515 516
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
517
	.clear_young		= kvm_mmu_notifier_clear_young,
A
Andrea Arcangeli 已提交
518
	.test_young		= kvm_mmu_notifier_test_young,
519
	.change_pte		= kvm_mmu_notifier_change_pte,
520
	.release		= kvm_mmu_notifier_release,
521
};
522 523 524 525 526 527 528 529 530 531 532 533 534 535

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

536 537
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

538
static struct kvm_memslots *kvm_alloc_memslots(void)
539 540
{
	int i;
541
	struct kvm_memslots *slots;
542

M
Michal Hocko 已提交
543
	slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
544 545 546
	if (!slots)
		return NULL;

547
	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
548
		slots->id_to_index[i] = slots->memslots[i].id = i;
549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586

	return slots;
}

static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
	if (!memslot->dirty_bitmap)
		return;

	kvfree(memslot->dirty_bitmap);
	memslot->dirty_bitmap = NULL;
}

/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
			      struct kvm_memory_slot *dont)
{
	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		kvm_destroy_dirty_bitmap(free);

	kvm_arch_free_memslot(kvm, free, dont);

	free->npages = 0;
}

static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
{
	struct kvm_memory_slot *memslot;

	if (!slots)
		return;

	kvm_for_each_memslot(memslot, slots)
		kvm_free_memslot(kvm, memslot, NULL);

	kvfree(slots);
587 588
}

589 590 591 592 593 594 595 596 597
static void kvm_destroy_vm_debugfs(struct kvm *kvm)
{
	int i;

	if (!kvm->debugfs_dentry)
		return;

	debugfs_remove_recursive(kvm->debugfs_dentry);

598 599 600 601 602
	if (kvm->debugfs_stat_data) {
		for (i = 0; i < kvm_debugfs_num_entries; i++)
			kfree(kvm->debugfs_stat_data[i]);
		kfree(kvm->debugfs_stat_data);
	}
603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
}

static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
{
	char dir_name[ITOA_MAX_LEN * 2];
	struct kvm_stat_data *stat_data;
	struct kvm_stats_debugfs_item *p;

	if (!debugfs_initialized())
		return 0;

	snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
	kvm->debugfs_dentry = debugfs_create_dir(dir_name,
						 kvm_debugfs_dir);
	if (!kvm->debugfs_dentry)
		return -ENOMEM;

	kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
					 sizeof(*kvm->debugfs_stat_data),
					 GFP_KERNEL);
	if (!kvm->debugfs_stat_data)
		return -ENOMEM;

	for (p = debugfs_entries; p->name; p++) {
		stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL);
		if (!stat_data)
			return -ENOMEM;

		stat_data->kvm = kvm;
		stat_data->offset = p->offset;
		kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
634
		if (!debugfs_create_file(p->name, 0644,
635 636 637 638 639 640 641 642
					 kvm->debugfs_dentry,
					 stat_data,
					 stat_fops_per_vm[p->kind]))
			return -ENOMEM;
	}
	return 0;
}

643
static struct kvm *kvm_create_vm(unsigned long type)
A
Avi Kivity 已提交
644
{
645 646
	int r, i;
	struct kvm *kvm = kvm_arch_alloc_vm();
A
Avi Kivity 已提交
647

648 649 650
	if (!kvm)
		return ERR_PTR(-ENOMEM);

651
	spin_lock_init(&kvm->mmu_lock);
V
Vegard Nossum 已提交
652
	mmgrab(current->mm);
653 654 655 656 657
	kvm->mm = current->mm;
	kvm_eventfd_init(kvm);
	mutex_init(&kvm->lock);
	mutex_init(&kvm->irq_lock);
	mutex_init(&kvm->slots_lock);
658
	refcount_set(&kvm->users_count, 1);
659 660
	INIT_LIST_HEAD(&kvm->devices);

661
	r = kvm_arch_init_vm(kvm, type);
662
	if (r)
663
		goto out_err_no_disable;
664 665 666

	r = hardware_enable_all();
	if (r)
667
		goto out_err_no_disable;
668

669
#ifdef CONFIG_HAVE_KVM_IRQFD
670
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
671
#endif
A
Avi Kivity 已提交
672

673 674
	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);

675
	r = -ENOMEM;
676
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
677 678
		struct kvm_memslots *slots = kvm_alloc_memslots();
		if (!slots)
679
			goto out_err_no_srcu;
680 681 682 683 684 685 686
		/*
		 * Generations must be different for each address space.
		 * Init kvm generation close to the maximum to easily test the
		 * code of handling generation number wrap-around.
		 */
		slots->generation = i * 2 - 150;
		rcu_assign_pointer(kvm->memslots[i], slots);
687
	}
688

689
	if (init_srcu_struct(&kvm->srcu))
690 691 692
		goto out_err_no_srcu;
	if (init_srcu_struct(&kvm->irq_srcu))
		goto out_err_no_irq_srcu;
M
Marcelo Tosatti 已提交
693
	for (i = 0; i < KVM_NR_BUSES; i++) {
694 695
		rcu_assign_pointer(kvm->buses[i],
			kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL));
696
		if (!kvm->buses[i])
M
Marcelo Tosatti 已提交
697 698
			goto out_err;
	}
699

700 701 702 703
	r = kvm_init_mmu_notifier(kvm);
	if (r)
		goto out_err;

704
	spin_lock(&kvm_lock);
705
	list_add(&kvm->vm_list, &vm_list);
706
	spin_unlock(&kvm_lock);
707

708 709
	preempt_notifier_inc();

710
	return kvm;
711 712

out_err:
713 714
	cleanup_srcu_struct(&kvm->irq_srcu);
out_err_no_irq_srcu:
715
	cleanup_srcu_struct(&kvm->srcu);
716
out_err_no_srcu:
717
	hardware_disable_all();
718
out_err_no_disable:
M
Marcelo Tosatti 已提交
719
	for (i = 0; i < KVM_NR_BUSES; i++)
720
		kfree(rcu_access_pointer(kvm->buses[i]));
721
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
722 723
		kvm_free_memslots(kvm,
			rcu_dereference_protected(kvm->memslots[i], 1));
724
	kvm_arch_free_vm(kvm);
725
	mmdrop(current->mm);
726
	return ERR_PTR(r);
727 728
}

729 730
static void kvm_destroy_devices(struct kvm *kvm)
{
G
Geliang Tang 已提交
731
	struct kvm_device *dev, *tmp;
732

733 734 735 736 737
	/*
	 * We do not need to take the kvm->lock here, because nobody else
	 * has a reference to the struct kvm at this point and therefore
	 * cannot access the devices list anyhow.
	 */
G
Geliang Tang 已提交
738 739
	list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
		list_del(&dev->vm_node);
740 741 742 743
		dev->ops->destroy(dev);
	}
}

744 745
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
746
	int i;
747 748
	struct mm_struct *mm = kvm->mm;

749
	kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
750
	kvm_destroy_vm_debugfs(kvm);
751
	kvm_arch_sync_events(kvm);
752
	spin_lock(&kvm_lock);
753
	list_del(&kvm->vm_list);
754
	spin_unlock(&kvm_lock);
755
	kvm_free_irq_routing(kvm);
756
	for (i = 0; i < KVM_NR_BUSES; i++) {
757 758 759 760 761
		struct kvm_io_bus *bus;

		bus = rcu_dereference_protected(kvm->buses[i], 1);
		if (bus)
			kvm_io_bus_destroy(bus);
762 763
		kvm->buses[i] = NULL;
	}
764
	kvm_coalesced_mmio_free(kvm);
765 766
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
767
#else
768
	kvm_arch_flush_shadow_all(kvm);
769
#endif
770
	kvm_arch_destroy_vm(kvm);
771
	kvm_destroy_devices(kvm);
772
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
773 774
		kvm_free_memslots(kvm,
			rcu_dereference_protected(kvm->memslots[i], 1));
775
	cleanup_srcu_struct(&kvm->irq_srcu);
776 777
	cleanup_srcu_struct(&kvm->srcu);
	kvm_arch_free_vm(kvm);
778
	preempt_notifier_dec();
779
	hardware_disable_all();
780
	mmdrop(mm);
781 782
}

I
Izik Eidus 已提交
783 784
void kvm_get_kvm(struct kvm *kvm)
{
785
	refcount_inc(&kvm->users_count);
I
Izik Eidus 已提交
786 787 788 789 790
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
791
	if (refcount_dec_and_test(&kvm->users_count))
I
Izik Eidus 已提交
792 793 794 795 796
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


797 798 799 800
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
801 802
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
803
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
804 805 806
	return 0;
}

807 808
/*
 * Allocation size is twice as large as the actual dirty bitmap size.
809
 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
810
 */
811 812
static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
{
813
	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
814

M
Michal Hocko 已提交
815
	memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL);
816 817 818 819 820 821
	if (!memslot->dirty_bitmap)
		return -ENOMEM;

	return 0;
}

822
/*
823 824 825 826
 * Insert memslot and re-sort memslots based on their GFN,
 * so binary search could be used to lookup GFN.
 * Sorting algorithm takes advantage of having initially
 * sorted array and known changed memslot position.
827
 */
828 829
static void update_memslots(struct kvm_memslots *slots,
			    struct kvm_memory_slot *new)
830
{
831 832
	int id = new->id;
	int i = slots->id_to_index[id];
833
	struct kvm_memory_slot *mslots = slots->memslots;
834

835
	WARN_ON(mslots[i].id != id);
836
	if (!new->npages) {
837
		WARN_ON(!mslots[i].npages);
838 839 840 841 842 843
		if (mslots[i].npages)
			slots->used_slots--;
	} else {
		if (!mslots[i].npages)
			slots->used_slots++;
	}
844

845
	while (i < KVM_MEM_SLOTS_NUM - 1 &&
846 847 848
	       new->base_gfn <= mslots[i + 1].base_gfn) {
		if (!mslots[i + 1].npages)
			break;
849 850 851 852
		mslots[i] = mslots[i + 1];
		slots->id_to_index[mslots[i].id] = i;
		i++;
	}
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869

	/*
	 * The ">=" is needed when creating a slot with base_gfn == 0,
	 * so that it moves before all those with base_gfn == npages == 0.
	 *
	 * On the other hand, if new->npages is zero, the above loop has
	 * already left i pointing to the beginning of the empty part of
	 * mslots, and the ">=" would move the hole backwards in this
	 * case---which is wrong.  So skip the loop when deleting a slot.
	 */
	if (new->npages) {
		while (i > 0 &&
		       new->base_gfn >= mslots[i - 1].base_gfn) {
			mslots[i] = mslots[i - 1];
			slots->id_to_index[mslots[i].id] = i;
			i--;
		}
870 871
	} else
		WARN_ON_ONCE(i != slots->used_slots);
872

873 874
	mslots[i] = *new;
	slots->id_to_index[mslots[i].id] = i;
875 876
}

877
static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
878
{
X
Xiao Guangrong 已提交
879 880
	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;

881
#ifdef __KVM_HAVE_READONLY_MEM
X
Xiao Guangrong 已提交
882 883 884 885
	valid_flags |= KVM_MEM_READONLY;
#endif

	if (mem->flags & ~valid_flags)
886 887 888 889 890
		return -EINVAL;

	return 0;
}

891
static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
892
		int as_id, struct kvm_memslots *slots)
893
{
894
	struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
895

896 897 898 899 900 901 902
	/*
	 * Set the low bit in the generation, which disables SPTE caching
	 * until the end of synchronize_srcu_expedited.
	 */
	WARN_ON(old_memslots->generation & 1);
	slots->generation = old_memslots->generation + 1;

903
	rcu_assign_pointer(kvm->memslots[as_id], slots);
904
	synchronize_srcu_expedited(&kvm->srcu);
905

906 907 908 909
	/*
	 * Increment the new memslot generation a second time. This prevents
	 * vm exits that race with memslot updates from caching a memslot
	 * generation that will (potentially) be valid forever.
910 911 912 913 914 915
	 *
	 * Generations must be unique even across address spaces.  We do not need
	 * a global counter for that, instead the generation space is evenly split
	 * across address spaces.  For example, with two address spaces, address
	 * space 0 will use generations 0, 4, 8, ... while * address space 1 will
	 * use generations 2, 6, 10, 14, ...
916
	 */
917
	slots->generation += KVM_ADDRESS_SPACE_NUM * 2 - 1;
918

919
	kvm_arch_memslots_updated(kvm, slots);
920 921

	return old_memslots;
922 923
}

A
Avi Kivity 已提交
924 925 926 927 928
/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
929
 *
930
 * Must be called holding kvm->slots_lock for write.
A
Avi Kivity 已提交
931
 */
932
int __kvm_set_memory_region(struct kvm *kvm,
933
			    const struct kvm_userspace_memory_region *mem)
A
Avi Kivity 已提交
934
{
935
	int r;
A
Avi Kivity 已提交
936
	gfn_t base_gfn;
937
	unsigned long npages;
938
	struct kvm_memory_slot *slot;
A
Avi Kivity 已提交
939
	struct kvm_memory_slot old, new;
940
	struct kvm_memslots *slots = NULL, *old_memslots;
941
	int as_id, id;
942
	enum kvm_mr_change change;
A
Avi Kivity 已提交
943

944 945 946 947
	r = check_memory_region_flags(mem);
	if (r)
		goto out;

A
Avi Kivity 已提交
948
	r = -EINVAL;
949 950 951
	as_id = mem->slot >> 16;
	id = (u16)mem->slot;

A
Avi Kivity 已提交
952 953 954 955 956
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
957
	/* We can read the guest memory with __xxx_user() later on. */
958
	if ((id < KVM_USER_MEM_SLOTS) &&
959
	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
960 961 962
	     !access_ok(VERIFY_WRITE,
			(void __user *)(unsigned long)mem->userspace_addr,
			mem->memory_size)))
963
		goto out;
964
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
A
Avi Kivity 已提交
965 966 967 968
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

969
	slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
A
Avi Kivity 已提交
970 971 972
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

973 974 975
	if (npages > KVM_MEM_MAX_NR_PAGES)
		goto out;

976
	new = old = *slot;
A
Avi Kivity 已提交
977

978
	new.id = id;
A
Avi Kivity 已提交
979 980 981 982
	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

983 984 985 986 987
	if (npages) {
		if (!old.npages)
			change = KVM_MR_CREATE;
		else { /* Modify an existing slot. */
			if ((mem->userspace_addr != old.userspace_addr) ||
988 989
			    (npages != old.npages) ||
			    ((new.flags ^ old.flags) & KVM_MEM_READONLY))
990 991 992 993 994 995 996 997 998 999 1000
				goto out;

			if (base_gfn != old.base_gfn)
				change = KVM_MR_MOVE;
			else if (new.flags != old.flags)
				change = KVM_MR_FLAGS_ONLY;
			else { /* Nothing to change. */
				r = 0;
				goto out;
			}
		}
1001 1002 1003 1004
	} else {
		if (!old.npages)
			goto out;

1005
		change = KVM_MR_DELETE;
1006 1007 1008
		new.base_gfn = 0;
		new.flags = 0;
	}
A
Avi Kivity 已提交
1009

1010
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
1011 1012
		/* Check for overlaps */
		r = -EEXIST;
1013
		kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
1014
			if ((slot->id >= KVM_USER_MEM_SLOTS) ||
1015
			    (slot->id == id))
1016 1017 1018 1019 1020
				continue;
			if (!((base_gfn + npages <= slot->base_gfn) ||
			      (base_gfn >= slot->base_gfn + slot->npages)))
				goto out;
		}
A
Avi Kivity 已提交
1021 1022 1023 1024
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
1025
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
1026 1027

	r = -ENOMEM;
1028
	if (change == KVM_MR_CREATE) {
1029
		new.userspace_addr = mem->userspace_addr;
1030

1031
		if (kvm_arch_create_memslot(kvm, &new, npages))
1032
			goto out_free;
A
Avi Kivity 已提交
1033
	}
1034

A
Avi Kivity 已提交
1035 1036
	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
1037
		if (kvm_create_dirty_bitmap(&new) < 0)
1038
			goto out_free;
A
Avi Kivity 已提交
1039 1040
	}

M
Michal Hocko 已提交
1041
	slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
1042 1043
	if (!slots)
		goto out_free;
1044
	memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
1045

1046
	if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
1047
		slot = id_to_memslot(slots, id);
1048 1049
		slot->flags |= KVM_MEMSLOT_INVALID;

1050
		old_memslots = install_new_memslots(kvm, as_id, slots);
1051

1052 1053
		/* From this point no new shadow pages pointing to a deleted,
		 * or moved, memslot will be created.
1054 1055
		 *
		 * validation of sp->gfn happens in:
1056 1057
		 *	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
		 *	- kvm_is_visible_gfn (mmu_check_roots)
1058
		 */
1059
		kvm_arch_flush_shadow_memslot(kvm, slot);
1060 1061 1062 1063 1064 1065

		/*
		 * We can re-use the old_memslots from above, the only difference
		 * from the currently installed memslots is the invalid flag.  This
		 * will get overwritten by update_memslots anyway.
		 */
1066
		slots = old_memslots;
1067
	}
1068

1069
	r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
1070
	if (r)
1071
		goto out_slots;
1072

1073
	/* actual memory is freed via old in kvm_free_memslot below */
1074
	if (change == KVM_MR_DELETE) {
1075
		new.dirty_bitmap = NULL;
1076
		memset(&new.arch, 0, sizeof(new.arch));
1077 1078
	}

1079
	update_memslots(slots, &new);
1080
	old_memslots = install_new_memslots(kvm, as_id, slots);
1081

1082
	kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
1083

1084
	kvm_free_memslot(kvm, &old, &new);
1085
	kvfree(old_memslots);
A
Avi Kivity 已提交
1086 1087
	return 0;

1088
out_slots:
1089
	kvfree(slots);
1090
out_free:
1091
	kvm_free_memslot(kvm, &new, &old);
A
Avi Kivity 已提交
1092 1093
out:
	return r;
1094
}
1095 1096 1097
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
1098
			  const struct kvm_userspace_memory_region *mem)
1099 1100 1101
{
	int r;

1102
	mutex_lock(&kvm->slots_lock);
1103
	r = __kvm_set_memory_region(kvm, mem);
1104
	mutex_unlock(&kvm->slots_lock);
1105 1106
	return r;
}
1107 1108
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

1109 1110
static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
					  struct kvm_userspace_memory_region *mem)
1111
{
1112
	if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1113
		return -EINVAL;
1114

1115
	return kvm_set_memory_region(kvm, mem);
A
Avi Kivity 已提交
1116 1117
}

1118 1119
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
1120
{
1121
	struct kvm_memslots *slots;
A
Avi Kivity 已提交
1122
	struct kvm_memory_slot *memslot;
1123
	int i, as_id, id;
1124
	unsigned long n;
A
Avi Kivity 已提交
1125 1126
	unsigned long any = 0;

1127 1128 1129
	as_id = log->slot >> 16;
	id = (u16)log->slot;
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1130
		return -EINVAL;
A
Avi Kivity 已提交
1131

1132 1133
	slots = __kvm_memslots(kvm, as_id);
	memslot = id_to_memslot(slots, id);
A
Avi Kivity 已提交
1134
	if (!memslot->dirty_bitmap)
1135
		return -ENOENT;
A
Avi Kivity 已提交
1136

1137
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
1138

1139
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
1140 1141 1142
		any = memslot->dirty_bitmap[i];

	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1143
		return -EFAULT;
A
Avi Kivity 已提交
1144

1145 1146
	if (any)
		*is_dirty = 1;
1147
	return 0;
A
Avi Kivity 已提交
1148
}
1149
EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
A
Avi Kivity 已提交
1150

1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
/**
 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
 *	are dirty write protect them for next write.
 * @kvm:	pointer to kvm instance
 * @log:	slot id and address to which we copy the log
 * @is_dirty:	flag set if any page is dirty
 *
 * We need to keep it in mind that VCPU threads can write to the bitmap
 * concurrently. So, to avoid losing track of dirty pages we keep the
 * following order:
 *
 *    1. Take a snapshot of the bit and clear it if needed.
 *    2. Write protect the corresponding page.
 *    3. Copy the snapshot to the userspace.
 *    4. Upon return caller flushes TLB's if needed.
 *
 * Between 2 and 4, the guest may write to the page using the remaining TLB
 * entry.  This is not a problem because the page is reported dirty using
 * the snapshot taken before and step 4 ensures that writes done after
 * exiting to userspace will be logged for the next call.
 *
 */
int kvm_get_dirty_log_protect(struct kvm *kvm,
			struct kvm_dirty_log *log, bool *is_dirty)
{
1177
	struct kvm_memslots *slots;
1178
	struct kvm_memory_slot *memslot;
1179
	int i, as_id, id;
1180 1181 1182 1183
	unsigned long n;
	unsigned long *dirty_bitmap;
	unsigned long *dirty_bitmap_buffer;

1184 1185 1186
	as_id = log->slot >> 16;
	id = (u16)log->slot;
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1187
		return -EINVAL;
1188

1189 1190
	slots = __kvm_memslots(kvm, as_id);
	memslot = id_to_memslot(slots, id);
1191 1192 1193

	dirty_bitmap = memslot->dirty_bitmap;
	if (!dirty_bitmap)
1194
		return -ENOENT;
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214

	n = kvm_dirty_bitmap_bytes(memslot);

	dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
	memset(dirty_bitmap_buffer, 0, n);

	spin_lock(&kvm->mmu_lock);
	*is_dirty = false;
	for (i = 0; i < n / sizeof(long); i++) {
		unsigned long mask;
		gfn_t offset;

		if (!dirty_bitmap[i])
			continue;

		*is_dirty = true;

		mask = xchg(&dirty_bitmap[i], 0);
		dirty_bitmap_buffer[i] = mask;

1215 1216 1217 1218 1219
		if (mask) {
			offset = i * BITS_PER_LONG;
			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
								offset, mask);
		}
1220 1221 1222 1223
	}

	spin_unlock(&kvm->mmu_lock);
	if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
1224 1225
		return -EFAULT;
	return 0;
1226 1227 1228 1229
}
EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
#endif

1230 1231 1232 1233 1234
bool kvm_largepages_enabled(void)
{
	return largepages_enabled;
}

1235 1236 1237 1238 1239 1240
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

1241 1242 1243 1244
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
A
Avi Kivity 已提交
1245
EXPORT_SYMBOL_GPL(gfn_to_memslot);
A
Avi Kivity 已提交
1246

1247 1248 1249 1250 1251
struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
}

1252
bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1253
{
1254
	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1255

1256
	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1257
	      memslot->flags & KVM_MEMSLOT_INVALID)
1258
		return false;
1259

1260
	return true;
1261 1262 1263
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

J
Joerg Roedel 已提交
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

X
Xiao Guangrong 已提交
1288 1289 1290 1291 1292 1293 1294
static bool memslot_is_readonly(struct kvm_memory_slot *slot)
{
	return slot->flags & KVM_MEM_READONLY;
}

static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				       gfn_t *nr_pages, bool write)
I
Izik Eidus 已提交
1295
{
1296
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
X
Xiao Guangrong 已提交
1297
		return KVM_HVA_ERR_BAD;
1298

X
Xiao Guangrong 已提交
1299 1300
	if (memslot_is_readonly(slot) && write)
		return KVM_HVA_ERR_RO_BAD;
1301 1302 1303 1304

	if (nr_pages)
		*nr_pages = slot->npages - (gfn - slot->base_gfn);

X
Xiao Guangrong 已提交
1305
	return __gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
1306
}
1307

X
Xiao Guangrong 已提交
1308 1309 1310 1311
static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				     gfn_t *nr_pages)
{
	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
I
Izik Eidus 已提交
1312
}
1313

X
Xiao Guangrong 已提交
1314
unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1315
					gfn_t gfn)
X
Xiao Guangrong 已提交
1316 1317 1318 1319 1320
{
	return gfn_to_hva_many(slot, gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);

1321 1322
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
1323
	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1324
}
1325
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1326

1327 1328 1329 1330 1331 1332
unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);

1333
/*
1334 1335
 * If writable is set to false, the hva returned by this function is only
 * allowed to be read.
1336
 */
1337 1338
unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
				      gfn_t gfn, bool *writable)
1339
{
1340 1341 1342
	unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);

	if (!kvm_is_error_hva(hva) && writable)
1343 1344
		*writable = !memslot_is_readonly(slot);

1345
	return hva;
1346 1347
}

1348 1349 1350 1351 1352 1353 1354
unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

1355 1356 1357 1358 1359 1360 1361
unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
{
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

1362 1363
static int get_user_page_nowait(unsigned long start, int write,
		struct page **page)
1364
{
L
Lorenzo Stoakes 已提交
1365
	int flags = FOLL_NOWAIT | FOLL_HWPOISON;
1366 1367 1368 1369

	if (write)
		flags |= FOLL_WRITE;

L
Lorenzo Stoakes 已提交
1370
	return get_user_pages(start, 1, flags, page, NULL);
1371 1372
}

1373 1374
static inline int check_user_page_hwpoison(unsigned long addr)
{
L
Lorenzo Stoakes 已提交
1375
	int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
1376

L
Lorenzo Stoakes 已提交
1377
	rc = get_user_pages(addr, 1, flags, NULL, NULL);
1378 1379 1380
	return rc == -EHWPOISON;
}

X
Xiao Guangrong 已提交
1381 1382 1383 1384 1385
/*
 * The atomic path to get the writable pfn which will be stored in @pfn,
 * true indicates success, otherwise false is returned.
 */
static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
D
Dan Williams 已提交
1386
			    bool write_fault, bool *writable, kvm_pfn_t *pfn)
A
Avi Kivity 已提交
1387
{
1388
	struct page *page[1];
X
Xiao Guangrong 已提交
1389
	int npages;
A
Avi Kivity 已提交
1390

X
Xiao Guangrong 已提交
1391 1392
	if (!(async || atomic))
		return false;
1393

1394 1395 1396 1397 1398 1399 1400
	/*
	 * Fast pin a writable pfn only if it is a write fault request
	 * or the caller allows to map a writable pfn for a read fault
	 * request.
	 */
	if (!(write_fault || writable))
		return false;
1401

X
Xiao Guangrong 已提交
1402 1403 1404
	npages = __get_user_pages_fast(addr, 1, 1, page);
	if (npages == 1) {
		*pfn = page_to_pfn(page[0]);
1405

X
Xiao Guangrong 已提交
1406 1407 1408 1409
		if (writable)
			*writable = true;
		return true;
	}
1410

X
Xiao Guangrong 已提交
1411 1412
	return false;
}
1413

X
Xiao Guangrong 已提交
1414 1415 1416 1417 1418
/*
 * The slow path to get the pfn of the specified host virtual address,
 * 1 indicates success, -errno is returned if error is detected.
 */
static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
D
Dan Williams 已提交
1419
			   bool *writable, kvm_pfn_t *pfn)
X
Xiao Guangrong 已提交
1420 1421 1422
{
	struct page *page[1];
	int npages = 0;
1423

X
Xiao Guangrong 已提交
1424 1425 1426 1427 1428 1429 1430
	might_sleep();

	if (writable)
		*writable = write_fault;

	if (async) {
		down_read(&current->mm->mmap_sem);
1431
		npages = get_user_page_nowait(addr, write_fault, page);
X
Xiao Guangrong 已提交
1432
		up_read(&current->mm->mmap_sem);
1433
	} else {
1434
		unsigned int flags = FOLL_HWPOISON;
1435 1436 1437 1438

		if (write_fault)
			flags |= FOLL_WRITE;

1439
		npages = get_user_pages_unlocked(addr, 1, page, flags);
1440
	}
X
Xiao Guangrong 已提交
1441 1442 1443 1444
	if (npages != 1)
		return npages;

	/* map read fault as writable if possible */
1445
	if (unlikely(!write_fault) && writable) {
X
Xiao Guangrong 已提交
1446 1447 1448 1449 1450 1451 1452
		struct page *wpage[1];

		npages = __get_user_pages_fast(addr, 1, 1, wpage);
		if (npages == 1) {
			*writable = true;
			put_page(page[0]);
			page[0] = wpage[0];
1453
		}
X
Xiao Guangrong 已提交
1454 1455

		npages = 1;
1456
	}
X
Xiao Guangrong 已提交
1457 1458 1459
	*pfn = page_to_pfn(page[0]);
	return npages;
}
I
Izik Eidus 已提交
1460

X
Xiao Guangrong 已提交
1461 1462 1463 1464
static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
{
	if (unlikely(!(vma->vm_flags & VM_READ)))
		return false;
1465

X
Xiao Guangrong 已提交
1466 1467
	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
		return false;
1468

X
Xiao Guangrong 已提交
1469 1470
	return true;
}
1471

1472 1473 1474 1475
static int hva_to_pfn_remapped(struct vm_area_struct *vma,
			       unsigned long addr, bool *async,
			       bool write_fault, kvm_pfn_t *p_pfn)
{
1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514
	unsigned long pfn;
	int r;

	r = follow_pfn(vma, addr, &pfn);
	if (r) {
		/*
		 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
		 * not call the fault handler, so do it here.
		 */
		bool unlocked = false;
		r = fixup_user_fault(current, current->mm, addr,
				     (write_fault ? FAULT_FLAG_WRITE : 0),
				     &unlocked);
		if (unlocked)
			return -EAGAIN;
		if (r)
			return r;

		r = follow_pfn(vma, addr, &pfn);
		if (r)
			return r;

	}


	/*
	 * Get a reference here because callers of *hva_to_pfn* and
	 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
	 * returned pfn.  This is only needed if the VMA has VM_MIXEDMAP
	 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
	 * simply do nothing for reserved pfns.
	 *
	 * Whoever called remap_pfn_range is also going to call e.g.
	 * unmap_mapping_range before the underlying pages are freed,
	 * causing a call to our MMU notifier.
	 */ 
	kvm_get_pfn(pfn);

	*p_pfn = pfn;
1515 1516 1517
	return 0;
}

1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531
/*
 * Pin guest page in memory and return its pfn.
 * @addr: host virtual address which maps memory to the guest
 * @atomic: whether this function can sleep
 * @async: whether this function need to wait IO complete if the
 *         host page is not in the memory
 * @write_fault: whether we should get a writable host page
 * @writable: whether it allows to map a writable host page for !@write_fault
 *
 * The function will map a writable host page for these two cases:
 * 1): @write_fault = true
 * 2): @write_fault = false && @writable, @writable will tell the caller
 *     whether the mapping is writable.
 */
D
Dan Williams 已提交
1532
static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
X
Xiao Guangrong 已提交
1533 1534 1535
			bool write_fault, bool *writable)
{
	struct vm_area_struct *vma;
D
Dan Williams 已提交
1536
	kvm_pfn_t pfn = 0;
1537
	int npages, r;
1538

X
Xiao Guangrong 已提交
1539 1540
	/* we can do it either atomically or asynchronously, not both */
	BUG_ON(atomic && async);
1541

X
Xiao Guangrong 已提交
1542 1543 1544 1545 1546 1547 1548 1549 1550
	if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
		return pfn;

	if (atomic)
		return KVM_PFN_ERR_FAULT;

	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
	if (npages == 1)
		return pfn;
1551

X
Xiao Guangrong 已提交
1552 1553 1554 1555 1556 1557 1558
	down_read(&current->mm->mmap_sem);
	if (npages == -EHWPOISON ||
	      (!async && check_user_page_hwpoison(addr))) {
		pfn = KVM_PFN_ERR_HWPOISON;
		goto exit;
	}

1559
retry:
X
Xiao Guangrong 已提交
1560 1561 1562 1563
	vma = find_vma_intersection(current->mm, addr, addr + 1);

	if (vma == NULL)
		pfn = KVM_PFN_ERR_FAULT;
1564 1565
	else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
		r = hva_to_pfn_remapped(vma, addr, async, write_fault, &pfn);
1566 1567
		if (r == -EAGAIN)
			goto retry;
1568 1569
		if (r < 0)
			pfn = KVM_PFN_ERR_FAULT;
X
Xiao Guangrong 已提交
1570
	} else {
X
Xiao Guangrong 已提交
1571
		if (async && vma_is_valid(vma, write_fault))
X
Xiao Guangrong 已提交
1572 1573 1574 1575 1576
			*async = true;
		pfn = KVM_PFN_ERR_FAULT;
	}
exit:
	up_read(&current->mm->mmap_sem);
1577
	return pfn;
1578 1579
}

D
Dan Williams 已提交
1580 1581 1582
kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
			       bool atomic, bool *async, bool write_fault,
			       bool *writable)
1583
{
X
Xiao Guangrong 已提交
1584 1585
	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);

1586 1587 1588
	if (addr == KVM_HVA_ERR_RO_BAD) {
		if (writable)
			*writable = false;
X
Xiao Guangrong 已提交
1589
		return KVM_PFN_ERR_RO_FAULT;
1590
	}
X
Xiao Guangrong 已提交
1591

1592 1593 1594
	if (kvm_is_error_hva(addr)) {
		if (writable)
			*writable = false;
1595
		return KVM_PFN_NOSLOT;
1596
	}
X
Xiao Guangrong 已提交
1597 1598 1599 1600 1601 1602 1603 1604 1605

	/* Do not map writable pfn in the readonly memslot. */
	if (writable && memslot_is_readonly(slot)) {
		*writable = false;
		writable = NULL;
	}

	return hva_to_pfn(addr, atomic, async, write_fault,
			  writable);
1606
}
1607
EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1608

D
Dan Williams 已提交
1609
kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1610 1611
		      bool *writable)
{
P
Paolo Bonzini 已提交
1612 1613
	return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
				    write_fault, writable);
1614 1615 1616
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);

D
Dan Williams 已提交
1617
kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1618
{
X
Xiao Guangrong 已提交
1619
	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1620
}
P
Paolo Bonzini 已提交
1621
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1622

D
Dan Williams 已提交
1623
kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1624
{
X
Xiao Guangrong 已提交
1625
	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1626
}
1627
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1628

D
Dan Williams 已提交
1629
kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
P
Paolo Bonzini 已提交
1630 1631 1632 1633 1634
{
	return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);

D
Dan Williams 已提交
1635
kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1636 1637 1638 1639 1640
{
	return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);

D
Dan Williams 已提交
1641
kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
P
Paolo Bonzini 已提交
1642 1643 1644 1645 1646
{
	return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn);

D
Dan Williams 已提交
1647
kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1648 1649 1650 1651 1652
{
	return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);

1653 1654
int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
			    struct page **pages, int nr_pages)
1655 1656 1657 1658
{
	unsigned long addr;
	gfn_t entry;

1659
	addr = gfn_to_hva_many(slot, gfn, &entry);
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
	if (kvm_is_error_hva(addr))
		return -1;

	if (entry < nr_pages)
		return 0;

	return __get_user_pages_fast(addr, nr_pages, 1, pages);
}
EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);

D
Dan Williams 已提交
1670
static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
X
Xiao Guangrong 已提交
1671
{
1672
	if (is_error_noslot_pfn(pfn))
1673
		return KVM_ERR_PTR_BAD_PAGE;
X
Xiao Guangrong 已提交
1674

1675
	if (kvm_is_reserved_pfn(pfn)) {
1676
		WARN_ON(1);
1677
		return KVM_ERR_PTR_BAD_PAGE;
1678
	}
X
Xiao Guangrong 已提交
1679 1680 1681 1682

	return pfn_to_page(pfn);
}

1683 1684
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
D
Dan Williams 已提交
1685
	kvm_pfn_t pfn;
1686 1687 1688

	pfn = gfn_to_pfn(kvm, gfn);

X
Xiao Guangrong 已提交
1689
	return kvm_pfn_to_page(pfn);
A
Avi Kivity 已提交
1690 1691 1692
}
EXPORT_SYMBOL_GPL(gfn_to_page);

1693 1694
struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
{
D
Dan Williams 已提交
1695
	kvm_pfn_t pfn;
1696 1697 1698 1699 1700 1701 1702

	pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);

	return kvm_pfn_to_page(pfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);

1703 1704
void kvm_release_page_clean(struct page *page)
{
1705 1706
	WARN_ON(is_error_page(page));

1707
	kvm_release_pfn_clean(page_to_pfn(page));
1708 1709 1710
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

D
Dan Williams 已提交
1711
void kvm_release_pfn_clean(kvm_pfn_t pfn)
1712
{
1713
	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1714
		put_page(pfn_to_page(pfn));
1715 1716 1717
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1718
void kvm_release_page_dirty(struct page *page)
1719
{
X
Xiao Guangrong 已提交
1720 1721
	WARN_ON(is_error_page(page));

1722 1723 1724 1725
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

D
Dan Williams 已提交
1726
static void kvm_release_pfn_dirty(kvm_pfn_t pfn)
1727 1728 1729 1730 1731
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}

D
Dan Williams 已提交
1732
void kvm_set_pfn_dirty(kvm_pfn_t pfn)
1733
{
1734
	if (!kvm_is_reserved_pfn(pfn)) {
1735
		struct page *page = pfn_to_page(pfn);
1736

1737 1738 1739
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1740
}
1741 1742
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

D
Dan Williams 已提交
1743
void kvm_set_pfn_accessed(kvm_pfn_t pfn)
1744
{
1745
	if (!kvm_is_reserved_pfn(pfn))
1746
		mark_page_accessed(pfn_to_page(pfn));
1747 1748 1749
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

D
Dan Williams 已提交
1750
void kvm_get_pfn(kvm_pfn_t pfn)
1751
{
1752
	if (!kvm_is_reserved_pfn(pfn))
1753
		get_page(pfn_to_page(pfn));
1754 1755
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1756

1757 1758 1759 1760 1761 1762 1763 1764
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

1765 1766
static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
				 void *data, int offset, int len)
1767
{
1768 1769
	int r;
	unsigned long addr;
1770

1771
	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1772 1773
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1774
	r = __copy_from_user(data, (void __user *)addr + offset, len);
1775
	if (r)
1776 1777 1778
		return -EFAULT;
	return 0;
}
1779 1780 1781 1782 1783 1784 1785 1786

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return __kvm_read_guest_page(slot, gfn, data, offset, len);
}
1787 1788
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

1789 1790 1791 1792 1793 1794 1795 1796 1797
int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
			     int offset, int len)
{
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	return __kvm_read_guest_page(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);

1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1818
int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
1819 1820
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
1821
	int seg;
1822
	int offset = offset_in_page(gpa);
1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
1837

1838 1839 1840 1841 1842 1843 1844
static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
			           void *data, int offset, unsigned long len)
{
	int r;
	unsigned long addr;

	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1845 1846
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1847
	pagefault_disable();
1848
	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1849
	pagefault_enable();
1850 1851 1852 1853 1854
	if (r)
		return -EFAULT;
	return 0;
}

1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
	int offset = offset_in_page(gpa);

	return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);

int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
			       void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
	int offset = offset_in_page(gpa);

	return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);

static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
			          const void *data, int offset, int len)
1879
{
1880 1881
	int r;
	unsigned long addr;
1882

1883
	addr = gfn_to_hva_memslot(memslot, gfn);
1884 1885
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1886
	r = __copy_to_user((void __user *)addr + offset, data, len);
1887
	if (r)
1888
		return -EFAULT;
1889
	mark_page_dirty_in_slot(memslot, gfn);
1890 1891
	return 0;
}
1892 1893 1894 1895 1896 1897 1898 1899

int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
			 const void *data, int offset, int len)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return __kvm_write_guest_page(slot, gfn, data, offset, len);
}
1900 1901
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

1902 1903 1904 1905 1906 1907 1908 1909 1910
int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
			      const void *data, int offset, int len)
{
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	return __kvm_write_guest_page(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);

1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929
int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
1930
EXPORT_SYMBOL_GPL(kvm_write_guest);
1931

1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952
int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
		         unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);

1953 1954 1955
static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
				       struct gfn_to_hva_cache *ghc,
				       gpa_t gpa, unsigned long len)
1956 1957
{
	int offset = offset_in_page(gpa);
1958 1959 1960 1961
	gfn_t start_gfn = gpa >> PAGE_SHIFT;
	gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
	gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
	gfn_t nr_pages_avail;
1962 1963 1964

	ghc->gpa = gpa;
	ghc->generation = slots->generation;
1965
	ghc->len = len;
1966
	ghc->memslot = __gfn_to_memslot(slots, start_gfn);
1967 1968
	ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL);
	if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) {
1969
		ghc->hva += offset;
1970 1971 1972 1973 1974 1975
	} else {
		/*
		 * If the requested region crosses two memslots, we still
		 * verify that the entire region is valid here.
		 */
		while (start_gfn <= end_gfn) {
1976
			ghc->memslot = __gfn_to_memslot(slots, start_gfn);
1977 1978 1979 1980 1981 1982 1983 1984 1985
			ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
						   &nr_pages_avail);
			if (kvm_is_error_hva(ghc->hva))
				return -EFAULT;
			start_gfn += nr_pages_avail;
		}
		/* Use the slow path for cross page reads and writes. */
		ghc->memslot = NULL;
	}
1986 1987
	return 0;
}
1988

1989
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1990 1991
			      gpa_t gpa, unsigned long len)
{
1992
	struct kvm_memslots *slots = kvm_memslots(kvm);
1993 1994
	return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
}
1995
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1996

1997 1998
int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, int offset, unsigned long len)
1999
{
2000
	struct kvm_memslots *slots = kvm_memslots(kvm);
2001
	int r;
2002
	gpa_t gpa = ghc->gpa + offset;
2003

2004
	BUG_ON(len + offset > ghc->len);
2005

2006
	if (slots->generation != ghc->generation)
2007
		__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
2008 2009

	if (unlikely(!ghc->memslot))
2010
		return kvm_write_guest(kvm, gpa, data, len);
2011 2012 2013 2014

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

2015
	r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
2016 2017
	if (r)
		return -EFAULT;
2018
	mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
2019 2020 2021

	return 0;
}
2022
EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
2023

2024 2025
int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
2026
{
2027
	return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
2028
}
2029
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
2030

2031 2032
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
2033
{
2034
	struct kvm_memslots *slots = kvm_memslots(kvm);
2035 2036
	int r;

2037 2038
	BUG_ON(len > ghc->len);

2039
	if (slots->generation != ghc->generation)
2040
		__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
2041 2042

	if (unlikely(!ghc->memslot))
2043
		return kvm_read_guest(kvm, ghc->gpa, data, len);
2044 2045 2046 2047 2048 2049 2050 2051 2052 2053

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

	r = __copy_from_user(data, (void __user *)ghc->hva, len);
	if (r)
		return -EFAULT;

	return 0;
}
2054
EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
2055

2056 2057
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
2058 2059 2060
	const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));

	return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
2061 2062 2063 2064 2065 2066 2067 2068 2069 2070
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

2071
	while ((seg = next_segment(len, offset)) != 0) {
2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

2083
static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
2084
				    gfn_t gfn)
A
Avi Kivity 已提交
2085
{
R
Rusty Russell 已提交
2086 2087
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
2088

2089
		set_bit_le(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
2090 2091 2092
	}
}

2093 2094 2095 2096 2097
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = gfn_to_memslot(kvm, gfn);
2098
	mark_page_dirty_in_slot(memslot, gfn);
2099
}
2100
EXPORT_SYMBOL_GPL(mark_page_dirty);
2101

2102 2103 2104 2105 2106 2107 2108 2109 2110
void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
	mark_page_dirty_in_slot(memslot, gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);

W
Wanpeng Li 已提交
2111 2112
static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
{
2113
	unsigned int old, val, grow;
W
Wanpeng Li 已提交
2114

2115
	old = val = vcpu->halt_poll_ns;
2116
	grow = READ_ONCE(halt_poll_ns_grow);
W
Wanpeng Li 已提交
2117
	/* 10us base */
2118
	if (val == 0 && grow)
W
Wanpeng Li 已提交
2119 2120
		val = 10000;
	else
2121
		val *= grow;
W
Wanpeng Li 已提交
2122

2123 2124 2125
	if (val > halt_poll_ns)
		val = halt_poll_ns;

W
Wanpeng Li 已提交
2126
	vcpu->halt_poll_ns = val;
2127
	trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
W
Wanpeng Li 已提交
2128 2129 2130 2131
}

static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
{
2132
	unsigned int old, val, shrink;
W
Wanpeng Li 已提交
2133

2134
	old = val = vcpu->halt_poll_ns;
2135 2136
	shrink = READ_ONCE(halt_poll_ns_shrink);
	if (shrink == 0)
W
Wanpeng Li 已提交
2137 2138
		val = 0;
	else
2139
		val /= shrink;
W
Wanpeng Li 已提交
2140 2141

	vcpu->halt_poll_ns = val;
2142
	trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
W
Wanpeng Li 已提交
2143 2144
}

2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158
static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
{
	if (kvm_arch_vcpu_runnable(vcpu)) {
		kvm_make_request(KVM_REQ_UNHALT, vcpu);
		return -EINTR;
	}
	if (kvm_cpu_has_pending_timer(vcpu))
		return -EINTR;
	if (signal_pending(current))
		return -EINTR;

	return 0;
}

E
Eddie Dong 已提交
2159 2160 2161
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
2162
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2163
{
2164
	ktime_t start, cur;
2165
	DECLARE_SWAITQUEUE(wait);
2166
	bool waited = false;
W
Wanpeng Li 已提交
2167
	u64 block_ns;
2168 2169

	start = cur = ktime_get();
W
Wanpeng Li 已提交
2170 2171
	if (vcpu->halt_poll_ns) {
		ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2172

2173
		++vcpu->stat.halt_attempted_poll;
2174 2175 2176 2177 2178 2179 2180
		do {
			/*
			 * This sets KVM_REQ_UNHALT if an interrupt
			 * arrives.
			 */
			if (kvm_vcpu_check_block(vcpu) < 0) {
				++vcpu->stat.halt_successful_poll;
2181 2182
				if (!vcpu_valid_wakeup(vcpu))
					++vcpu->stat.halt_poll_invalid;
2183 2184 2185 2186 2187
				goto out;
			}
			cur = ktime_get();
		} while (single_task_running() && ktime_before(cur, stop));
	}
2188

2189 2190
	kvm_arch_vcpu_blocking(vcpu);

2191
	for (;;) {
2192
		prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2193

2194
		if (kvm_vcpu_check_block(vcpu) < 0)
2195 2196
			break;

2197
		waited = true;
E
Eddie Dong 已提交
2198 2199
		schedule();
	}
2200

2201
	finish_swait(&vcpu->wq, &wait);
2202 2203
	cur = ktime_get();

2204
	kvm_arch_vcpu_unblocking(vcpu);
2205
out:
W
Wanpeng Li 已提交
2206 2207
	block_ns = ktime_to_ns(cur) - ktime_to_ns(start);

2208 2209 2210
	if (!vcpu_valid_wakeup(vcpu))
		shrink_halt_poll_ns(vcpu);
	else if (halt_poll_ns) {
W
Wanpeng Li 已提交
2211 2212 2213
		if (block_ns <= vcpu->halt_poll_ns)
			;
		/* we had a long block, shrink polling */
2214
		else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
W
Wanpeng Li 已提交
2215 2216 2217 2218 2219
			shrink_halt_poll_ns(vcpu);
		/* we had a short halt and our poll time is too small */
		else if (vcpu->halt_poll_ns < halt_poll_ns &&
			block_ns < halt_poll_ns)
			grow_halt_poll_ns(vcpu);
2220 2221
	} else
		vcpu->halt_poll_ns = 0;
W
Wanpeng Li 已提交
2222

2223 2224
	trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
	kvm_arch_vcpu_block_finish(vcpu);
E
Eddie Dong 已提交
2225
}
2226
EXPORT_SYMBOL_GPL(kvm_vcpu_block);
E
Eddie Dong 已提交
2227

2228
bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2229
{
2230
	struct swait_queue_head *wqp;
2231 2232

	wqp = kvm_arch_vcpu_wq(vcpu);
2233 2234
	if (swait_active(wqp)) {
		swake_up(wqp);
2235
		++vcpu->stat.halt_wakeup;
2236
		return true;
2237 2238
	}

2239
	return false;
2240 2241 2242
}
EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);

2243
#ifndef CONFIG_S390
2244 2245 2246 2247 2248 2249 2250 2251
/*
 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
 */
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
	int me;
	int cpu = vcpu->cpu;

2252 2253 2254
	if (kvm_vcpu_wake_up(vcpu))
		return;

2255 2256 2257 2258 2259 2260
	me = get_cpu();
	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
		if (kvm_arch_vcpu_should_kick(vcpu))
			smp_send_reschedule(cpu);
	put_cpu();
}
2261
EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2262
#endif /* !CONFIG_S390 */
2263

2264
int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2265 2266 2267
{
	struct pid *pid;
	struct task_struct *task = NULL;
2268
	int ret = 0;
2269 2270 2271 2272

	rcu_read_lock();
	pid = rcu_dereference(target->pid);
	if (pid)
2273
		task = get_pid_task(pid, PIDTYPE_PID);
2274 2275
	rcu_read_unlock();
	if (!task)
2276 2277
		return ret;
	ret = yield_to(task, 1);
2278
	put_task_struct(task);
2279 2280

	return ret;
2281 2282 2283
}
EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);

2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305
/*
 * Helper that checks whether a VCPU is eligible for directed yield.
 * Most eligible candidate to yield is decided by following heuristics:
 *
 *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
 *  (preempted lock holder), indicated by @in_spin_loop.
 *  Set at the beiginning and cleared at the end of interception/PLE handler.
 *
 *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
 *  chance last time (mostly it has become eligible now since we have probably
 *  yielded to lockholder in last iteration. This is done by toggling
 *  @dy_eligible each time a VCPU checked for eligibility.)
 *
 *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
 *  to preempted lock-holder could result in wrong VCPU selection and CPU
 *  burning. Giving priority for a potential lock-holder increases lock
 *  progress.
 *
 *  Since algorithm is based on heuristics, accessing another VCPU data without
 *  locking does not harm. It may result in trying to yield to  same VCPU, fail
 *  and continue with next VCPU and so on.
 */
2306
static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2307
{
2308
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2309 2310 2311
	bool eligible;

	eligible = !vcpu->spin_loop.in_spin_loop ||
2312
		    vcpu->spin_loop.dy_eligible;
2313 2314 2315 2316 2317

	if (vcpu->spin_loop.in_spin_loop)
		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);

	return eligible;
2318 2319
#else
	return true;
2320
#endif
2321
}
2322

2323
void kvm_vcpu_on_spin(struct kvm_vcpu *me)
Z
Zhai, Edwin 已提交
2324
{
2325 2326 2327 2328
	struct kvm *kvm = me->kvm;
	struct kvm_vcpu *vcpu;
	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
	int yielded = 0;
2329
	int try = 3;
2330 2331
	int pass;
	int i;
Z
Zhai, Edwin 已提交
2332

2333
	kvm_vcpu_set_in_spin_loop(me, true);
2334 2335 2336 2337 2338 2339 2340
	/*
	 * We boost the priority of a VCPU that is runnable but not
	 * currently running, because it got preempted by something
	 * else and called schedule in __vcpu_run.  Hopefully that
	 * VCPU is holding the lock that we need and will release it.
	 * We approximate round-robin by starting at the last boosted VCPU.
	 */
2341
	for (pass = 0; pass < 2 && !yielded && try; pass++) {
2342
		kvm_for_each_vcpu(i, vcpu, kvm) {
2343
			if (!pass && i <= last_boosted_vcpu) {
2344 2345 2346 2347
				i = last_boosted_vcpu;
				continue;
			} else if (pass && i > last_boosted_vcpu)
				break;
2348 2349
			if (!ACCESS_ONCE(vcpu->preempted))
				continue;
2350 2351
			if (vcpu == me)
				continue;
2352
			if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
2353
				continue;
2354 2355
			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
				continue;
2356 2357 2358

			yielded = kvm_vcpu_yield_to(vcpu);
			if (yielded > 0) {
2359 2360
				kvm->last_boosted_vcpu = i;
				break;
2361 2362 2363 2364
			} else if (yielded < 0) {
				try--;
				if (!try)
					break;
2365 2366 2367
			}
		}
	}
2368
	kvm_vcpu_set_in_spin_loop(me, false);
2369 2370 2371

	/* Ensure vcpu is not eligible during next spinloop */
	kvm_vcpu_set_dy_eligible(me, false);
Z
Zhai, Edwin 已提交
2372 2373 2374
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

2375
static int kvm_vcpu_fault(struct vm_fault *vmf)
2376
{
2377
	struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
2378 2379
	struct page *page;

2380
	if (vmf->pgoff == 0)
2381
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
2382
#ifdef CONFIG_X86
2383
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2384
		page = virt_to_page(vcpu->arch.pio_data);
2385
#endif
2386
#ifdef CONFIG_KVM_MMIO
2387 2388
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
2389
#endif
2390
	else
2391
		return kvm_arch_vcpu_fault(vcpu, vmf);
2392
	get_page(page);
2393 2394
	vmf->page = page;
	return 0;
2395 2396
}

2397
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2398
	.fault = kvm_vcpu_fault,
2399 2400 2401 2402 2403 2404 2405 2406
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
2407 2408 2409 2410
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

2411
	debugfs_remove_recursive(vcpu->debugfs_dentry);
A
Al Viro 已提交
2412
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
2413 2414 2415
	return 0;
}

2416
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
2417 2418
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
2419
#ifdef CONFIG_KVM_COMPAT
2420 2421
	.compat_ioctl   = kvm_vcpu_compat_ioctl,
#endif
2422
	.mmap           = kvm_vcpu_mmap,
2423
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
2424 2425 2426 2427 2428 2429 2430
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
2431
	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
A
Avi Kivity 已提交
2432 2433
}

2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459
static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
{
	char dir_name[ITOA_MAX_LEN * 2];
	int ret;

	if (!kvm_arch_has_vcpu_debugfs())
		return 0;

	if (!debugfs_initialized())
		return 0;

	snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
	vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
								vcpu->kvm->debugfs_dentry);
	if (!vcpu->debugfs_dentry)
		return -ENOMEM;

	ret = kvm_arch_create_vcpu_debugfs(vcpu);
	if (ret < 0) {
		debugfs_remove_recursive(vcpu->debugfs_dentry);
		return ret;
	}

	return 0;
}

2460 2461 2462
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
2463
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2464 2465
{
	int r;
2466
	struct kvm_vcpu *vcpu;
2467

G
Greg Kurz 已提交
2468
	if (id >= KVM_MAX_VCPU_ID)
2469 2470
		return -EINVAL;

2471 2472 2473 2474 2475 2476 2477 2478 2479
	mutex_lock(&kvm->lock);
	if (kvm->created_vcpus == KVM_MAX_VCPUS) {
		mutex_unlock(&kvm->lock);
		return -EINVAL;
	}

	kvm->created_vcpus++;
	mutex_unlock(&kvm->lock);

2480
	vcpu = kvm_arch_vcpu_create(kvm, id);
2481 2482 2483 2484
	if (IS_ERR(vcpu)) {
		r = PTR_ERR(vcpu);
		goto vcpu_decrement;
	}
2485

2486 2487
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

2488 2489
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
2490
		goto vcpu_destroy;
2491

2492 2493 2494 2495
	r = kvm_create_vcpu_debugfs(vcpu);
	if (r)
		goto vcpu_destroy;

S
Shaohua Li 已提交
2496
	mutex_lock(&kvm->lock);
2497 2498 2499 2500
	if (kvm_get_vcpu_by_id(kvm, id)) {
		r = -EEXIST;
		goto unlock_vcpu_destroy;
	}
2501 2502

	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2503

R
Rusty Russell 已提交
2504
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
2505
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
2506
	r = create_vcpu_fd(vcpu);
2507 2508
	if (r < 0) {
		kvm_put_kvm(kvm);
2509
		goto unlock_vcpu_destroy;
2510 2511 2512
	}

	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
2513 2514 2515 2516 2517

	/*
	 * Pairs with smp_rmb() in kvm_get_vcpu.  Write kvm->vcpus
	 * before kvm->online_vcpu's incremented value.
	 */
2518 2519 2520 2521
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

	mutex_unlock(&kvm->lock);
2522
	kvm_arch_vcpu_postcreate(vcpu);
R
Rusty Russell 已提交
2523
	return r;
2524

2525
unlock_vcpu_destroy:
2526
	mutex_unlock(&kvm->lock);
2527
	debugfs_remove_recursive(vcpu->debugfs_dentry);
2528
vcpu_destroy:
2529
	kvm_arch_vcpu_destroy(vcpu);
2530 2531 2532 2533
vcpu_decrement:
	mutex_lock(&kvm->lock);
	kvm->created_vcpus--;
	mutex_unlock(&kvm->lock);
2534 2535 2536
	return r;
}

A
Avi Kivity 已提交
2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
2548 2549
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
2550
{
A
Avi Kivity 已提交
2551
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
2552
	void __user *argp = (void __user *)arg;
2553
	int r;
2554 2555
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
2556

2557 2558
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
2559

2560 2561 2562
	if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
		return -EINVAL;

2563
#if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
2564 2565 2566 2567
	/*
	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
	 * so vcpu_load() would break it.
	 */
2568
	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT)
2569 2570 2571 2572
		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
#endif


2573 2574 2575
	r = vcpu_load(vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
2576
	switch (ioctl) {
2577 2578
	case KVM_RUN: {
		struct pid *oldpid;
2579 2580 2581
		r = -EINVAL;
		if (arg)
			goto out;
2582 2583
		oldpid = rcu_access_pointer(vcpu->pid);
		if (unlikely(oldpid != current->pids[PIDTYPE_PID].pid)) {
2584 2585
			/* The thread running this VCPU changed. */
			struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
2586

2587 2588 2589 2590 2591
			rcu_assign_pointer(vcpu->pid, newpid);
			if (oldpid)
				synchronize_rcu();
			put_pid(oldpid);
		}
2592
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2593
		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
A
Avi Kivity 已提交
2594
		break;
2595
	}
A
Avi Kivity 已提交
2596
	case KVM_GET_REGS: {
2597
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2598

2599 2600 2601
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
2602
			goto out;
2603 2604 2605
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
2606
		r = -EFAULT;
2607 2608
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
2609
		r = 0;
2610 2611
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
2612 2613 2614
		break;
	}
	case KVM_SET_REGS: {
2615
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2616

2617
		r = -ENOMEM;
2618 2619 2620
		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
		if (IS_ERR(kvm_regs)) {
			r = PTR_ERR(kvm_regs);
A
Avi Kivity 已提交
2621
			goto out;
2622
		}
2623 2624
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
		kfree(kvm_regs);
A
Avi Kivity 已提交
2625 2626 2627
		break;
	}
	case KVM_GET_SREGS: {
2628 2629 2630 2631 2632
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2633 2634 2635
		if (r)
			goto out;
		r = -EFAULT;
2636
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
2637 2638 2639 2640 2641
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
2642 2643 2644
		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
		if (IS_ERR(kvm_sregs)) {
			r = PTR_ERR(kvm_sregs);
G
Guo Chao 已提交
2645
			kvm_sregs = NULL;
A
Avi Kivity 已提交
2646
			goto out;
2647
		}
2648
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2649 2650
		break;
	}
2651 2652 2653 2654 2655 2656 2657
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
2658
		if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2659 2660 2661 2662 2663 2664 2665 2666
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
2667
		if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2668 2669 2670 2671
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		break;
	}
A
Avi Kivity 已提交
2672 2673 2674 2675
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
2676
		if (copy_from_user(&tr, argp, sizeof(tr)))
A
Avi Kivity 已提交
2677
			goto out;
2678
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2679 2680 2681
		if (r)
			goto out;
		r = -EFAULT;
2682
		if (copy_to_user(argp, &tr, sizeof(tr)))
A
Avi Kivity 已提交
2683 2684 2685 2686
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
2687 2688
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
2689 2690

		r = -EFAULT;
2691
		if (copy_from_user(&dbg, argp, sizeof(dbg)))
A
Avi Kivity 已提交
2692
			goto out;
J
Jan Kiszka 已提交
2693
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
2694 2695
		break;
	}
A
Avi Kivity 已提交
2696 2697 2698 2699 2700 2701 2702 2703 2704
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
2705
					   sizeof(kvm_sigmask)))
A
Avi Kivity 已提交
2706 2707
				goto out;
			r = -EINVAL;
2708
			if (kvm_sigmask.len != sizeof(sigset))
A
Avi Kivity 已提交
2709 2710 2711
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
2712
					   sizeof(sigset)))
A
Avi Kivity 已提交
2713 2714 2715
				goto out;
			p = &sigset;
		}
2716
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
A
Avi Kivity 已提交
2717 2718
		break;
	}
A
Avi Kivity 已提交
2719
	case KVM_GET_FPU: {
2720 2721 2722 2723 2724
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2725 2726 2727
		if (r)
			goto out;
		r = -EFAULT;
2728
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
2729 2730 2731 2732 2733
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
2734 2735 2736
		fpu = memdup_user(argp, sizeof(*fpu));
		if (IS_ERR(fpu)) {
			r = PTR_ERR(fpu);
G
Guo Chao 已提交
2737
			fpu = NULL;
A
Avi Kivity 已提交
2738
			goto out;
2739
		}
2740
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2741 2742
		break;
	}
A
Avi Kivity 已提交
2743
	default:
2744
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2745 2746
	}
out:
2747
	vcpu_put(vcpu);
2748 2749
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
2750 2751 2752
	return r;
}

2753
#ifdef CONFIG_KVM_COMPAT
2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773
static long kvm_vcpu_compat_ioctl(struct file *filp,
				  unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = compat_ptr(arg);
	int r;

	if (vcpu->kvm->mm != current->mm)
		return -EIO;

	switch (ioctl) {
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		compat_sigset_t csigset;
		sigset_t sigset;

		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
2774
					   sizeof(kvm_sigmask)))
2775 2776
				goto out;
			r = -EINVAL;
2777
			if (kvm_sigmask.len != sizeof(csigset))
2778 2779 2780
				goto out;
			r = -EFAULT;
			if (copy_from_user(&csigset, sigmask_arg->sigset,
2781
					   sizeof(csigset)))
2782
				goto out;
2783 2784 2785 2786
			sigset_from_compat(&sigset, &csigset);
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		} else
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797
		break;
	}
	default:
		r = kvm_vcpu_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

S
Scott Wood 已提交
2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844
static int kvm_device_ioctl_attr(struct kvm_device *dev,
				 int (*accessor)(struct kvm_device *dev,
						 struct kvm_device_attr *attr),
				 unsigned long arg)
{
	struct kvm_device_attr attr;

	if (!accessor)
		return -EPERM;

	if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
		return -EFAULT;

	return accessor(dev, &attr);
}

static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
			     unsigned long arg)
{
	struct kvm_device *dev = filp->private_data;

	switch (ioctl) {
	case KVM_SET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
	case KVM_GET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
	case KVM_HAS_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
	default:
		if (dev->ops->ioctl)
			return dev->ops->ioctl(dev, ioctl, arg);

		return -ENOTTY;
	}
}

static int kvm_device_release(struct inode *inode, struct file *filp)
{
	struct kvm_device *dev = filp->private_data;
	struct kvm *kvm = dev->kvm;

	kvm_put_kvm(kvm);
	return 0;
}

static const struct file_operations kvm_device_fops = {
	.unlocked_ioctl = kvm_device_ioctl,
2845
#ifdef CONFIG_KVM_COMPAT
2846 2847
	.compat_ioctl = kvm_device_ioctl,
#endif
S
Scott Wood 已提交
2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858
	.release = kvm_device_release,
};

struct kvm_device *kvm_device_from_filp(struct file *filp)
{
	if (filp->f_op != &kvm_device_fops)
		return NULL;

	return filp->private_data;
}

2859
static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2860
#ifdef CONFIG_KVM_MPIC
2861 2862
	[KVM_DEV_TYPE_FSL_MPIC_20]	= &kvm_mpic_ops,
	[KVM_DEV_TYPE_FSL_MPIC_42]	= &kvm_mpic_ops,
2863
#endif
2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877
};

int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
{
	if (type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENOSPC;

	if (kvm_device_ops_table[type] != NULL)
		return -EEXIST;

	kvm_device_ops_table[type] = ops;
	return 0;
}

2878 2879 2880 2881 2882 2883
void kvm_unregister_device_ops(u32 type)
{
	if (kvm_device_ops_table[type] != NULL)
		kvm_device_ops_table[type] = NULL;
}

S
Scott Wood 已提交
2884 2885 2886 2887 2888 2889 2890 2891
static int kvm_ioctl_create_device(struct kvm *kvm,
				   struct kvm_create_device *cd)
{
	struct kvm_device_ops *ops = NULL;
	struct kvm_device *dev;
	bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
	int ret;

2892 2893 2894 2895 2896
	if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENODEV;

	ops = kvm_device_ops_table[cd->type];
	if (ops == NULL)
S
Scott Wood 已提交
2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908
		return -ENODEV;

	if (test)
		return 0;

	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
	if (!dev)
		return -ENOMEM;

	dev->ops = ops;
	dev->kvm = kvm;

2909
	mutex_lock(&kvm->lock);
S
Scott Wood 已提交
2910 2911
	ret = ops->create(dev, cd->type);
	if (ret < 0) {
2912
		mutex_unlock(&kvm->lock);
S
Scott Wood 已提交
2913 2914 2915
		kfree(dev);
		return ret;
	}
2916 2917
	list_add(&dev->vm_node, &kvm->devices);
	mutex_unlock(&kvm->lock);
S
Scott Wood 已提交
2918

2919 2920 2921
	if (ops->init)
		ops->init(dev);

2922
	ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
S
Scott Wood 已提交
2923
	if (ret < 0) {
2924 2925 2926
		mutex_lock(&kvm->lock);
		list_del(&dev->vm_node);
		mutex_unlock(&kvm->lock);
2927
		ops->destroy(dev);
S
Scott Wood 已提交
2928 2929 2930 2931 2932 2933 2934 2935
		return ret;
	}

	kvm_get_kvm(kvm);
	cd->fd = ret;
	return 0;
}

2936 2937 2938 2939 2940 2941 2942 2943 2944 2945
static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
{
	switch (arg) {
	case KVM_CAP_USER_MEMORY:
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
	case KVM_CAP_INTERNAL_ERROR_DATA:
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_CAP_SIGNAL_MSI:
#endif
2946
#ifdef CONFIG_HAVE_KVM_IRQFD
2947
	case KVM_CAP_IRQFD:
2948 2949
	case KVM_CAP_IRQFD_RESAMPLE:
#endif
2950
	case KVM_CAP_IOEVENTFD_ANY_LENGTH:
2951 2952
	case KVM_CAP_CHECK_EXTENSION_VM:
		return 1;
2953
#ifdef CONFIG_KVM_MMIO
2954 2955 2956
	case KVM_CAP_COALESCED_MMIO:
		return KVM_COALESCED_MMIO_PAGE_OFFSET;
#endif
2957 2958 2959
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_CAP_IRQ_ROUTING:
		return KVM_MAX_IRQ_ROUTES;
2960 2961 2962 2963
#endif
#if KVM_ADDRESS_SPACE_NUM > 1
	case KVM_CAP_MULTI_ADDRESS_SPACE:
		return KVM_ADDRESS_SPACE_NUM;
2964
#endif
G
Greg Kurz 已提交
2965 2966
	case KVM_CAP_MAX_VCPU_ID:
		return KVM_MAX_VCPU_ID;
2967 2968 2969 2970 2971 2972
	default:
		break;
	}
	return kvm_vm_ioctl_check_extension(kvm, arg);
}

A
Avi Kivity 已提交
2973 2974 2975 2976 2977
static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
2978
	int r;
A
Avi Kivity 已提交
2979

2980 2981
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
2982 2983 2984 2985
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		break;
2986 2987 2988 2989 2990
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
2991
						sizeof(kvm_userspace_mem)))
2992 2993
			goto out;

2994
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
A
Avi Kivity 已提交
2995 2996 2997 2998 2999 3000
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
3001
		if (copy_from_user(&log, argp, sizeof(log)))
A
Avi Kivity 已提交
3002
			goto out;
3003
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
3004 3005
		break;
	}
3006
#ifdef CONFIG_KVM_MMIO
3007 3008
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
3009

3010
		r = -EFAULT;
3011
		if (copy_from_user(&zone, argp, sizeof(zone)))
3012 3013 3014 3015 3016 3017
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
3018

3019
		r = -EFAULT;
3020
		if (copy_from_user(&zone, argp, sizeof(zone)))
3021 3022 3023 3024 3025
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		break;
	}
#endif
G
Gregory Haskins 已提交
3026 3027 3028 3029
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
3030
		if (copy_from_user(&data, argp, sizeof(data)))
G
Gregory Haskins 已提交
3031
			goto out;
3032
		r = kvm_irqfd(kvm, &data);
G
Gregory Haskins 已提交
3033 3034
		break;
	}
G
Gregory Haskins 已提交
3035 3036 3037 3038
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
3039
		if (copy_from_user(&data, argp, sizeof(data)))
G
Gregory Haskins 已提交
3040 3041 3042 3043
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
3044 3045 3046 3047 3048
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_SIGNAL_MSI: {
		struct kvm_msi msi;

		r = -EFAULT;
3049
		if (copy_from_user(&msi, argp, sizeof(msi)))
3050 3051 3052 3053
			goto out;
		r = kvm_send_userspace_msi(kvm, &msi);
		break;
	}
3054 3055 3056 3057 3058 3059 3060
#endif
#ifdef __KVM_HAVE_IRQ_LINE
	case KVM_IRQ_LINE_STATUS:
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
3061
		if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
3062 3063
			goto out;

3064 3065
		r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
					ioctl == KVM_IRQ_LINE_STATUS);
3066 3067 3068 3069 3070
		if (r)
			goto out;

		r = -EFAULT;
		if (ioctl == KVM_IRQ_LINE_STATUS) {
3071
			if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
3072 3073 3074 3075 3076 3077
				goto out;
		}

		r = 0;
		break;
	}
3078
#endif
3079 3080 3081 3082
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_SET_GSI_ROUTING: {
		struct kvm_irq_routing routing;
		struct kvm_irq_routing __user *urouting;
3083
		struct kvm_irq_routing_entry *entries = NULL;
3084 3085 3086 3087 3088

		r = -EFAULT;
		if (copy_from_user(&routing, argp, sizeof(routing)))
			goto out;
		r = -EINVAL;
3089 3090
		if (!kvm_arch_can_set_irq_routing(kvm))
			goto out;
3091
		if (routing.nr > KVM_MAX_IRQ_ROUTES)
3092 3093 3094
			goto out;
		if (routing.flags)
			goto out;
3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105
		if (routing.nr) {
			r = -ENOMEM;
			entries = vmalloc(routing.nr * sizeof(*entries));
			if (!entries)
				goto out;
			r = -EFAULT;
			urouting = argp;
			if (copy_from_user(entries, urouting->entries,
					   routing.nr * sizeof(*entries)))
				goto out_free_irq_routing;
		}
3106 3107
		r = kvm_set_irq_routing(kvm, entries, routing.nr,
					routing.flags);
3108
out_free_irq_routing:
3109 3110 3111 3112
		vfree(entries);
		break;
	}
#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
S
Scott Wood 已提交
3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130
	case KVM_CREATE_DEVICE: {
		struct kvm_create_device cd;

		r = -EFAULT;
		if (copy_from_user(&cd, argp, sizeof(cd)))
			goto out;

		r = kvm_ioctl_create_device(kvm, &cd);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(argp, &cd, sizeof(cd)))
			goto out;

		r = 0;
		break;
	}
3131 3132 3133
	case KVM_CHECK_EXTENSION:
		r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
		break;
3134
	default:
3135
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
3136 3137 3138 3139 3140
	}
out:
	return r;
}

3141
#ifdef CONFIG_KVM_COMPAT
3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
3166
			return -EFAULT;
3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}
	return r;
}
#endif

3182
static struct file_operations kvm_vm_fops = {
3183 3184
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
3185
#ifdef CONFIG_KVM_COMPAT
3186 3187
	.compat_ioctl   = kvm_vm_compat_ioctl,
#endif
3188
	.llseek		= noop_llseek,
3189 3190
};

3191
static int kvm_dev_ioctl_create_vm(unsigned long type)
3192
{
3193
	int r;
3194
	struct kvm *kvm;
3195
	struct file *file;
3196

3197
	kvm = kvm_create_vm(type);
3198 3199
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
3200
#ifdef CONFIG_KVM_MMIO
3201 3202 3203 3204 3205 3206
	r = kvm_coalesced_mmio_init(kvm);
	if (r < 0) {
		kvm_put_kvm(kvm);
		return r;
	}
#endif
3207
	r = get_unused_fd_flags(O_CLOEXEC);
3208
	if (r < 0) {
A
Al Viro 已提交
3209
		kvm_put_kvm(kvm);
3210 3211
		return r;
	}
3212 3213 3214 3215 3216 3217
	file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
	if (IS_ERR(file)) {
		put_unused_fd(r);
		kvm_put_kvm(kvm);
		return PTR_ERR(file);
	}
3218

3219 3220 3221 3222 3223 3224
	/*
	 * Don't call kvm_put_kvm anymore at this point; file->f_op is
	 * already set, with ->release() being kvm_vm_release().  In error
	 * cases it will be called by the final fput(file) and will take
	 * care of doing kvm_put_kvm(kvm).
	 */
3225
	if (kvm_create_vm_debugfs(kvm, r) < 0) {
3226 3227
		put_unused_fd(r);
		fput(file);
3228 3229
		return -ENOMEM;
	}
3230
	kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
3231

3232
	fd_install(r, file);
3233
	return r;
3234 3235 3236 3237 3238
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
3239
	long r = -EINVAL;
3240 3241 3242

	switch (ioctl) {
	case KVM_GET_API_VERSION:
3243 3244
		if (arg)
			goto out;
3245 3246 3247
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
3248
		r = kvm_dev_ioctl_create_vm(arg);
3249
		break;
3250
	case KVM_CHECK_EXTENSION:
3251
		r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3252
		break;
3253 3254 3255
	case KVM_GET_VCPU_MMAP_SIZE:
		if (arg)
			goto out;
3256 3257 3258
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
3259
#endif
3260
#ifdef CONFIG_KVM_MMIO
3261
		r += PAGE_SIZE;    /* coalesced mmio ring page */
3262
#endif
3263
		break;
3264 3265 3266
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
3267
		r = -EOPNOTSUPP;
3268
		break;
A
Avi Kivity 已提交
3269
	default:
3270
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
3271 3272 3273 3274 3275 3276 3277 3278
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
3279
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
3280 3281 3282
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
3283
	KVM_MINOR,
A
Avi Kivity 已提交
3284 3285 3286 3287
	"kvm",
	&kvm_chardev_ops,
};

3288
static void hardware_enable_nolock(void *junk)
3289 3290
{
	int cpu = raw_smp_processor_id();
3291
	int r;
3292

3293
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3294
		return;
3295

3296
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
3297

3298
	r = kvm_arch_hardware_enable();
3299 3300 3301 3302

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
X
Xiubo Li 已提交
3303
		pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3304
	}
3305 3306
}

3307
static int kvm_starting_cpu(unsigned int cpu)
3308
{
3309
	raw_spin_lock(&kvm_count_lock);
3310 3311
	if (kvm_usage_count)
		hardware_enable_nolock(NULL);
3312
	raw_spin_unlock(&kvm_count_lock);
3313
	return 0;
3314 3315 3316
}

static void hardware_disable_nolock(void *junk)
3317 3318 3319
{
	int cpu = raw_smp_processor_id();

3320
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3321
		return;
3322
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3323
	kvm_arch_hardware_disable();
3324 3325
}

3326
static int kvm_dying_cpu(unsigned int cpu)
3327
{
3328
	raw_spin_lock(&kvm_count_lock);
3329 3330
	if (kvm_usage_count)
		hardware_disable_nolock(NULL);
3331
	raw_spin_unlock(&kvm_count_lock);
3332
	return 0;
3333 3334
}

3335 3336 3337 3338 3339 3340
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
3341
		on_each_cpu(hardware_disable_nolock, NULL, 1);
3342 3343 3344 3345
}

static void hardware_disable_all(void)
{
3346
	raw_spin_lock(&kvm_count_lock);
3347
	hardware_disable_all_nolock();
3348
	raw_spin_unlock(&kvm_count_lock);
3349 3350 3351 3352 3353 3354
}

static int hardware_enable_all(void)
{
	int r = 0;

3355
	raw_spin_lock(&kvm_count_lock);
3356 3357 3358 3359

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
3360
		on_each_cpu(hardware_enable_nolock, NULL, 1);
3361 3362 3363 3364 3365 3366 3367

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

3368
	raw_spin_unlock(&kvm_count_lock);
3369 3370 3371 3372

	return r;
}

3373
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
3374
		      void *v)
3375
{
3376 3377 3378 3379 3380 3381
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
X
Xiubo Li 已提交
3382
	pr_info("kvm: exiting hardware virtualization\n");
3383
	kvm_rebooting = true;
3384
	on_each_cpu(hardware_disable_nolock, NULL, 1);
3385 3386 3387 3388 3389 3390 3391 3392
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
3393
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3394 3395 3396 3397
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
3398
		struct kvm_io_device *pos = bus->range[i].dev;
3399 3400 3401

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
3402
	kfree(bus);
3403 3404
}

3405
static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
X
Xiubo Li 已提交
3406
				 const struct kvm_io_range *r2)
3407
{
J
Jason Wang 已提交
3408 3409 3410 3411
	gpa_t addr1 = r1->addr;
	gpa_t addr2 = r2->addr;

	if (addr1 < addr2)
3412
		return -1;
J
Jason Wang 已提交
3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424

	/* If r2->len == 0, match the exact address.  If r2->len != 0,
	 * accept any overlapping write.  Any order is acceptable for
	 * overlapping ranges, because kvm_io_bus_get_first_dev ensures
	 * we process all of them.
	 */
	if (r2->len) {
		addr1 += r1->len;
		addr2 += r2->len;
	}

	if (addr1 > addr2)
3425
		return 1;
J
Jason Wang 已提交
3426

3427 3428 3429
	return 0;
}

3430 3431
static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
{
3432
	return kvm_io_bus_cmp(p1, p2);
3433 3434
}

G
Geoff Levand 已提交
3435
static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449
			  gpa_t addr, int len)
{
	bus->range[bus->dev_count++] = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
		.dev = dev,
	};

	sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
		kvm_io_bus_sort_cmp, NULL);

	return 0;
}

G
Geoff Levand 已提交
3450
static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467
			     gpa_t addr, int len)
{
	struct kvm_io_range *range, key;
	int off;

	key = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	range = bsearch(&key, bus->range, bus->dev_count,
			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
	if (range == NULL)
		return -ENOENT;

	off = range - bus->range;

3468
	while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3469 3470 3471 3472 3473
		off--;

	return off;
}

3474
static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
C
Cornelia Huck 已提交
3475 3476 3477 3478 3479 3480 3481 3482 3483
			      struct kvm_io_range *range, const void *val)
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
3484
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3485
		if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
C
Cornelia Huck 已提交
3486 3487 3488 3489 3490 3491 3492 3493
					range->len, val))
			return idx;
		idx++;
	}

	return -EOPNOTSUPP;
}

3494
/* kvm_io_bus_write - called under kvm->slots_lock */
3495
int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3496
		     int len, const void *val)
3497
{
3498
	struct kvm_io_bus *bus;
3499
	struct kvm_io_range range;
C
Cornelia Huck 已提交
3500
	int r;
3501 3502 3503 3504 3505

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
3506

3507
	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3508 3509
	if (!bus)
		return -ENOMEM;
3510
	r = __kvm_io_bus_write(vcpu, bus, &range, val);
C
Cornelia Huck 已提交
3511 3512 3513 3514
	return r < 0 ? r : 0;
}

/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3515 3516
int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
			    gpa_t addr, int len, const void *val, long cookie)
C
Cornelia Huck 已提交
3517 3518 3519 3520 3521 3522 3523 3524 3525
{
	struct kvm_io_bus *bus;
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

3526
	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3527 3528
	if (!bus)
		return -ENOMEM;
C
Cornelia Huck 已提交
3529 3530 3531

	/* First try the device referenced by cookie. */
	if ((cookie >= 0) && (cookie < bus->dev_count) &&
3532
	    (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3533
		if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
C
Cornelia Huck 已提交
3534 3535 3536 3537 3538 3539 3540
					val))
			return cookie;

	/*
	 * cookie contained garbage; fall back to search and return the
	 * correct cookie value.
	 */
3541
	return __kvm_io_bus_write(vcpu, bus, &range, val);
C
Cornelia Huck 已提交
3542 3543
}

3544 3545
static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
			     struct kvm_io_range *range, void *val)
C
Cornelia Huck 已提交
3546 3547 3548 3549
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3550 3551 3552 3553
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
3554
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3555
		if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
C
Cornelia Huck 已提交
3556 3557
				       range->len, val))
			return idx;
3558 3559 3560
		idx++;
	}

3561 3562
	return -EOPNOTSUPP;
}
3563
EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3564

3565
/* kvm_io_bus_read - called under kvm->slots_lock */
3566
int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
M
Marcelo Tosatti 已提交
3567
		    int len, void *val)
3568
{
3569
	struct kvm_io_bus *bus;
3570
	struct kvm_io_range range;
C
Cornelia Huck 已提交
3571
	int r;
3572 3573 3574 3575 3576

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
M
Marcelo Tosatti 已提交
3577

3578
	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3579 3580
	if (!bus)
		return -ENOMEM;
3581
	r = __kvm_io_bus_read(vcpu, bus, &range, val);
C
Cornelia Huck 已提交
3582 3583
	return r < 0 ? r : 0;
}
3584

3585

3586
/* Caller must hold slots_lock. */
3587 3588
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, struct kvm_io_device *dev)
3589
{
M
Marcelo Tosatti 已提交
3590
	struct kvm_io_bus *new_bus, *bus;
3591

3592
	bus = kvm_get_bus(kvm, bus_idx);
3593 3594 3595
	if (!bus)
		return -ENOMEM;

3596 3597
	/* exclude ioeventfd which is limited by maximum fd */
	if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3598
		return -ENOSPC;
3599

3600
	new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) *
3601
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
M
Marcelo Tosatti 已提交
3602 3603
	if (!new_bus)
		return -ENOMEM;
3604 3605
	memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
	       sizeof(struct kvm_io_range)));
3606
	kvm_io_bus_insert_dev(new_bus, dev, addr, len);
M
Marcelo Tosatti 已提交
3607 3608 3609
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
3610 3611 3612 3613

	return 0;
}

3614
/* Caller must hold slots_lock. */
3615 3616
void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			       struct kvm_io_device *dev)
3617
{
3618
	int i;
M
Marcelo Tosatti 已提交
3619
	struct kvm_io_bus *new_bus, *bus;
3620

3621
	bus = kvm_get_bus(kvm, bus_idx);
3622
	if (!bus)
3623
		return;
3624

3625 3626
	for (i = 0; i < bus->dev_count; i++)
		if (bus->range[i].dev == dev) {
3627 3628
			break;
		}
M
Marcelo Tosatti 已提交
3629

3630 3631
	if (i == bus->dev_count)
		return;
3632

3633
	new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) *
3634
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
3635 3636 3637 3638
	if (!new_bus)  {
		pr_err("kvm: failed to shrink bus, removing it completely\n");
		goto broken;
	}
3639 3640 3641 3642 3643

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count--;
	memcpy(new_bus->range + i, bus->range + i + 1,
	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
3644

3645
broken:
M
Marcelo Tosatti 已提交
3646 3647 3648
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
3649
	return;
3650 3651
}

3652 3653 3654 3655 3656 3657 3658 3659 3660 3661
struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
					 gpa_t addr)
{
	struct kvm_io_bus *bus;
	int dev_idx, srcu_idx;
	struct kvm_io_device *iodev = NULL;

	srcu_idx = srcu_read_lock(&kvm->srcu);

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
3662 3663
	if (!bus)
		goto out_unlock;
3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677

	dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
	if (dev_idx < 0)
		goto out_unlock;

	iodev = bus->range[dev_idx].dev;

out_unlock:
	srcu_read_unlock(&kvm->srcu, srcu_idx);

	return iodev;
}
EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);

3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689
static int kvm_debugfs_open(struct inode *inode, struct file *file,
			   int (*get)(void *, u64 *), int (*set)(void *, u64),
			   const char *fmt)
{
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
					  inode->i_private;

	/* The debugfs files are a reference to the kvm struct which
	 * is still valid when kvm_destroy_vm is called.
	 * To avoid the race between open and the removal of the debugfs
	 * directory we test against the users count.
	 */
3690
	if (!refcount_inc_not_zero(&stat_data->kvm->users_count))
3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715
		return -ENOENT;

	if (simple_attr_open(inode, file, get, set, fmt)) {
		kvm_put_kvm(stat_data->kvm);
		return -ENOMEM;
	}

	return 0;
}

static int kvm_debugfs_release(struct inode *inode, struct file *file)
{
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
					  inode->i_private;

	simple_attr_release(inode, file);
	kvm_put_kvm(stat_data->kvm);

	return 0;
}

static int vm_stat_get_per_vm(void *data, u64 *val)
{
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;

3716
	*val = *(ulong *)((void *)stat_data->kvm + stat_data->offset);
3717 3718 3719 3720

	return 0;
}

3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732
static int vm_stat_clear_per_vm(void *data, u64 val)
{
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;

	if (val)
		return -EINVAL;

	*(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0;

	return 0;
}

3733 3734 3735 3736
static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file)
{
	__simple_attr_check_format("%llu\n", 0ull);
	return kvm_debugfs_open(inode, file, vm_stat_get_per_vm,
3737
				vm_stat_clear_per_vm, "%llu\n");
3738 3739 3740 3741 3742 3743 3744 3745
}

static const struct file_operations vm_stat_get_per_vm_fops = {
	.owner   = THIS_MODULE,
	.open    = vm_stat_get_per_vm_open,
	.release = kvm_debugfs_release,
	.read    = simple_attr_read,
	.write   = simple_attr_write,
3746
	.llseek  = no_llseek,
3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757
};

static int vcpu_stat_get_per_vm(void *data, u64 *val)
{
	int i;
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
	struct kvm_vcpu *vcpu;

	*val = 0;

	kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
3758
		*val += *(u64 *)((void *)vcpu + stat_data->offset);
3759 3760 3761 3762

	return 0;
}

3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777
static int vcpu_stat_clear_per_vm(void *data, u64 val)
{
	int i;
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
	struct kvm_vcpu *vcpu;

	if (val)
		return -EINVAL;

	kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
		*(u64 *)((void *)vcpu + stat_data->offset) = 0;

	return 0;
}

3778 3779 3780 3781
static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file)
{
	__simple_attr_check_format("%llu\n", 0ull);
	return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm,
3782
				 vcpu_stat_clear_per_vm, "%llu\n");
3783 3784 3785 3786 3787 3788 3789 3790
}

static const struct file_operations vcpu_stat_get_per_vm_fops = {
	.owner   = THIS_MODULE,
	.open    = vcpu_stat_get_per_vm_open,
	.release = kvm_debugfs_release,
	.read    = simple_attr_read,
	.write   = simple_attr_write,
3791
	.llseek  = no_llseek,
3792 3793 3794 3795 3796 3797 3798
};

static const struct file_operations *stat_fops_per_vm[] = {
	[KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops,
	[KVM_STAT_VM]   = &vm_stat_get_per_vm_fops,
};

3799
static int vm_stat_get(void *_offset, u64 *val)
3800 3801 3802
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
3803 3804
	struct kvm_stat_data stat_tmp = {.offset = offset};
	u64 tmp_val;
3805

3806
	*val = 0;
3807
	spin_lock(&kvm_lock);
3808 3809 3810 3811 3812
	list_for_each_entry(kvm, &vm_list, vm_list) {
		stat_tmp.kvm = kvm;
		vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
		*val += tmp_val;
	}
3813
	spin_unlock(&kvm_lock);
3814
	return 0;
3815 3816
}

3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836
static int vm_stat_clear(void *_offset, u64 val)
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_stat_data stat_tmp = {.offset = offset};

	if (val)
		return -EINVAL;

	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list) {
		stat_tmp.kvm = kvm;
		vm_stat_clear_per_vm((void *)&stat_tmp, 0);
	}
	spin_unlock(&kvm_lock);

	return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
3837

3838
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
3839 3840 3841
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
3842 3843
	struct kvm_stat_data stat_tmp = {.offset = offset};
	u64 tmp_val;
A
Avi Kivity 已提交
3844

3845
	*val = 0;
3846
	spin_lock(&kvm_lock);
3847 3848 3849 3850 3851
	list_for_each_entry(kvm, &vm_list, vm_list) {
		stat_tmp.kvm = kvm;
		vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
		*val += tmp_val;
	}
3852
	spin_unlock(&kvm_lock);
3853
	return 0;
A
Avi Kivity 已提交
3854 3855
}

3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876
static int vcpu_stat_clear(void *_offset, u64 val)
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_stat_data stat_tmp = {.offset = offset};

	if (val)
		return -EINVAL;

	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list) {
		stat_tmp.kvm = kvm;
		vcpu_stat_clear_per_vm((void *)&stat_tmp, 0);
	}
	spin_unlock(&kvm_lock);

	return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
			"%llu\n");
3877

3878
static const struct file_operations *stat_fops[] = {
3879 3880 3881
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
3882

3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908
static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
{
	struct kobj_uevent_env *env;
	unsigned long long created, active;

	if (!kvm_dev.this_device || !kvm)
		return;

	spin_lock(&kvm_lock);
	if (type == KVM_EVENT_CREATE_VM) {
		kvm_createvm_count++;
		kvm_active_vms++;
	} else if (type == KVM_EVENT_DESTROY_VM) {
		kvm_active_vms--;
	}
	created = kvm_createvm_count;
	active = kvm_active_vms;
	spin_unlock(&kvm_lock);

	env = kzalloc(sizeof(*env), GFP_KERNEL);
	if (!env)
		return;

	add_uevent_var(env, "CREATED=%llu", created);
	add_uevent_var(env, "COUNT=%llu", active);

3909
	if (type == KVM_EVENT_CREATE_VM) {
3910
		add_uevent_var(env, "EVENT=create");
3911 3912
		kvm->userspace_pid = task_pid_nr(current);
	} else if (type == KVM_EVENT_DESTROY_VM) {
3913
		add_uevent_var(env, "EVENT=destroy");
3914 3915
	}
	add_uevent_var(env, "PID=%d", kvm->userspace_pid);
3916 3917

	if (kvm->debugfs_dentry) {
3918 3919 3920 3921 3922 3923 3924
		char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL);

		if (p) {
			tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
			if (!IS_ERR(tmp))
				add_uevent_var(env, "STATS_PATH=%s", tmp);
			kfree(p);
3925 3926 3927 3928 3929 3930 3931 3932
		}
	}
	/* no need for checks, since we are adding at most only 5 keys */
	env->envp[env->envp_idx++] = NULL;
	kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
	kfree(env);
}

3933
static int kvm_init_debug(void)
A
Avi Kivity 已提交
3934
{
3935
	int r = -EEXIST;
A
Avi Kivity 已提交
3936 3937
	struct kvm_stats_debugfs_item *p;

3938
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3939 3940 3941
	if (kvm_debugfs_dir == NULL)
		goto out;

3942 3943
	kvm_debugfs_num_entries = 0;
	for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
3944
		if (!debugfs_create_file(p->name, 0644, kvm_debugfs_dir,
3945 3946
					 (void *)(long)p->offset,
					 stat_fops[p->kind]))
3947 3948 3949 3950 3951 3952 3953 3954 3955
			goto out_dir;
	}

	return 0;

out_dir:
	debugfs_remove_recursive(kvm_debugfs_dir);
out:
	return r;
A
Avi Kivity 已提交
3956 3957
}

3958
static int kvm_suspend(void)
3959
{
3960
	if (kvm_usage_count)
3961
		hardware_disable_nolock(NULL);
3962 3963 3964
	return 0;
}

3965
static void kvm_resume(void)
3966
{
3967
	if (kvm_usage_count) {
3968
		WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3969
		hardware_enable_nolock(NULL);
3970
	}
3971 3972
}

3973
static struct syscore_ops kvm_syscore_ops = {
3974 3975 3976 3977
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

3978 3979 3980 3981 3982 3983 3984 3985 3986
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3987

3988 3989
	if (vcpu->preempted)
		vcpu->preempted = false;
3990

R
Radim Krčmář 已提交
3991 3992
	kvm_arch_sched_in(vcpu, cpu);

3993
	kvm_arch_vcpu_load(vcpu, cpu);
3994 3995 3996 3997 3998 3999 4000
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

4001 4002
	if (current->state == TASK_RUNNING)
		vcpu->preempted = true;
4003
	kvm_arch_vcpu_put(vcpu);
4004 4005
}

4006
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
4007
		  struct module *module)
A
Avi Kivity 已提交
4008 4009
{
	int r;
Y
Yang, Sheng 已提交
4010
	int cpu;
A
Avi Kivity 已提交
4011

4012 4013
	r = kvm_arch_init(opaque);
	if (r)
4014
		goto out_fail;
4015

4016 4017 4018 4019
	/*
	 * kvm_arch_init makes sure there's at most one caller
	 * for architectures that support multiple implementations,
	 * like intel and amd on x86.
P
Paolo Bonzini 已提交
4020 4021
	 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
	 * conflicts in case kvm is already setup for another implementation.
4022
	 */
P
Paolo Bonzini 已提交
4023 4024 4025
	r = kvm_irqfd_init();
	if (r)
		goto out_irqfd;
4026

4027
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
4028 4029 4030 4031
		r = -ENOMEM;
		goto out_free_0;
	}

4032
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
4033
	if (r < 0)
4034
		goto out_free_0a;
A
Avi Kivity 已提交
4035

Y
Yang, Sheng 已提交
4036 4037
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
4038
				kvm_arch_check_processor_compat,
4039
				&r, 1);
Y
Yang, Sheng 已提交
4040
		if (r < 0)
4041
			goto out_free_1;
Y
Yang, Sheng 已提交
4042 4043
	}

T
Thomas Gleixner 已提交
4044
	r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
4045
				      kvm_starting_cpu, kvm_dying_cpu);
A
Avi Kivity 已提交
4046
	if (r)
4047
		goto out_free_2;
A
Avi Kivity 已提交
4048 4049
	register_reboot_notifier(&kvm_reboot_notifier);

4050
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
4051 4052 4053
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
J
Joe Perches 已提交
4054
					   0, NULL);
4055 4056
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
4057
		goto out_free_3;
4058 4059
	}

4060 4061 4062 4063
	r = kvm_async_pf_init();
	if (r)
		goto out_free;

A
Avi Kivity 已提交
4064
	kvm_chardev_ops.owner = module;
4065 4066
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
4067 4068 4069

	r = misc_register(&kvm_dev);
	if (r) {
X
Xiubo Li 已提交
4070
		pr_err("kvm: misc device register failed\n");
4071
		goto out_unreg;
A
Avi Kivity 已提交
4072 4073
	}

4074 4075
	register_syscore_ops(&kvm_syscore_ops);

4076 4077 4078
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

4079 4080
	r = kvm_init_debug();
	if (r) {
X
Xiubo Li 已提交
4081
		pr_err("kvm: create debugfs files failed\n");
4082 4083
		goto out_undebugfs;
	}
4084

P
Paolo Bonzini 已提交
4085 4086 4087
	r = kvm_vfio_ops_init();
	WARN_ON(r);

4088
	return 0;
A
Avi Kivity 已提交
4089

4090 4091
out_undebugfs:
	unregister_syscore_ops(&kvm_syscore_ops);
4092
	misc_deregister(&kvm_dev);
4093 4094
out_unreg:
	kvm_async_pf_deinit();
A
Avi Kivity 已提交
4095
out_free:
4096
	kmem_cache_destroy(kvm_vcpu_cache);
4097
out_free_3:
A
Avi Kivity 已提交
4098
	unregister_reboot_notifier(&kvm_reboot_notifier);
4099
	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4100 4101
out_free_2:
out_free_1:
4102
	kvm_arch_hardware_unsetup();
4103 4104
out_free_0a:
	free_cpumask_var(cpus_hardware_enabled);
4105
out_free_0:
4106
	kvm_irqfd_exit();
P
Paolo Bonzini 已提交
4107
out_irqfd:
4108 4109
	kvm_arch_exit();
out_fail:
A
Avi Kivity 已提交
4110 4111
	return r;
}
4112
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
4113

4114
void kvm_exit(void)
A
Avi Kivity 已提交
4115
{
4116
	debugfs_remove_recursive(kvm_debugfs_dir);
A
Avi Kivity 已提交
4117
	misc_deregister(&kvm_dev);
4118
	kmem_cache_destroy(kvm_vcpu_cache);
4119
	kvm_async_pf_deinit();
4120
	unregister_syscore_ops(&kvm_syscore_ops);
A
Avi Kivity 已提交
4121
	unregister_reboot_notifier(&kvm_reboot_notifier);
4122
	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4123
	on_each_cpu(hardware_disable_nolock, NULL, 1);
4124
	kvm_arch_hardware_unsetup();
4125
	kvm_arch_exit();
4126
	kvm_irqfd_exit();
4127
	free_cpumask_var(cpus_hardware_enabled);
4128
	kvm_vfio_ops_exit();
A
Avi Kivity 已提交
4129
}
4130
EXPORT_SYMBOL_GPL(kvm_exit);