kvm_main.c 106.3 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
A
Avi Kivity 已提交
2 3 4 5 6 7 8
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
9
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
10 11 12 13 14 15
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 */

16
#include <kvm/iodev.h>
A
Avi Kivity 已提交
17

18
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
19 20 21 22 23 24 25 26 27 28 29
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
30
#include <linux/syscore_ops.h>
A
Avi Kivity 已提交
31
#include <linux/cpu.h>
32
#include <linux/sched/signal.h>
33
#include <linux/sched/mm.h>
34
#include <linux/sched/stat.h>
35 36
#include <linux/cpumask.h>
#include <linux/smp.h>
37
#include <linux/anon_inodes.h>
38
#include <linux/profile.h>
39
#include <linux/kvm_para.h>
40
#include <linux/pagemap.h>
41
#include <linux/mman.h>
42
#include <linux/swap.h>
43
#include <linux/bitops.h>
44
#include <linux/spinlock.h>
45
#include <linux/compat.h>
46
#include <linux/srcu.h>
J
Joerg Roedel 已提交
47
#include <linux/hugetlb.h>
48
#include <linux/slab.h>
49 50
#include <linux/sort.h>
#include <linux/bsearch.h>
P
Paolo Bonzini 已提交
51
#include <linux/io.h>
52
#include <linux/lockdep.h>
53
#include <linux/kthread.h>
A
Avi Kivity 已提交
54

A
Avi Kivity 已提交
55
#include <asm/processor.h>
56
#include <asm/ioctl.h>
57
#include <linux/uaccess.h>
58
#include <asm/pgtable.h>
A
Avi Kivity 已提交
59

60
#include "coalesced_mmio.h"
61
#include "async_pf.h"
P
Paolo Bonzini 已提交
62
#include "vfio.h"
63

64 65 66
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

67 68 69
/* Worst case buffer size needed for holding an integer. */
#define ITOA_MAX_LEN 12

A
Avi Kivity 已提交
70 71 72
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

73
/* Architectures should define their poll value according to the halt latency */
74
unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
75
module_param(halt_poll_ns, uint, 0644);
76
EXPORT_SYMBOL_GPL(halt_poll_ns);
77

W
Wanpeng Li 已提交
78
/* Default doubles per-vcpu halt_poll_ns. */
79
unsigned int halt_poll_ns_grow = 2;
80
module_param(halt_poll_ns_grow, uint, 0644);
81
EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
W
Wanpeng Li 已提交
82

83 84 85 86 87
/* The start value to grow halt_poll_ns from */
unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
module_param(halt_poll_ns_grow_start, uint, 0644);
EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);

W
Wanpeng Li 已提交
88
/* Default resets per-vcpu halt_poll_ns . */
89
unsigned int halt_poll_ns_shrink;
90
module_param(halt_poll_ns_shrink, uint, 0644);
91
EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
W
Wanpeng Li 已提交
92

93 94 95
/*
 * Ordering of locks:
 *
96
 *	kvm->lock --> kvm->slots_lock --> kvm->irq_lock
97 98
 */

J
Junaid Shahid 已提交
99
DEFINE_MUTEX(kvm_lock);
100
static DEFINE_RAW_SPINLOCK(kvm_count_lock);
101
LIST_HEAD(vm_list);
102

103
static cpumask_var_t cpus_hardware_enabled;
104
static int kvm_usage_count;
105
static atomic_t hardware_enable_failed;
106

107
static struct kmem_cache *kvm_vcpu_cache;
A
Avi Kivity 已提交
108

109 110
static __read_mostly struct preempt_ops kvm_preempt_ops;

111
struct dentry *kvm_debugfs_dir;
112
EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
A
Avi Kivity 已提交
113

114
static int kvm_debugfs_num_entries;
115
static const struct file_operations stat_fops_per_vm;
116

A
Avi Kivity 已提交
117 118
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
119
#ifdef CONFIG_KVM_COMPAT
120 121
static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
				  unsigned long arg);
122 123
#define KVM_COMPAT(c)	.compat_ioctl	= (c)
#else
124 125 126 127 128 129 130
/*
 * For architectures that don't implement a compat infrastructure,
 * adopt a double line of defense:
 * - Prevent a compat task from opening /dev/kvm
 * - If the open has been done by a 64bit task, and the KVM fd
 *   passed to a compat task, let the ioctls fail.
 */
131 132
static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
				unsigned long arg) { return -EINVAL; }
133 134 135 136 137 138 139

static int kvm_no_compat_open(struct inode *inode, struct file *file)
{
	return is_compat_task() ? -ENODEV : 0;
}
#define KVM_COMPAT(c)	.compat_ioctl	= kvm_no_compat_ioctl,	\
			.open		= kvm_no_compat_open
140
#endif
141 142
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
143

M
Marcelo Tosatti 已提交
144
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
145

146
static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
M
Marcelo Tosatti 已提交
147

148
__visible bool kvm_rebooting;
149
EXPORT_SYMBOL_GPL(kvm_rebooting);
150

151 152
static bool largepages_enabled = true;

153 154 155 156 157 158
#define KVM_EVENT_CREATE_VM 0
#define KVM_EVENT_DESTROY_VM 1
static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
static unsigned long long kvm_createvm_count;
static unsigned long long kvm_active_vms;

159 160
__weak int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
		unsigned long start, unsigned long end, bool blockable)
161
{
162
	return 0;
163 164
}

165 166 167 168 169 170 171 172 173 174 175 176 177 178
bool kvm_is_zone_device_pfn(kvm_pfn_t pfn)
{
	/*
	 * The metadata used by is_zone_device_page() to determine whether or
	 * not a page is ZONE_DEVICE is guaranteed to be valid if and only if
	 * the device has been pinned, e.g. by get_user_pages().  WARN if the
	 * page_count() is zero to help detect bad usage of this helper.
	 */
	if (!pfn_valid(pfn) || WARN_ON_ONCE(!page_count(pfn_to_page(pfn))))
		return false;

	return is_zone_device_page(pfn_to_page(pfn));
}

D
Dan Williams 已提交
179
bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
B
Ben-Ami Yassour 已提交
180
{
181 182 183 184 185
	/*
	 * ZONE_DEVICE pages currently set PG_reserved, but from a refcounting
	 * perspective they are "normal" pages, albeit with slightly different
	 * usage rules.
	 */
186
	if (pfn_valid(pfn))
187 188
		return PageReserved(pfn_to_page(pfn)) &&
		       !kvm_is_zone_device_pfn(pfn);
B
Ben-Ami Yassour 已提交
189 190 191 192

	return true;
}

A
Avi Kivity 已提交
193 194 195
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
196
void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
197
{
198
	int cpu = get_cpu();
199
	preempt_notifier_register(&vcpu->preempt_notifier);
200
	kvm_arch_vcpu_load(vcpu, cpu);
201
	put_cpu();
A
Avi Kivity 已提交
202
}
203
EXPORT_SYMBOL_GPL(vcpu_load);
A
Avi Kivity 已提交
204

205
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
206
{
207
	preempt_disable();
208
	kvm_arch_vcpu_put(vcpu);
209 210
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
211
}
212
EXPORT_SYMBOL_GPL(vcpu_put);
A
Avi Kivity 已提交
213

214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
/* TODO: merge with kvm_arch_vcpu_should_kick */
static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
{
	int mode = kvm_vcpu_exiting_guest_mode(vcpu);

	/*
	 * We need to wait for the VCPU to reenable interrupts and get out of
	 * READING_SHADOW_PAGE_TABLES mode.
	 */
	if (req & KVM_REQUEST_WAIT)
		return mode != OUTSIDE_GUEST_MODE;

	/*
	 * Need to kick a running VCPU, but otherwise there is nothing to do.
	 */
	return mode == IN_GUEST_MODE;
}

232 233 234 235
static void ack_flush(void *_completed)
{
}

236 237 238 239 240 241 242 243 244 245 246 247
static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
{
	if (unlikely(!cpus))
		cpus = cpu_online_mask;

	if (cpumask_empty(cpus))
		return false;

	smp_call_function_many(cpus, ack_flush, NULL, wait);
	return true;
}

248 249
bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
				 unsigned long *vcpu_bitmap, cpumask_var_t tmp)
250
{
251
	int i, cpu, me;
252
	struct kvm_vcpu *vcpu;
253
	bool called;
254

255
	me = get_cpu();
256

257
	kvm_for_each_vcpu(i, vcpu, kvm) {
258
		if (vcpu_bitmap && !test_bit(i, vcpu_bitmap))
259 260
			continue;

261
		kvm_make_request(req, vcpu);
262
		cpu = vcpu->cpu;
263

264 265
		if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
			continue;
266

267
		if (tmp != NULL && cpu != -1 && cpu != me &&
268
		    kvm_request_needs_ipi(vcpu, req))
269
			__cpumask_set_cpu(cpu, tmp);
270
	}
271 272

	called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT));
273
	put_cpu();
274 275 276 277 278 279 280 281 282 283 284

	return called;
}

bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
{
	cpumask_var_t cpus;
	bool called;

	zalloc_cpumask_var(&cpus, GFP_ATOMIC);

285
	called = kvm_make_vcpus_request_mask(kvm, req, NULL, cpus);
286

287
	free_cpumask_var(cpus);
288
	return called;
289 290
}

291
#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
292
void kvm_flush_remote_tlbs(struct kvm *kvm)
293
{
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
	/*
	 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
	 * kvm_make_all_cpus_request.
	 */
	long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);

	/*
	 * We want to publish modifications to the page tables before reading
	 * mode. Pairs with a memory barrier in arch-specific code.
	 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
	 * and smp_mb in walk_shadow_page_lockless_begin/end.
	 * - powerpc: smp_mb in kvmppc_prepare_to_enter.
	 *
	 * There is already an smp_mb__after_atomic() before
	 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
	 * barrier here.
	 */
311 312
	if (!kvm_arch_flush_remote_tlb(kvm)
	    || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
313
		++kvm->stat.remote_tlb_flush;
314
	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
315
}
316
EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
317
#endif
318

319 320
void kvm_reload_remote_mmus(struct kvm *kvm)
{
321
	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
322
}
323

324
static int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
R
Rusty Russell 已提交
325 326 327 328 329 330 331 332
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
333
	vcpu->pid = NULL;
334
	init_swait_queue_head(&vcpu->wq);
335
	kvm_async_pf_vcpu_init(vcpu);
R
Rusty Russell 已提交
336

337 338 339
	vcpu->pre_pcpu = -1;
	INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);

R
Rusty Russell 已提交
340 341 342 343 344 345 346
	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

347 348
	kvm_vcpu_set_in_spin_loop(vcpu, false);
	kvm_vcpu_set_dy_eligible(vcpu, false);
349
	vcpu->preempted = false;
350
	vcpu->ready = false;
351

352
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
353
	if (r < 0)
354
		goto fail_free_run;
R
Rusty Russell 已提交
355 356 357 358 359
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
360
	return r;
R
Rusty Russell 已提交
361 362
}

363
static void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
R
Rusty Russell 已提交
364
{
365 366 367 368 369 370
	/*
	 * no need for rcu_read_lock as VCPU_RUN is the only place that
	 * will change the vcpu->pid pointer and on uninit all file
	 * descriptors are already gone.
	 */
	put_pid(rcu_dereference_protected(vcpu->pid, 1));
371
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
372 373 374
	free_page((unsigned long)vcpu->run);
}

375 376 377
void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
{
	kvm_arch_vcpu_destroy(vcpu);
378 379 380

	kvm_vcpu_uninit(vcpu);
	kmem_cache_free(kvm_vcpu_cache, vcpu);
381 382 383
}
EXPORT_SYMBOL_GPL(kvm_vcpu_destroy);

384 385 386 387 388 389
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

390 391 392 393 394 395
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
396
	int idx;
397

398
	idx = srcu_read_lock(&kvm->srcu);
399 400
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
401 402 403 404

	if (kvm_set_spte_hva(kvm, address, pte))
		kvm_flush_remote_tlbs(kvm);

405
	spin_unlock(&kvm->mmu_lock);
406
	srcu_read_unlock(&kvm->srcu, idx);
407 408
}

409
static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
410
					const struct mmu_notifier_range *range)
411 412
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
413
	int need_tlb_flush = 0, idx;
414
	int ret;
415

416
	idx = srcu_read_lock(&kvm->srcu);
417 418 419 420 421 422 423
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
424
	need_tlb_flush = kvm_unmap_hva_range(kvm, range->start, range->end);
425
	need_tlb_flush |= kvm->tlbs_dirty;
426 427 428
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
429 430

	spin_unlock(&kvm->mmu_lock);
431

432
	ret = kvm_arch_mmu_notifier_invalidate_range(kvm, range->start,
433 434
					range->end,
					mmu_notifier_range_blockable(range));
435

436
	srcu_read_unlock(&kvm->srcu, idx);
437 438

	return ret;
439 440 441
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
442
					const struct mmu_notifier_range *range)
443 444 445 446 447 448 449 450 451 452
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
453
	smp_wmb();
454 455
	/*
	 * The above sequence increase must be visible before the
456 457
	 * below count decrease, which is ensured by the smp_wmb above
	 * in conjunction with the smp_rmb in mmu_notifier_retry().
458 459 460 461 462 463 464 465 466
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
A
Andres Lagar-Cavilla 已提交
467 468
					      unsigned long start,
					      unsigned long end)
469 470
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
471
	int young, idx;
472

473
	idx = srcu_read_lock(&kvm->srcu);
474 475
	spin_lock(&kvm->mmu_lock);

A
Andres Lagar-Cavilla 已提交
476
	young = kvm_age_hva(kvm, start, end);
477 478 479
	if (young)
		kvm_flush_remote_tlbs(kvm);

480 481 482
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

483 484 485
	return young;
}

486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long start,
					unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	/*
	 * Even though we do not flush TLB, this will still adversely
	 * affect performance on pre-Haswell Intel EPT, where there is
	 * no EPT Access Bit to clear so that we have to tear down EPT
	 * tables instead. If we find this unacceptable, we can always
	 * add a parameter to kvm_age_hva so that it effectively doesn't
	 * do anything on clear_young.
	 *
	 * Also note that currently we never issue secondary TLB flushes
	 * from clear_young, leaving this job up to the regular system
	 * cadence. If we find this inaccurate, we might come up with a
	 * more sophisticated heuristic later.
	 */
	young = kvm_age_hva(kvm, start, end);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

A
Andrea Arcangeli 已提交
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531
static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
				       struct mm_struct *mm,
				       unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	young = kvm_test_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

532 533 534 535
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
536 537 538
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
539
	kvm_arch_flush_shadow_all(kvm);
540
	srcu_read_unlock(&kvm->srcu, idx);
541 542
}

543 544 545 546
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
547
	.clear_young		= kvm_mmu_notifier_clear_young,
A
Andrea Arcangeli 已提交
548
	.test_young		= kvm_mmu_notifier_test_young,
549
	.change_pte		= kvm_mmu_notifier_change_pte,
550
	.release		= kvm_mmu_notifier_release,
551
};
552 553 554 555 556 557 558 559 560 561 562 563 564 565

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

566 567
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

568
static struct kvm_memslots *kvm_alloc_memslots(void)
569 570
{
	int i;
571
	struct kvm_memslots *slots;
572

573
	slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
574 575 576
	if (!slots)
		return NULL;

577
	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
578
		slots->id_to_index[i] = slots->memslots[i].id = i;
579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616

	return slots;
}

static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
	if (!memslot->dirty_bitmap)
		return;

	kvfree(memslot->dirty_bitmap);
	memslot->dirty_bitmap = NULL;
}

/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
			      struct kvm_memory_slot *dont)
{
	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		kvm_destroy_dirty_bitmap(free);

	kvm_arch_free_memslot(kvm, free, dont);

	free->npages = 0;
}

static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
{
	struct kvm_memory_slot *memslot;

	if (!slots)
		return;

	kvm_for_each_memslot(memslot, slots)
		kvm_free_memslot(kvm, memslot, NULL);

	kvfree(slots);
617 618
}

619 620 621 622 623 624 625 626 627
static void kvm_destroy_vm_debugfs(struct kvm *kvm)
{
	int i;

	if (!kvm->debugfs_dentry)
		return;

	debugfs_remove_recursive(kvm->debugfs_dentry);

628 629 630 631 632
	if (kvm->debugfs_stat_data) {
		for (i = 0; i < kvm_debugfs_num_entries; i++)
			kfree(kvm->debugfs_stat_data[i]);
		kfree(kvm->debugfs_stat_data);
	}
633 634 635 636 637 638 639 640 641 642 643 644
}

static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
{
	char dir_name[ITOA_MAX_LEN * 2];
	struct kvm_stat_data *stat_data;
	struct kvm_stats_debugfs_item *p;

	if (!debugfs_initialized())
		return 0;

	snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
645
	kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir);
646 647 648

	kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
					 sizeof(*kvm->debugfs_stat_data),
649
					 GFP_KERNEL_ACCOUNT);
650 651 652 653
	if (!kvm->debugfs_stat_data)
		return -ENOMEM;

	for (p = debugfs_entries; p->name; p++) {
654
		stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
655 656 657 658
		if (!stat_data)
			return -ENOMEM;

		stat_data->kvm = kvm;
659
		stat_data->dbgfs_item = p;
660
		kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
661 662 663
		debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p),
				    kvm->debugfs_dentry, stat_data,
				    &stat_fops_per_vm);
664 665 666 667
	}
	return 0;
}

668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
/*
 * Called after the VM is otherwise initialized, but just before adding it to
 * the vm_list.
 */
int __weak kvm_arch_post_init_vm(struct kvm *kvm)
{
	return 0;
}

/*
 * Called just after removing the VM from the vm_list, but before doing any
 * other destruction.
 */
void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm)
{
}

685
static struct kvm *kvm_create_vm(unsigned long type)
A
Avi Kivity 已提交
686
{
687
	struct kvm *kvm = kvm_arch_alloc_vm();
688 689
	int r = -ENOMEM;
	int i;
A
Avi Kivity 已提交
690

691 692 693
	if (!kvm)
		return ERR_PTR(-ENOMEM);

694
	spin_lock_init(&kvm->mmu_lock);
V
Vegard Nossum 已提交
695
	mmgrab(current->mm);
696 697 698 699 700 701 702
	kvm->mm = current->mm;
	kvm_eventfd_init(kvm);
	mutex_init(&kvm->lock);
	mutex_init(&kvm->irq_lock);
	mutex_init(&kvm->slots_lock);
	INIT_LIST_HEAD(&kvm->devices);

703 704
	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);

705 706 707 708 709
	if (init_srcu_struct(&kvm->srcu))
		goto out_err_no_srcu;
	if (init_srcu_struct(&kvm->irq_srcu))
		goto out_err_no_irq_srcu;

710
	refcount_set(&kvm->users_count, 1);
711
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
712
		struct kvm_memslots *slots = kvm_alloc_memslots();
713

714
		if (!slots)
715
			goto out_err_no_arch_destroy_vm;
716
		/* Generations must be different for each address space. */
717
		slots->generation = i;
718
		rcu_assign_pointer(kvm->memslots[i], slots);
719
	}
720

M
Marcelo Tosatti 已提交
721
	for (i = 0; i < KVM_NR_BUSES; i++) {
722
		rcu_assign_pointer(kvm->buses[i],
723
			kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
724
		if (!kvm->buses[i])
725
			goto out_err_no_arch_destroy_vm;
M
Marcelo Tosatti 已提交
726
	}
727

728
	r = kvm_arch_init_vm(kvm, type);
729
	if (r)
730
		goto out_err_no_arch_destroy_vm;
731 732 733

	r = hardware_enable_all();
	if (r)
734
		goto out_err_no_disable;
735

736
#ifdef CONFIG_HAVE_KVM_IRQFD
737
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
738
#endif
A
Avi Kivity 已提交
739

740
	r = kvm_init_mmu_notifier(kvm);
741 742 743 744
	if (r)
		goto out_err_no_mmu_notifier;

	r = kvm_arch_post_init_vm(kvm);
745 746 747
	if (r)
		goto out_err;

J
Junaid Shahid 已提交
748
	mutex_lock(&kvm_lock);
749
	list_add(&kvm->vm_list, &vm_list);
J
Junaid Shahid 已提交
750
	mutex_unlock(&kvm_lock);
751

752 753
	preempt_notifier_inc();

754
	return kvm;
755 756

out_err:
757 758 759 760 761
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	if (kvm->mmu_notifier.ops)
		mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
#endif
out_err_no_mmu_notifier:
762
	hardware_disable_all();
763
out_err_no_disable:
764 765
	kvm_arch_destroy_vm(kvm);
out_err_no_arch_destroy_vm:
766
	WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
M
Marcelo Tosatti 已提交
767
	for (i = 0; i < KVM_NR_BUSES; i++)
768
		kfree(kvm_get_bus(kvm, i));
769
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
770
		kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
771 772 773 774
	cleanup_srcu_struct(&kvm->irq_srcu);
out_err_no_irq_srcu:
	cleanup_srcu_struct(&kvm->srcu);
out_err_no_srcu:
775
	kvm_arch_free_vm(kvm);
776
	mmdrop(current->mm);
777
	return ERR_PTR(r);
778 779
}

780 781
static void kvm_destroy_devices(struct kvm *kvm)
{
G
Geliang Tang 已提交
782
	struct kvm_device *dev, *tmp;
783

784 785 786 787 788
	/*
	 * We do not need to take the kvm->lock here, because nobody else
	 * has a reference to the struct kvm at this point and therefore
	 * cannot access the devices list anyhow.
	 */
G
Geliang Tang 已提交
789 790
	list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
		list_del(&dev->vm_node);
791 792 793 794
		dev->ops->destroy(dev);
	}
}

795 796
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
797
	int i;
798 799
	struct mm_struct *mm = kvm->mm;

800
	kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
801
	kvm_destroy_vm_debugfs(kvm);
802
	kvm_arch_sync_events(kvm);
J
Junaid Shahid 已提交
803
	mutex_lock(&kvm_lock);
804
	list_del(&kvm->vm_list);
J
Junaid Shahid 已提交
805
	mutex_unlock(&kvm_lock);
806 807
	kvm_arch_pre_destroy_vm(kvm);

808
	kvm_free_irq_routing(kvm);
809
	for (i = 0; i < KVM_NR_BUSES; i++) {
810
		struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
811 812 813

		if (bus)
			kvm_io_bus_destroy(bus);
814 815
		kvm->buses[i] = NULL;
	}
816
	kvm_coalesced_mmio_free(kvm);
817 818
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
819
#else
820
	kvm_arch_flush_shadow_all(kvm);
821
#endif
822
	kvm_arch_destroy_vm(kvm);
823
	kvm_destroy_devices(kvm);
824
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
825
		kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
826
	cleanup_srcu_struct(&kvm->irq_srcu);
827 828
	cleanup_srcu_struct(&kvm->srcu);
	kvm_arch_free_vm(kvm);
829
	preempt_notifier_dec();
830
	hardware_disable_all();
831
	mmdrop(mm);
832 833
}

I
Izik Eidus 已提交
834 835
void kvm_get_kvm(struct kvm *kvm)
{
836
	refcount_inc(&kvm->users_count);
I
Izik Eidus 已提交
837 838 839 840 841
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
842
	if (refcount_dec_and_test(&kvm->users_count))
I
Izik Eidus 已提交
843 844 845 846
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);

847 848 849 850 851 852 853 854 855 856 857 858
/*
 * Used to put a reference that was taken on behalf of an object associated
 * with a user-visible file descriptor, e.g. a vcpu or device, if installation
 * of the new file descriptor fails and the reference cannot be transferred to
 * its final owner.  In such cases, the caller is still actively using @kvm and
 * will fail miserably if the refcount unexpectedly hits zero.
 */
void kvm_put_kvm_no_destroy(struct kvm *kvm)
{
	WARN_ON(refcount_dec_and_test(&kvm->users_count));
}
EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy);
I
Izik Eidus 已提交
859

860 861 862 863
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
864 865
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
866
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
867 868 869
	return 0;
}

870 871
/*
 * Allocation size is twice as large as the actual dirty bitmap size.
872
 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
873
 */
874 875
static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
{
876
	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
877

878
	memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);
879 880 881 882 883 884
	if (!memslot->dirty_bitmap)
		return -ENOMEM;

	return 0;
}

885
/*
886 887 888 889
 * Insert memslot and re-sort memslots based on their GFN,
 * so binary search could be used to lookup GFN.
 * Sorting algorithm takes advantage of having initially
 * sorted array and known changed memslot position.
890
 */
891
static void update_memslots(struct kvm_memslots *slots,
892 893
			    struct kvm_memory_slot *new,
			    enum kvm_mr_change change)
894
{
895 896
	int id = new->id;
	int i = slots->id_to_index[id];
897
	struct kvm_memory_slot *mslots = slots->memslots;
898

899
	WARN_ON(mslots[i].id != id);
900 901 902 903 904 905 906 907 908 909 910
	switch (change) {
	case KVM_MR_CREATE:
		slots->used_slots++;
		WARN_ON(mslots[i].npages || !new->npages);
		break;
	case KVM_MR_DELETE:
		slots->used_slots--;
		WARN_ON(new->npages || !mslots[i].npages);
		break;
	default:
		break;
911
	}
912

913
	while (i < KVM_MEM_SLOTS_NUM - 1 &&
914 915 916
	       new->base_gfn <= mslots[i + 1].base_gfn) {
		if (!mslots[i + 1].npages)
			break;
917 918 919 920
		mslots[i] = mslots[i + 1];
		slots->id_to_index[mslots[i].id] = i;
		i++;
	}
921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937

	/*
	 * The ">=" is needed when creating a slot with base_gfn == 0,
	 * so that it moves before all those with base_gfn == npages == 0.
	 *
	 * On the other hand, if new->npages is zero, the above loop has
	 * already left i pointing to the beginning of the empty part of
	 * mslots, and the ">=" would move the hole backwards in this
	 * case---which is wrong.  So skip the loop when deleting a slot.
	 */
	if (new->npages) {
		while (i > 0 &&
		       new->base_gfn >= mslots[i - 1].base_gfn) {
			mslots[i] = mslots[i - 1];
			slots->id_to_index[mslots[i].id] = i;
			i--;
		}
938 939
	} else
		WARN_ON_ONCE(i != slots->used_slots);
940

941 942
	mslots[i] = *new;
	slots->id_to_index[mslots[i].id] = i;
943 944
}

945
static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
946
{
X
Xiao Guangrong 已提交
947 948
	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;

949
#ifdef __KVM_HAVE_READONLY_MEM
X
Xiao Guangrong 已提交
950 951 952 953
	valid_flags |= KVM_MEM_READONLY;
#endif

	if (mem->flags & ~valid_flags)
954 955 956 957 958
		return -EINVAL;

	return 0;
}

959
static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
960
		int as_id, struct kvm_memslots *slots)
961
{
962
	struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
963
	u64 gen = old_memslots->generation;
964

965 966
	WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
	slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
967

968
	rcu_assign_pointer(kvm->memslots[as_id], slots);
969
	synchronize_srcu_expedited(&kvm->srcu);
970

971
	/*
972
	 * Increment the new memslot generation a second time, dropping the
M
Miaohe Lin 已提交
973
	 * update in-progress flag and incrementing the generation based on
974 975 976 977 978 979
	 * the number of address spaces.  This provides a unique and easily
	 * identifiable generation number while the memslots are in flux.
	 */
	gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;

	/*
980 981 982
	 * Generations must be unique even across address spaces.  We do not need
	 * a global counter for that, instead the generation space is evenly split
	 * across address spaces.  For example, with two address spaces, address
983 984
	 * space 0 will use generations 0, 2, 4, ... while address space 1 will
	 * use generations 1, 3, 5, ...
985
	 */
986
	gen += KVM_ADDRESS_SPACE_NUM;
987

988
	kvm_arch_memslots_updated(kvm, gen);
989

990
	slots->generation = gen;
991 992

	return old_memslots;
993 994
}

A
Avi Kivity 已提交
995 996 997 998 999
/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
1000
 *
1001
 * Must be called holding kvm->slots_lock for write.
A
Avi Kivity 已提交
1002
 */
1003
int __kvm_set_memory_region(struct kvm *kvm,
1004
			    const struct kvm_userspace_memory_region *mem)
A
Avi Kivity 已提交
1005
{
1006
	int r;
A
Avi Kivity 已提交
1007
	gfn_t base_gfn;
1008
	unsigned long npages;
1009
	struct kvm_memory_slot *slot;
A
Avi Kivity 已提交
1010
	struct kvm_memory_slot old, new;
1011
	struct kvm_memslots *slots = NULL, *old_memslots;
1012
	int as_id, id;
1013
	enum kvm_mr_change change;
A
Avi Kivity 已提交
1014

1015 1016 1017 1018
	r = check_memory_region_flags(mem);
	if (r)
		goto out;

A
Avi Kivity 已提交
1019
	r = -EINVAL;
1020 1021 1022
	as_id = mem->slot >> 16;
	id = (u16)mem->slot;

A
Avi Kivity 已提交
1023 1024 1025 1026 1027
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
1028
	/* We can read the guest memory with __xxx_user() later on. */
1029
	if ((id < KVM_USER_MEM_SLOTS) &&
1030
	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
1031
	     !access_ok((void __user *)(unsigned long)mem->userspace_addr,
1032
			mem->memory_size)))
1033
		goto out;
1034
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
A
Avi Kivity 已提交
1035 1036 1037 1038
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

1039
	slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
A
Avi Kivity 已提交
1040 1041 1042
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

1043 1044 1045
	if (npages > KVM_MEM_MAX_NR_PAGES)
		goto out;

1046
	new = old = *slot;
A
Avi Kivity 已提交
1047

1048
	new.id = id;
A
Avi Kivity 已提交
1049 1050 1051 1052
	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

1053 1054 1055 1056 1057
	if (npages) {
		if (!old.npages)
			change = KVM_MR_CREATE;
		else { /* Modify an existing slot. */
			if ((mem->userspace_addr != old.userspace_addr) ||
1058 1059
			    (npages != old.npages) ||
			    ((new.flags ^ old.flags) & KVM_MEM_READONLY))
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
				goto out;

			if (base_gfn != old.base_gfn)
				change = KVM_MR_MOVE;
			else if (new.flags != old.flags)
				change = KVM_MR_FLAGS_ONLY;
			else { /* Nothing to change. */
				r = 0;
				goto out;
			}
		}
1071 1072 1073 1074
	} else {
		if (!old.npages)
			goto out;

1075
		change = KVM_MR_DELETE;
1076 1077 1078
		new.base_gfn = 0;
		new.flags = 0;
	}
A
Avi Kivity 已提交
1079

1080
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
1081 1082
		/* Check for overlaps */
		r = -EEXIST;
1083
		kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
1084
			if (slot->id == id)
1085 1086 1087 1088 1089
				continue;
			if (!((base_gfn + npages <= slot->base_gfn) ||
			      (base_gfn >= slot->base_gfn + slot->npages)))
				goto out;
		}
A
Avi Kivity 已提交
1090 1091 1092 1093
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
1094
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
1095 1096

	r = -ENOMEM;
1097
	if (change == KVM_MR_CREATE) {
1098
		new.userspace_addr = mem->userspace_addr;
1099

1100
		if (kvm_arch_create_memslot(kvm, &new, npages))
1101
			goto out_free;
A
Avi Kivity 已提交
1102
	}
1103

A
Avi Kivity 已提交
1104 1105
	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
1106
		if (kvm_create_dirty_bitmap(&new) < 0)
1107
			goto out_free;
A
Avi Kivity 已提交
1108 1109
	}

1110
	slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
1111 1112
	if (!slots)
		goto out_free;
1113
	memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
1114

1115
	if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
1116
		slot = id_to_memslot(slots, id);
1117 1118
		slot->flags |= KVM_MEMSLOT_INVALID;

1119
		old_memslots = install_new_memslots(kvm, as_id, slots);
1120

1121 1122
		/* From this point no new shadow pages pointing to a deleted,
		 * or moved, memslot will be created.
1123 1124
		 *
		 * validation of sp->gfn happens in:
1125
		 *	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1126
		 *	- kvm_is_visible_gfn (mmu_check_root)
1127
		 */
1128
		kvm_arch_flush_shadow_memslot(kvm, slot);
1129 1130 1131 1132 1133 1134

		/*
		 * We can re-use the old_memslots from above, the only difference
		 * from the currently installed memslots is the invalid flag.  This
		 * will get overwritten by update_memslots anyway.
		 */
1135
		slots = old_memslots;
1136
	}
1137

1138
	r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
1139
	if (r)
1140
		goto out_slots;
1141

1142
	/* actual memory is freed via old in kvm_free_memslot below */
1143
	if (change == KVM_MR_DELETE) {
1144
		new.dirty_bitmap = NULL;
1145
		memset(&new.arch, 0, sizeof(new.arch));
1146 1147
	}

1148
	update_memslots(slots, &new, change);
1149
	old_memslots = install_new_memslots(kvm, as_id, slots);
1150

1151
	kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
1152

1153
	kvm_free_memslot(kvm, &old, &new);
1154
	kvfree(old_memslots);
A
Avi Kivity 已提交
1155 1156
	return 0;

1157
out_slots:
1158
	kvfree(slots);
1159
out_free:
1160
	kvm_free_memslot(kvm, &new, &old);
A
Avi Kivity 已提交
1161 1162
out:
	return r;
1163
}
1164 1165 1166
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
1167
			  const struct kvm_userspace_memory_region *mem)
1168 1169 1170
{
	int r;

1171
	mutex_lock(&kvm->slots_lock);
1172
	r = __kvm_set_memory_region(kvm, mem);
1173
	mutex_unlock(&kvm->slots_lock);
1174 1175
	return r;
}
1176 1177
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

1178 1179
static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
					  struct kvm_userspace_memory_region *mem)
1180
{
1181
	if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1182
		return -EINVAL;
1183

1184
	return kvm_set_memory_region(kvm, mem);
A
Avi Kivity 已提交
1185 1186
}

1187 1188
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
1189
{
1190
	struct kvm_memslots *slots;
A
Avi Kivity 已提交
1191
	struct kvm_memory_slot *memslot;
1192
	int i, as_id, id;
1193
	unsigned long n;
A
Avi Kivity 已提交
1194 1195
	unsigned long any = 0;

1196 1197 1198
	as_id = log->slot >> 16;
	id = (u16)log->slot;
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1199
		return -EINVAL;
A
Avi Kivity 已提交
1200

1201 1202
	slots = __kvm_memslots(kvm, as_id);
	memslot = id_to_memslot(slots, id);
A
Avi Kivity 已提交
1203
	if (!memslot->dirty_bitmap)
1204
		return -ENOENT;
A
Avi Kivity 已提交
1205

1206
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
1207

1208
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
1209 1210 1211
		any = memslot->dirty_bitmap[i];

	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1212
		return -EFAULT;
A
Avi Kivity 已提交
1213

1214 1215
	if (any)
		*is_dirty = 1;
1216
	return 0;
A
Avi Kivity 已提交
1217
}
1218
EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
A
Avi Kivity 已提交
1219

1220 1221
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
/**
J
Jiang Biao 已提交
1222
 * kvm_get_dirty_log_protect - get a snapshot of dirty pages
1223
 *	and reenable dirty page tracking for the corresponding pages.
1224 1225
 * @kvm:	pointer to kvm instance
 * @log:	slot id and address to which we copy the log
J
Jiang Biao 已提交
1226
 * @flush:	true if TLB flush is needed by caller
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
 *
 * We need to keep it in mind that VCPU threads can write to the bitmap
 * concurrently. So, to avoid losing track of dirty pages we keep the
 * following order:
 *
 *    1. Take a snapshot of the bit and clear it if needed.
 *    2. Write protect the corresponding page.
 *    3. Copy the snapshot to the userspace.
 *    4. Upon return caller flushes TLB's if needed.
 *
 * Between 2 and 4, the guest may write to the page using the remaining TLB
 * entry.  This is not a problem because the page is reported dirty using
 * the snapshot taken before and step 4 ensures that writes done after
 * exiting to userspace will be logged for the next call.
 *
 */
int kvm_get_dirty_log_protect(struct kvm *kvm,
1244
			struct kvm_dirty_log *log, bool *flush)
1245
{
1246
	struct kvm_memslots *slots;
1247
	struct kvm_memory_slot *memslot;
1248
	int i, as_id, id;
1249 1250 1251 1252
	unsigned long n;
	unsigned long *dirty_bitmap;
	unsigned long *dirty_bitmap_buffer;

1253 1254 1255
	as_id = log->slot >> 16;
	id = (u16)log->slot;
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1256
		return -EINVAL;
1257

1258 1259
	slots = __kvm_memslots(kvm, as_id);
	memslot = id_to_memslot(slots, id);
1260 1261 1262

	dirty_bitmap = memslot->dirty_bitmap;
	if (!dirty_bitmap)
1263
		return -ENOENT;
1264 1265

	n = kvm_dirty_bitmap_bytes(memslot);
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
	*flush = false;
	if (kvm->manual_dirty_log_protect) {
		/*
		 * Unlike kvm_get_dirty_log, we always return false in *flush,
		 * because no flush is needed until KVM_CLEAR_DIRTY_LOG.  There
		 * is some code duplication between this function and
		 * kvm_get_dirty_log, but hopefully all architecture
		 * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
		 * can be eliminated.
		 */
		dirty_bitmap_buffer = dirty_bitmap;
	} else {
		dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
		memset(dirty_bitmap_buffer, 0, n);
1280

1281 1282 1283 1284
		spin_lock(&kvm->mmu_lock);
		for (i = 0; i < n / sizeof(long); i++) {
			unsigned long mask;
			gfn_t offset;
1285

1286 1287 1288 1289 1290 1291 1292
			if (!dirty_bitmap[i])
				continue;

			*flush = true;
			mask = xchg(&dirty_bitmap[i], 0);
			dirty_bitmap_buffer[i] = mask;

1293 1294 1295
			offset = i * BITS_PER_LONG;
			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
								offset, mask);
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
		}
		spin_unlock(&kvm->mmu_lock);
	}

	if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);

/**
 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
 *	and reenable dirty page tracking for the corresponding pages.
 * @kvm:	pointer to kvm instance
 * @log:	slot id and address from which to fetch the bitmap of dirty pages
J
Jiang Biao 已提交
1311
 * @flush:	true if TLB flush is needed by caller
1312 1313 1314 1315 1316 1317
 */
int kvm_clear_dirty_log_protect(struct kvm *kvm,
				struct kvm_clear_dirty_log *log, bool *flush)
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
1318
	int as_id, id;
1319
	gfn_t offset;
1320
	unsigned long i, n;
1321 1322 1323 1324 1325 1326 1327 1328
	unsigned long *dirty_bitmap;
	unsigned long *dirty_bitmap_buffer;

	as_id = log->slot >> 16;
	id = (u16)log->slot;
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
		return -EINVAL;

1329
	if (log->first_page & 63)
1330 1331 1332 1333 1334 1335 1336 1337 1338
		return -EINVAL;

	slots = __kvm_memslots(kvm, as_id);
	memslot = id_to_memslot(slots, id);

	dirty_bitmap = memslot->dirty_bitmap;
	if (!dirty_bitmap)
		return -ENOENT;

1339
	n = ALIGN(log->num_pages, BITS_PER_LONG) / 8;
1340 1341

	if (log->first_page > memslot->npages ||
1342 1343 1344
	    log->num_pages > memslot->npages - log->first_page ||
	    (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))
	    return -EINVAL;
1345

1346
	*flush = false;
1347 1348 1349
	dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
	if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
		return -EFAULT;
1350

1351
	spin_lock(&kvm->mmu_lock);
1352 1353
	for (offset = log->first_page, i = offset / BITS_PER_LONG,
		 n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
1354 1355 1356 1357
	     i++, offset += BITS_PER_LONG) {
		unsigned long mask = *dirty_bitmap_buffer++;
		atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
		if (!mask)
1358 1359
			continue;

1360
		mask &= atomic_long_fetch_andnot(mask, p);
1361

1362 1363 1364 1365 1366 1367
		/*
		 * mask contains the bits that really have been cleared.  This
		 * never includes any bits beyond the length of the memslot (if
		 * the length is not aligned to 64 pages), therefore it is not
		 * a problem if userspace sets them in log->dirty_bitmap.
		*/
1368
		if (mask) {
1369
			*flush = true;
1370 1371 1372
			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
								offset, mask);
		}
1373 1374
	}
	spin_unlock(&kvm->mmu_lock);
1375

1376
	return 0;
1377
}
1378
EXPORT_SYMBOL_GPL(kvm_clear_dirty_log_protect);
1379 1380
#endif

1381 1382 1383 1384 1385
bool kvm_largepages_enabled(void)
{
	return largepages_enabled;
}

1386 1387 1388 1389 1390 1391
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

1392 1393 1394 1395
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
A
Avi Kivity 已提交
1396
EXPORT_SYMBOL_GPL(gfn_to_memslot);
A
Avi Kivity 已提交
1397

1398 1399 1400 1401 1402
struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
}

1403
bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1404
{
1405
	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1406

1407
	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1408
	      memslot->flags & KVM_MEMSLOT_INVALID)
1409
		return false;
1410

1411
	return true;
1412 1413 1414
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

J
Joerg Roedel 已提交
1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

X
Xiao Guangrong 已提交
1439 1440 1441 1442 1443 1444 1445
static bool memslot_is_readonly(struct kvm_memory_slot *slot)
{
	return slot->flags & KVM_MEM_READONLY;
}

static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				       gfn_t *nr_pages, bool write)
I
Izik Eidus 已提交
1446
{
1447
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
X
Xiao Guangrong 已提交
1448
		return KVM_HVA_ERR_BAD;
1449

X
Xiao Guangrong 已提交
1450 1451
	if (memslot_is_readonly(slot) && write)
		return KVM_HVA_ERR_RO_BAD;
1452 1453 1454 1455

	if (nr_pages)
		*nr_pages = slot->npages - (gfn - slot->base_gfn);

X
Xiao Guangrong 已提交
1456
	return __gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
1457
}
1458

X
Xiao Guangrong 已提交
1459 1460 1461 1462
static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				     gfn_t *nr_pages)
{
	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
I
Izik Eidus 已提交
1463
}
1464

X
Xiao Guangrong 已提交
1465
unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1466
					gfn_t gfn)
X
Xiao Guangrong 已提交
1467 1468 1469 1470 1471
{
	return gfn_to_hva_many(slot, gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);

1472 1473
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
1474
	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1475
}
1476
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1477

1478 1479 1480 1481 1482 1483
unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);

1484
/*
1485 1486 1487 1488 1489 1490
 * Return the hva of a @gfn and the R/W attribute if possible.
 *
 * @slot: the kvm_memory_slot which contains @gfn
 * @gfn: the gfn to be translated
 * @writable: used to return the read/write attribute of the @slot if the hva
 * is valid and @writable is not NULL
1491
 */
1492 1493
unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
				      gfn_t gfn, bool *writable)
1494
{
1495 1496 1497
	unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);

	if (!kvm_is_error_hva(hva) && writable)
1498 1499
		*writable = !memslot_is_readonly(slot);

1500
	return hva;
1501 1502
}

1503 1504 1505 1506 1507 1508 1509
unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

1510 1511 1512 1513 1514 1515 1516
unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
{
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

1517 1518
static inline int check_user_page_hwpoison(unsigned long addr)
{
L
Lorenzo Stoakes 已提交
1519
	int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
1520

L
Lorenzo Stoakes 已提交
1521
	rc = get_user_pages(addr, 1, flags, NULL, NULL);
1522 1523 1524
	return rc == -EHWPOISON;
}

X
Xiao Guangrong 已提交
1525
/*
1526 1527
 * The fast path to get the writable pfn which will be stored in @pfn,
 * true indicates success, otherwise false is returned.  It's also the
M
Miaohe Lin 已提交
1528
 * only part that runs if we can in atomic context.
X
Xiao Guangrong 已提交
1529
 */
1530 1531
static bool hva_to_pfn_fast(unsigned long addr, bool write_fault,
			    bool *writable, kvm_pfn_t *pfn)
A
Avi Kivity 已提交
1532
{
1533
	struct page *page[1];
X
Xiao Guangrong 已提交
1534
	int npages;
A
Avi Kivity 已提交
1535

1536 1537 1538 1539 1540 1541 1542
	/*
	 * Fast pin a writable pfn only if it is a write fault request
	 * or the caller allows to map a writable pfn for a read fault
	 * request.
	 */
	if (!(write_fault || writable))
		return false;
1543

X
Xiao Guangrong 已提交
1544 1545 1546
	npages = __get_user_pages_fast(addr, 1, 1, page);
	if (npages == 1) {
		*pfn = page_to_pfn(page[0]);
1547

X
Xiao Guangrong 已提交
1548 1549 1550 1551
		if (writable)
			*writable = true;
		return true;
	}
1552

X
Xiao Guangrong 已提交
1553 1554
	return false;
}
1555

X
Xiao Guangrong 已提交
1556 1557 1558 1559 1560
/*
 * The slow path to get the pfn of the specified host virtual address,
 * 1 indicates success, -errno is returned if error is detected.
 */
static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
D
Dan Williams 已提交
1561
			   bool *writable, kvm_pfn_t *pfn)
X
Xiao Guangrong 已提交
1562
{
1563 1564
	unsigned int flags = FOLL_HWPOISON;
	struct page *page;
X
Xiao Guangrong 已提交
1565
	int npages = 0;
1566

X
Xiao Guangrong 已提交
1567 1568 1569 1570 1571
	might_sleep();

	if (writable)
		*writable = write_fault;

1572 1573 1574 1575
	if (write_fault)
		flags |= FOLL_WRITE;
	if (async)
		flags |= FOLL_NOWAIT;
1576

1577
	npages = get_user_pages_unlocked(addr, 1, &page, flags);
X
Xiao Guangrong 已提交
1578 1579 1580 1581
	if (npages != 1)
		return npages;

	/* map read fault as writable if possible */
1582
	if (unlikely(!write_fault) && writable) {
1583
		struct page *wpage;
X
Xiao Guangrong 已提交
1584

1585
		if (__get_user_pages_fast(addr, 1, 1, &wpage) == 1) {
X
Xiao Guangrong 已提交
1586
			*writable = true;
1587 1588
			put_page(page);
			page = wpage;
1589
		}
1590
	}
1591
	*pfn = page_to_pfn(page);
X
Xiao Guangrong 已提交
1592 1593
	return npages;
}
I
Izik Eidus 已提交
1594

X
Xiao Guangrong 已提交
1595 1596 1597 1598
static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
{
	if (unlikely(!(vma->vm_flags & VM_READ)))
		return false;
1599

X
Xiao Guangrong 已提交
1600 1601
	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
		return false;
1602

X
Xiao Guangrong 已提交
1603 1604
	return true;
}
1605

1606 1607
static int hva_to_pfn_remapped(struct vm_area_struct *vma,
			       unsigned long addr, bool *async,
1608 1609
			       bool write_fault, bool *writable,
			       kvm_pfn_t *p_pfn)
1610
{
1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
	unsigned long pfn;
	int r;

	r = follow_pfn(vma, addr, &pfn);
	if (r) {
		/*
		 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
		 * not call the fault handler, so do it here.
		 */
		bool unlocked = false;
		r = fixup_user_fault(current, current->mm, addr,
				     (write_fault ? FAULT_FLAG_WRITE : 0),
				     &unlocked);
		if (unlocked)
			return -EAGAIN;
		if (r)
			return r;

		r = follow_pfn(vma, addr, &pfn);
		if (r)
			return r;

	}

1635 1636
	if (writable)
		*writable = true;
1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651

	/*
	 * Get a reference here because callers of *hva_to_pfn* and
	 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
	 * returned pfn.  This is only needed if the VMA has VM_MIXEDMAP
	 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
	 * simply do nothing for reserved pfns.
	 *
	 * Whoever called remap_pfn_range is also going to call e.g.
	 * unmap_mapping_range before the underlying pages are freed,
	 * causing a call to our MMU notifier.
	 */ 
	kvm_get_pfn(pfn);

	*p_pfn = pfn;
1652 1653 1654
	return 0;
}

1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
/*
 * Pin guest page in memory and return its pfn.
 * @addr: host virtual address which maps memory to the guest
 * @atomic: whether this function can sleep
 * @async: whether this function need to wait IO complete if the
 *         host page is not in the memory
 * @write_fault: whether we should get a writable host page
 * @writable: whether it allows to map a writable host page for !@write_fault
 *
 * The function will map a writable host page for these two cases:
 * 1): @write_fault = true
 * 2): @write_fault = false && @writable, @writable will tell the caller
 *     whether the mapping is writable.
 */
D
Dan Williams 已提交
1669
static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
X
Xiao Guangrong 已提交
1670 1671 1672
			bool write_fault, bool *writable)
{
	struct vm_area_struct *vma;
D
Dan Williams 已提交
1673
	kvm_pfn_t pfn = 0;
1674
	int npages, r;
1675

X
Xiao Guangrong 已提交
1676 1677
	/* we can do it either atomically or asynchronously, not both */
	BUG_ON(atomic && async);
1678

1679
	if (hva_to_pfn_fast(addr, write_fault, writable, &pfn))
X
Xiao Guangrong 已提交
1680 1681 1682 1683 1684 1685 1686 1687
		return pfn;

	if (atomic)
		return KVM_PFN_ERR_FAULT;

	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
	if (npages == 1)
		return pfn;
1688

X
Xiao Guangrong 已提交
1689 1690 1691 1692 1693 1694 1695
	down_read(&current->mm->mmap_sem);
	if (npages == -EHWPOISON ||
	      (!async && check_user_page_hwpoison(addr))) {
		pfn = KVM_PFN_ERR_HWPOISON;
		goto exit;
	}

1696
retry:
X
Xiao Guangrong 已提交
1697 1698 1699 1700
	vma = find_vma_intersection(current->mm, addr, addr + 1);

	if (vma == NULL)
		pfn = KVM_PFN_ERR_FAULT;
1701
	else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
1702
		r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
1703 1704
		if (r == -EAGAIN)
			goto retry;
1705 1706
		if (r < 0)
			pfn = KVM_PFN_ERR_FAULT;
X
Xiao Guangrong 已提交
1707
	} else {
X
Xiao Guangrong 已提交
1708
		if (async && vma_is_valid(vma, write_fault))
X
Xiao Guangrong 已提交
1709 1710 1711 1712 1713
			*async = true;
		pfn = KVM_PFN_ERR_FAULT;
	}
exit:
	up_read(&current->mm->mmap_sem);
1714
	return pfn;
1715 1716
}

D
Dan Williams 已提交
1717 1718 1719
kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
			       bool atomic, bool *async, bool write_fault,
			       bool *writable)
1720
{
X
Xiao Guangrong 已提交
1721 1722
	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);

1723 1724 1725
	if (addr == KVM_HVA_ERR_RO_BAD) {
		if (writable)
			*writable = false;
X
Xiao Guangrong 已提交
1726
		return KVM_PFN_ERR_RO_FAULT;
1727
	}
X
Xiao Guangrong 已提交
1728

1729 1730 1731
	if (kvm_is_error_hva(addr)) {
		if (writable)
			*writable = false;
1732
		return KVM_PFN_NOSLOT;
1733
	}
X
Xiao Guangrong 已提交
1734 1735 1736 1737 1738 1739 1740 1741 1742

	/* Do not map writable pfn in the readonly memslot. */
	if (writable && memslot_is_readonly(slot)) {
		*writable = false;
		writable = NULL;
	}

	return hva_to_pfn(addr, atomic, async, write_fault,
			  writable);
1743
}
1744
EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1745

D
Dan Williams 已提交
1746
kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1747 1748
		      bool *writable)
{
P
Paolo Bonzini 已提交
1749 1750
	return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
				    write_fault, writable);
1751 1752 1753
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);

D
Dan Williams 已提交
1754
kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1755
{
X
Xiao Guangrong 已提交
1756
	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1757
}
P
Paolo Bonzini 已提交
1758
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1759

D
Dan Williams 已提交
1760
kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1761
{
X
Xiao Guangrong 已提交
1762
	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1763
}
1764
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1765

D
Dan Williams 已提交
1766
kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
P
Paolo Bonzini 已提交
1767 1768 1769 1770 1771
{
	return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);

D
Dan Williams 已提交
1772
kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1773 1774 1775 1776 1777
{
	return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);

D
Dan Williams 已提交
1778
kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
P
Paolo Bonzini 已提交
1779 1780 1781 1782 1783
{
	return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn);

D
Dan Williams 已提交
1784
kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1785 1786 1787 1788 1789
{
	return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);

1790 1791
int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
			    struct page **pages, int nr_pages)
1792 1793
{
	unsigned long addr;
1794
	gfn_t entry = 0;
1795

1796
	addr = gfn_to_hva_many(slot, gfn, &entry);
1797 1798 1799 1800 1801 1802 1803 1804 1805 1806
	if (kvm_is_error_hva(addr))
		return -1;

	if (entry < nr_pages)
		return 0;

	return __get_user_pages_fast(addr, nr_pages, 1, pages);
}
EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);

D
Dan Williams 已提交
1807
static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
X
Xiao Guangrong 已提交
1808
{
1809
	if (is_error_noslot_pfn(pfn))
1810
		return KVM_ERR_PTR_BAD_PAGE;
X
Xiao Guangrong 已提交
1811

1812
	if (kvm_is_reserved_pfn(pfn)) {
1813
		WARN_ON(1);
1814
		return KVM_ERR_PTR_BAD_PAGE;
1815
	}
X
Xiao Guangrong 已提交
1816 1817 1818 1819

	return pfn_to_page(pfn);
}

1820 1821
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
D
Dan Williams 已提交
1822
	kvm_pfn_t pfn;
1823 1824 1825

	pfn = gfn_to_pfn(kvm, gfn);

X
Xiao Guangrong 已提交
1826
	return kvm_pfn_to_page(pfn);
A
Avi Kivity 已提交
1827 1828 1829
}
EXPORT_SYMBOL_GPL(gfn_to_page);

1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846
static int __kvm_map_gfn(struct kvm_memory_slot *slot, gfn_t gfn,
			 struct kvm_host_map *map)
{
	kvm_pfn_t pfn;
	void *hva = NULL;
	struct page *page = KVM_UNMAPPED_PAGE;

	if (!map)
		return -EINVAL;

	pfn = gfn_to_pfn_memslot(slot, gfn);
	if (is_error_noslot_pfn(pfn))
		return -EINVAL;

	if (pfn_valid(pfn)) {
		page = pfn_to_page(pfn);
		hva = kmap(page);
P
Paolo Bonzini 已提交
1847
#ifdef CONFIG_HAS_IOMEM
1848 1849
	} else {
		hva = memremap(pfn_to_hpa(pfn), PAGE_SIZE, MEMREMAP_WB);
P
Paolo Bonzini 已提交
1850
#endif
1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
	}

	if (!hva)
		return -EFAULT;

	map->page = page;
	map->hva = hva;
	map->pfn = pfn;
	map->gfn = gfn;

	return 0;
}

int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map)
{
	return __kvm_map_gfn(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, map);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_map);

void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map,
		    bool dirty)
{
	if (!map)
		return;

	if (!map->hva)
		return;

1879
	if (map->page != KVM_UNMAPPED_PAGE)
1880
		kunmap(map->page);
1881
#ifdef CONFIG_HAS_IOMEM
1882 1883
	else
		memunmap(map->hva);
1884
#endif
1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897

	if (dirty) {
		kvm_vcpu_mark_page_dirty(vcpu, map->gfn);
		kvm_release_pfn_dirty(map->pfn);
	} else {
		kvm_release_pfn_clean(map->pfn);
	}

	map->hva = NULL;
	map->page = NULL;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_unmap);

1898 1899
struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
{
D
Dan Williams 已提交
1900
	kvm_pfn_t pfn;
1901 1902 1903 1904 1905 1906 1907

	pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);

	return kvm_pfn_to_page(pfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);

1908 1909
void kvm_release_page_clean(struct page *page)
{
1910 1911
	WARN_ON(is_error_page(page));

1912
	kvm_release_pfn_clean(page_to_pfn(page));
1913 1914 1915
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

D
Dan Williams 已提交
1916
void kvm_release_pfn_clean(kvm_pfn_t pfn)
1917
{
1918
	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1919
		put_page(pfn_to_page(pfn));
1920 1921 1922
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1923
void kvm_release_page_dirty(struct page *page)
1924
{
X
Xiao Guangrong 已提交
1925 1926
	WARN_ON(is_error_page(page));

1927 1928 1929 1930
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

1931
void kvm_release_pfn_dirty(kvm_pfn_t pfn)
1932 1933 1934 1935
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
1936
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1937

D
Dan Williams 已提交
1938
void kvm_set_pfn_dirty(kvm_pfn_t pfn)
1939
{
1940 1941
	if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn))
		SetPageDirty(pfn_to_page(pfn));
1942
}
1943 1944
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

D
Dan Williams 已提交
1945
void kvm_set_pfn_accessed(kvm_pfn_t pfn)
1946
{
1947
	if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn))
1948
		mark_page_accessed(pfn_to_page(pfn));
1949 1950 1951
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

D
Dan Williams 已提交
1952
void kvm_get_pfn(kvm_pfn_t pfn)
1953
{
1954
	if (!kvm_is_reserved_pfn(pfn))
1955
		get_page(pfn_to_page(pfn));
1956 1957
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1958

1959 1960 1961 1962 1963 1964 1965 1966
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

1967 1968
static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
				 void *data, int offset, int len)
1969
{
1970 1971
	int r;
	unsigned long addr;
1972

1973
	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1974 1975
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1976
	r = __copy_from_user(data, (void __user *)addr + offset, len);
1977
	if (r)
1978 1979 1980
		return -EFAULT;
	return 0;
}
1981 1982 1983 1984 1985 1986 1987 1988

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return __kvm_read_guest_page(slot, gfn, data, offset, len);
}
1989 1990
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

1991 1992 1993 1994 1995 1996 1997 1998 1999
int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
			     int offset, int len)
{
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	return __kvm_read_guest_page(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);

2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019
int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

2020
int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
2021 2022
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
2023
	int seg;
2024
	int offset = offset_in_page(gpa);
2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
2039

2040 2041 2042 2043 2044 2045 2046
static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
			           void *data, int offset, unsigned long len)
{
	int r;
	unsigned long addr;

	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
2047 2048
	if (kvm_is_error_hva(addr))
		return -EFAULT;
2049
	pagefault_disable();
2050
	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
2051
	pagefault_enable();
2052 2053 2054 2055 2056
	if (r)
		return -EFAULT;
	return 0;
}

2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
	int offset = offset_in_page(gpa);

	return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);

int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
			       void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
	int offset = offset_in_page(gpa);

	return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);

static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
			          const void *data, int offset, int len)
2081
{
2082 2083
	int r;
	unsigned long addr;
2084

2085
	addr = gfn_to_hva_memslot(memslot, gfn);
2086 2087
	if (kvm_is_error_hva(addr))
		return -EFAULT;
2088
	r = __copy_to_user((void __user *)addr + offset, data, len);
2089
	if (r)
2090
		return -EFAULT;
2091
	mark_page_dirty_in_slot(memslot, gfn);
2092 2093
	return 0;
}
2094 2095 2096 2097 2098 2099 2100 2101

int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
			 const void *data, int offset, int len)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return __kvm_write_guest_page(slot, gfn, data, offset, len);
}
2102 2103
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

2104 2105 2106 2107 2108 2109 2110 2111 2112
int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
			      const void *data, int offset, int len)
{
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	return __kvm_write_guest_page(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);

2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131
int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
2132
EXPORT_SYMBOL_GPL(kvm_write_guest);
2133

2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154
int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
		         unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);

2155 2156 2157
static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
				       struct gfn_to_hva_cache *ghc,
				       gpa_t gpa, unsigned long len)
2158 2159
{
	int offset = offset_in_page(gpa);
2160 2161 2162 2163
	gfn_t start_gfn = gpa >> PAGE_SHIFT;
	gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
	gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
	gfn_t nr_pages_avail;
2164
	int r = start_gfn <= end_gfn ? 0 : -EINVAL;
2165 2166 2167

	ghc->gpa = gpa;
	ghc->generation = slots->generation;
2168
	ghc->len = len;
2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185
	ghc->hva = KVM_HVA_ERR_BAD;

	/*
	 * If the requested region crosses two memslots, we still
	 * verify that the entire region is valid here.
	 */
	while (!r && start_gfn <= end_gfn) {
		ghc->memslot = __gfn_to_memslot(slots, start_gfn);
		ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
					   &nr_pages_avail);
		if (kvm_is_error_hva(ghc->hva))
			r = -EFAULT;
		start_gfn += nr_pages_avail;
	}

	/* Use the slow path for cross page reads and writes. */
	if (!r && nr_pages_needed == 1)
2186
		ghc->hva += offset;
2187
	else
2188
		ghc->memslot = NULL;
2189 2190

	return r;
2191
}
2192

2193
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2194 2195
			      gpa_t gpa, unsigned long len)
{
2196
	struct kvm_memslots *slots = kvm_memslots(kvm);
2197 2198
	return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
}
2199
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
2200

2201
int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2202 2203
				  void *data, unsigned int offset,
				  unsigned long len)
2204
{
2205
	struct kvm_memslots *slots = kvm_memslots(kvm);
2206
	int r;
2207
	gpa_t gpa = ghc->gpa + offset;
2208

2209
	BUG_ON(len + offset > ghc->len);
2210

2211
	if (slots->generation != ghc->generation)
2212
		__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
2213 2214

	if (unlikely(!ghc->memslot))
2215
		return kvm_write_guest(kvm, gpa, data, len);
2216 2217 2218 2219

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

2220
	r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
2221 2222
	if (r)
		return -EFAULT;
2223
	mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
2224 2225 2226

	return 0;
}
2227
EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
2228

2229 2230
int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
2231
{
2232
	return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
2233
}
2234
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
2235

2236 2237
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
2238
{
2239
	struct kvm_memslots *slots = kvm_memslots(kvm);
2240 2241
	int r;

2242 2243
	BUG_ON(len > ghc->len);

2244
	if (slots->generation != ghc->generation)
2245
		__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
2246 2247

	if (unlikely(!ghc->memslot))
2248
		return kvm_read_guest(kvm, ghc->gpa, data, len);
2249 2250 2251 2252 2253 2254 2255 2256 2257 2258

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

	r = __copy_from_user(data, (void __user *)ghc->hva, len);
	if (r)
		return -EFAULT;

	return 0;
}
2259
EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
2260

2261 2262
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
2263 2264 2265
	const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));

	return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
2266 2267 2268 2269 2270 2271 2272 2273 2274 2275
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

2276
	while ((seg = next_segment(len, offset)) != 0) {
2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

2288
static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
2289
				    gfn_t gfn)
A
Avi Kivity 已提交
2290
{
R
Rusty Russell 已提交
2291 2292
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
2293

2294
		set_bit_le(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
2295 2296 2297
	}
}

2298 2299 2300 2301 2302
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = gfn_to_memslot(kvm, gfn);
2303
	mark_page_dirty_in_slot(memslot, gfn);
2304
}
2305
EXPORT_SYMBOL_GPL(mark_page_dirty);
2306

2307 2308 2309 2310 2311 2312 2313 2314 2315
void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
	mark_page_dirty_in_slot(memslot, gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);

2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
void kvm_sigset_activate(struct kvm_vcpu *vcpu)
{
	if (!vcpu->sigset_active)
		return;

	/*
	 * This does a lockless modification of ->real_blocked, which is fine
	 * because, only current can change ->real_blocked and all readers of
	 * ->real_blocked don't care as long ->real_blocked is always a subset
	 * of ->blocked.
	 */
	sigprocmask(SIG_SETMASK, &vcpu->sigset, &current->real_blocked);
}

void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
{
	if (!vcpu->sigset_active)
		return;

	sigprocmask(SIG_SETMASK, &current->real_blocked, NULL);
	sigemptyset(&current->real_blocked);
}

W
Wanpeng Li 已提交
2339 2340
static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
{
2341
	unsigned int old, val, grow, grow_start;
W
Wanpeng Li 已提交
2342

2343
	old = val = vcpu->halt_poll_ns;
2344
	grow_start = READ_ONCE(halt_poll_ns_grow_start);
2345
	grow = READ_ONCE(halt_poll_ns_grow);
2346 2347 2348
	if (!grow)
		goto out;

2349 2350 2351
	val *= grow;
	if (val < grow_start)
		val = grow_start;
W
Wanpeng Li 已提交
2352

2353 2354 2355
	if (val > halt_poll_ns)
		val = halt_poll_ns;

W
Wanpeng Li 已提交
2356
	vcpu->halt_poll_ns = val;
2357
out:
2358
	trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
W
Wanpeng Li 已提交
2359 2360 2361 2362
}

static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
{
2363
	unsigned int old, val, shrink;
W
Wanpeng Li 已提交
2364

2365
	old = val = vcpu->halt_poll_ns;
2366 2367
	shrink = READ_ONCE(halt_poll_ns_shrink);
	if (shrink == 0)
W
Wanpeng Li 已提交
2368 2369
		val = 0;
	else
2370
		val /= shrink;
W
Wanpeng Li 已提交
2371 2372

	vcpu->halt_poll_ns = val;
2373
	trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
W
Wanpeng Li 已提交
2374 2375
}

2376 2377
static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
{
2378 2379 2380
	int ret = -EINTR;
	int idx = srcu_read_lock(&vcpu->kvm->srcu);

2381 2382
	if (kvm_arch_vcpu_runnable(vcpu)) {
		kvm_make_request(KVM_REQ_UNHALT, vcpu);
2383
		goto out;
2384 2385
	}
	if (kvm_cpu_has_pending_timer(vcpu))
2386
		goto out;
2387
	if (signal_pending(current))
2388
		goto out;
2389

2390 2391 2392 2393
	ret = 0;
out:
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	return ret;
2394 2395
}

E
Eddie Dong 已提交
2396 2397 2398
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
2399
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2400
{
2401
	ktime_t start, cur;
2402
	DECLARE_SWAITQUEUE(wait);
2403
	bool waited = false;
W
Wanpeng Li 已提交
2404
	u64 block_ns;
2405

2406 2407
	kvm_arch_vcpu_blocking(vcpu);

2408
	start = cur = ktime_get();
2409
	if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) {
W
Wanpeng Li 已提交
2410
		ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2411

2412
		++vcpu->stat.halt_attempted_poll;
2413 2414 2415 2416 2417 2418 2419
		do {
			/*
			 * This sets KVM_REQ_UNHALT if an interrupt
			 * arrives.
			 */
			if (kvm_vcpu_check_block(vcpu) < 0) {
				++vcpu->stat.halt_successful_poll;
2420 2421
				if (!vcpu_valid_wakeup(vcpu))
					++vcpu->stat.halt_poll_invalid;
2422 2423 2424 2425 2426
				goto out;
			}
			cur = ktime_get();
		} while (single_task_running() && ktime_before(cur, stop));
	}
2427 2428

	for (;;) {
2429
		prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2430

2431
		if (kvm_vcpu_check_block(vcpu) < 0)
2432 2433
			break;

2434
		waited = true;
E
Eddie Dong 已提交
2435 2436
		schedule();
	}
2437

2438
	finish_swait(&vcpu->wq, &wait);
2439 2440
	cur = ktime_get();
out:
2441
	kvm_arch_vcpu_unblocking(vcpu);
W
Wanpeng Li 已提交
2442 2443
	block_ns = ktime_to_ns(cur) - ktime_to_ns(start);

2444 2445
	if (!kvm_arch_no_poll(vcpu)) {
		if (!vcpu_valid_wakeup(vcpu)) {
W
Wanpeng Li 已提交
2446
			shrink_halt_poll_ns(vcpu);
2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460
		} else if (halt_poll_ns) {
			if (block_ns <= vcpu->halt_poll_ns)
				;
			/* we had a long block, shrink polling */
			else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
				shrink_halt_poll_ns(vcpu);
			/* we had a short halt and our poll time is too small */
			else if (vcpu->halt_poll_ns < halt_poll_ns &&
				block_ns < halt_poll_ns)
				grow_halt_poll_ns(vcpu);
		} else {
			vcpu->halt_poll_ns = 0;
		}
	}
W
Wanpeng Li 已提交
2461

2462 2463
	trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
	kvm_arch_vcpu_block_finish(vcpu);
E
Eddie Dong 已提交
2464
}
2465
EXPORT_SYMBOL_GPL(kvm_vcpu_block);
E
Eddie Dong 已提交
2466

2467
bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2468
{
2469
	struct swait_queue_head *wqp;
2470 2471

	wqp = kvm_arch_vcpu_wq(vcpu);
2472
	if (swq_has_sleeper(wqp)) {
2473
		swake_up_one(wqp);
2474
		WRITE_ONCE(vcpu->ready, true);
2475
		++vcpu->stat.halt_wakeup;
2476
		return true;
2477 2478
	}

2479
	return false;
2480 2481 2482
}
EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);

2483
#ifndef CONFIG_S390
2484 2485 2486 2487 2488 2489 2490 2491
/*
 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
 */
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
	int me;
	int cpu = vcpu->cpu;

2492 2493 2494
	if (kvm_vcpu_wake_up(vcpu))
		return;

2495 2496 2497 2498 2499 2500
	me = get_cpu();
	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
		if (kvm_arch_vcpu_should_kick(vcpu))
			smp_send_reschedule(cpu);
	put_cpu();
}
2501
EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2502
#endif /* !CONFIG_S390 */
2503

2504
int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2505 2506 2507
{
	struct pid *pid;
	struct task_struct *task = NULL;
2508
	int ret = 0;
2509 2510 2511 2512

	rcu_read_lock();
	pid = rcu_dereference(target->pid);
	if (pid)
2513
		task = get_pid_task(pid, PIDTYPE_PID);
2514 2515
	rcu_read_unlock();
	if (!task)
2516 2517
		return ret;
	ret = yield_to(task, 1);
2518
	put_task_struct(task);
2519 2520

	return ret;
2521 2522 2523
}
EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);

2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545
/*
 * Helper that checks whether a VCPU is eligible for directed yield.
 * Most eligible candidate to yield is decided by following heuristics:
 *
 *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
 *  (preempted lock holder), indicated by @in_spin_loop.
 *  Set at the beiginning and cleared at the end of interception/PLE handler.
 *
 *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
 *  chance last time (mostly it has become eligible now since we have probably
 *  yielded to lockholder in last iteration. This is done by toggling
 *  @dy_eligible each time a VCPU checked for eligibility.)
 *
 *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
 *  to preempted lock-holder could result in wrong VCPU selection and CPU
 *  burning. Giving priority for a potential lock-holder increases lock
 *  progress.
 *
 *  Since algorithm is based on heuristics, accessing another VCPU data without
 *  locking does not harm. It may result in trying to yield to  same VCPU, fail
 *  and continue with next VCPU and so on.
 */
2546
static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2547
{
2548
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2549 2550 2551
	bool eligible;

	eligible = !vcpu->spin_loop.in_spin_loop ||
2552
		    vcpu->spin_loop.dy_eligible;
2553 2554 2555 2556 2557

	if (vcpu->spin_loop.in_spin_loop)
		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);

	return eligible;
2558 2559
#else
	return true;
2560
#endif
2561
}
2562

2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585
/*
 * Unlike kvm_arch_vcpu_runnable, this function is called outside
 * a vcpu_load/vcpu_put pair.  However, for most architectures
 * kvm_arch_vcpu_runnable does not require vcpu_load.
 */
bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
{
	return kvm_arch_vcpu_runnable(vcpu);
}

static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu)
{
	if (kvm_arch_dy_runnable(vcpu))
		return true;

#ifdef CONFIG_KVM_ASYNC_PF
	if (!list_empty_careful(&vcpu->async_pf.done))
		return true;
#endif

	return false;
}

2586
void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
Z
Zhai, Edwin 已提交
2587
{
2588 2589 2590 2591
	struct kvm *kvm = me->kvm;
	struct kvm_vcpu *vcpu;
	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
	int yielded = 0;
2592
	int try = 3;
2593 2594
	int pass;
	int i;
Z
Zhai, Edwin 已提交
2595

2596
	kvm_vcpu_set_in_spin_loop(me, true);
2597 2598 2599 2600 2601 2602 2603
	/*
	 * We boost the priority of a VCPU that is runnable but not
	 * currently running, because it got preempted by something
	 * else and called schedule in __vcpu_run.  Hopefully that
	 * VCPU is holding the lock that we need and will release it.
	 * We approximate round-robin by starting at the last boosted VCPU.
	 */
2604
	for (pass = 0; pass < 2 && !yielded && try; pass++) {
2605
		kvm_for_each_vcpu(i, vcpu, kvm) {
2606
			if (!pass && i <= last_boosted_vcpu) {
2607 2608 2609 2610
				i = last_boosted_vcpu;
				continue;
			} else if (pass && i > last_boosted_vcpu)
				break;
2611
			if (!READ_ONCE(vcpu->ready))
2612
				continue;
2613 2614
			if (vcpu == me)
				continue;
2615
			if (swait_active(&vcpu->wq) && !vcpu_dy_runnable(vcpu))
2616
				continue;
2617 2618
			if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
				!kvm_arch_vcpu_in_kernel(vcpu))
2619
				continue;
2620 2621
			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
				continue;
2622 2623 2624

			yielded = kvm_vcpu_yield_to(vcpu);
			if (yielded > 0) {
2625 2626
				kvm->last_boosted_vcpu = i;
				break;
2627 2628 2629 2630
			} else if (yielded < 0) {
				try--;
				if (!try)
					break;
2631 2632 2633
			}
		}
	}
2634
	kvm_vcpu_set_in_spin_loop(me, false);
2635 2636 2637

	/* Ensure vcpu is not eligible during next spinloop */
	kvm_vcpu_set_dy_eligible(me, false);
Z
Zhai, Edwin 已提交
2638 2639 2640
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

2641
static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
2642
{
2643
	struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
2644 2645
	struct page *page;

2646
	if (vmf->pgoff == 0)
2647
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
2648
#ifdef CONFIG_X86
2649
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2650
		page = virt_to_page(vcpu->arch.pio_data);
2651
#endif
2652
#ifdef CONFIG_KVM_MMIO
2653 2654
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
2655
#endif
2656
	else
2657
		return kvm_arch_vcpu_fault(vcpu, vmf);
2658
	get_page(page);
2659 2660
	vmf->page = page;
	return 0;
2661 2662
}

2663
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2664
	.fault = kvm_vcpu_fault,
2665 2666 2667 2668 2669 2670 2671 2672
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
2673 2674 2675 2676
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

2677
	debugfs_remove_recursive(vcpu->debugfs_dentry);
A
Al Viro 已提交
2678
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
2679 2680 2681
	return 0;
}

2682
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
2683 2684
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
2685
	.mmap           = kvm_vcpu_mmap,
2686
	.llseek		= noop_llseek,
2687
	KVM_COMPAT(kvm_vcpu_compat_ioctl),
A
Avi Kivity 已提交
2688 2689 2690 2691 2692 2693 2694
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
2695 2696 2697 2698
	char name[8 + 1 + ITOA_MAX_LEN + 1];

	snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
	return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
A
Avi Kivity 已提交
2699 2700
}

2701
static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2702
{
2703
#ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS
2704 2705 2706
	char dir_name[ITOA_MAX_LEN * 2];

	if (!debugfs_initialized())
2707
		return;
2708 2709 2710

	snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
	vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
2711
						  vcpu->kvm->debugfs_dentry);
2712

2713
	kvm_arch_create_vcpu_debugfs(vcpu);
2714
#endif
2715 2716
}

2717 2718 2719
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
2720
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2721 2722
{
	int r;
2723
	struct kvm_vcpu *vcpu;
2724

G
Greg Kurz 已提交
2725
	if (id >= KVM_MAX_VCPU_ID)
2726 2727
		return -EINVAL;

2728 2729 2730 2731 2732 2733 2734 2735 2736
	mutex_lock(&kvm->lock);
	if (kvm->created_vcpus == KVM_MAX_VCPUS) {
		mutex_unlock(&kvm->lock);
		return -EINVAL;
	}

	kvm->created_vcpus++;
	mutex_unlock(&kvm->lock);

2737 2738 2739 2740
	r = kvm_arch_vcpu_precreate(kvm, id);
	if (r)
		goto vcpu_decrement;

2741 2742 2743
	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
	if (!vcpu) {
		r = -ENOMEM;
2744 2745
		goto vcpu_decrement;
	}
2746

2747 2748 2749 2750 2751 2752 2753 2754
	r = kvm_vcpu_init(vcpu, kvm, id);
	if (r)
		goto vcpu_free;

	r = kvm_arch_vcpu_create(vcpu);
	if (r)
		goto vcpu_uninit;

2755 2756
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

2757 2758
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
2759
		goto vcpu_destroy;
2760

2761
	kvm_create_vcpu_debugfs(vcpu);
2762

S
Shaohua Li 已提交
2763
	mutex_lock(&kvm->lock);
2764 2765 2766 2767
	if (kvm_get_vcpu_by_id(kvm, id)) {
		r = -EEXIST;
		goto unlock_vcpu_destroy;
	}
2768

2769 2770
	vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
	BUG_ON(kvm->vcpus[vcpu->vcpu_idx]);
2771

R
Rusty Russell 已提交
2772
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
2773
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
2774
	r = create_vcpu_fd(vcpu);
2775
	if (r < 0) {
2776
		kvm_put_kvm_no_destroy(kvm);
2777
		goto unlock_vcpu_destroy;
2778 2779
	}

2780
	kvm->vcpus[vcpu->vcpu_idx] = vcpu;
2781 2782 2783 2784 2785

	/*
	 * Pairs with smp_rmb() in kvm_get_vcpu.  Write kvm->vcpus
	 * before kvm->online_vcpu's incremented value.
	 */
2786 2787 2788 2789
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

	mutex_unlock(&kvm->lock);
2790
	kvm_arch_vcpu_postcreate(vcpu);
R
Rusty Russell 已提交
2791
	return r;
2792

2793
unlock_vcpu_destroy:
2794
	mutex_unlock(&kvm->lock);
2795
	debugfs_remove_recursive(vcpu->debugfs_dentry);
2796
vcpu_destroy:
2797
	kvm_arch_vcpu_destroy(vcpu);
2798 2799 2800 2801
vcpu_uninit:
	kvm_vcpu_uninit(vcpu);
vcpu_free:
	kmem_cache_free(kvm_vcpu_cache, vcpu);
2802 2803 2804 2805
vcpu_decrement:
	mutex_lock(&kvm->lock);
	kvm->created_vcpus--;
	mutex_unlock(&kvm->lock);
2806 2807 2808
	return r;
}

A
Avi Kivity 已提交
2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
2820 2821
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
2822
{
A
Avi Kivity 已提交
2823
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
2824
	void __user *argp = (void __user *)arg;
2825
	int r;
2826 2827
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
2828

2829 2830
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
2831

2832 2833 2834
	if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
		return -EINVAL;

2835
	/*
2836 2837
	 * Some architectures have vcpu ioctls that are asynchronous to vcpu
	 * execution; mutex_lock() would break them.
2838
	 */
2839 2840
	r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
	if (r != -ENOIOCTLCMD)
2841
		return r;
2842

2843 2844
	if (mutex_lock_killable(&vcpu->mutex))
		return -EINTR;
A
Avi Kivity 已提交
2845
	switch (ioctl) {
2846 2847
	case KVM_RUN: {
		struct pid *oldpid;
2848 2849 2850
		r = -EINVAL;
		if (arg)
			goto out;
2851
		oldpid = rcu_access_pointer(vcpu->pid);
2852
		if (unlikely(oldpid != task_pid(current))) {
2853
			/* The thread running this VCPU changed. */
2854
			struct pid *newpid;
2855

2856 2857 2858 2859 2860
			r = kvm_arch_vcpu_run_pid_change(vcpu);
			if (r)
				break;

			newpid = get_task_pid(current, PIDTYPE_PID);
2861 2862 2863 2864 2865
			rcu_assign_pointer(vcpu->pid, newpid);
			if (oldpid)
				synchronize_rcu();
			put_pid(oldpid);
		}
2866
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2867
		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
A
Avi Kivity 已提交
2868
		break;
2869
	}
A
Avi Kivity 已提交
2870
	case KVM_GET_REGS: {
2871
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2872

2873
		r = -ENOMEM;
2874
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT);
2875
		if (!kvm_regs)
A
Avi Kivity 已提交
2876
			goto out;
2877 2878 2879
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
2880
		r = -EFAULT;
2881 2882
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
2883
		r = 0;
2884 2885
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
2886 2887 2888
		break;
	}
	case KVM_SET_REGS: {
2889
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2890

2891
		r = -ENOMEM;
2892 2893 2894
		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
		if (IS_ERR(kvm_regs)) {
			r = PTR_ERR(kvm_regs);
A
Avi Kivity 已提交
2895
			goto out;
2896
		}
2897 2898
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
		kfree(kvm_regs);
A
Avi Kivity 已提交
2899 2900 2901
		break;
	}
	case KVM_GET_SREGS: {
2902 2903
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs),
				    GFP_KERNEL_ACCOUNT);
2904 2905 2906 2907
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2908 2909 2910
		if (r)
			goto out;
		r = -EFAULT;
2911
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
2912 2913 2914 2915 2916
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
2917 2918 2919
		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
		if (IS_ERR(kvm_sregs)) {
			r = PTR_ERR(kvm_sregs);
G
Guo Chao 已提交
2920
			kvm_sregs = NULL;
A
Avi Kivity 已提交
2921
			goto out;
2922
		}
2923
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2924 2925
		break;
	}
2926 2927 2928 2929 2930 2931 2932
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
2933
		if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2934 2935 2936 2937 2938 2939 2940 2941
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
2942
		if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2943 2944 2945 2946
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		break;
	}
A
Avi Kivity 已提交
2947 2948 2949 2950
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
2951
		if (copy_from_user(&tr, argp, sizeof(tr)))
A
Avi Kivity 已提交
2952
			goto out;
2953
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2954 2955 2956
		if (r)
			goto out;
		r = -EFAULT;
2957
		if (copy_to_user(argp, &tr, sizeof(tr)))
A
Avi Kivity 已提交
2958 2959 2960 2961
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
2962 2963
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
2964 2965

		r = -EFAULT;
2966
		if (copy_from_user(&dbg, argp, sizeof(dbg)))
A
Avi Kivity 已提交
2967
			goto out;
J
Jan Kiszka 已提交
2968
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
2969 2970
		break;
	}
A
Avi Kivity 已提交
2971 2972 2973 2974 2975 2976 2977 2978 2979
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
2980
					   sizeof(kvm_sigmask)))
A
Avi Kivity 已提交
2981 2982
				goto out;
			r = -EINVAL;
2983
			if (kvm_sigmask.len != sizeof(sigset))
A
Avi Kivity 已提交
2984 2985 2986
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
2987
					   sizeof(sigset)))
A
Avi Kivity 已提交
2988 2989 2990
				goto out;
			p = &sigset;
		}
2991
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
A
Avi Kivity 已提交
2992 2993
		break;
	}
A
Avi Kivity 已提交
2994
	case KVM_GET_FPU: {
2995
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT);
2996 2997 2998 2999
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
3000 3001 3002
		if (r)
			goto out;
		r = -EFAULT;
3003
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
3004 3005 3006 3007 3008
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
3009 3010 3011
		fpu = memdup_user(argp, sizeof(*fpu));
		if (IS_ERR(fpu)) {
			r = PTR_ERR(fpu);
G
Guo Chao 已提交
3012
			fpu = NULL;
A
Avi Kivity 已提交
3013
			goto out;
3014
		}
3015
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
3016 3017
		break;
	}
A
Avi Kivity 已提交
3018
	default:
3019
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
3020 3021
	}
out:
3022
	mutex_unlock(&vcpu->mutex);
3023 3024
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
3025 3026 3027
	return r;
}

3028
#ifdef CONFIG_KVM_COMPAT
3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047
static long kvm_vcpu_compat_ioctl(struct file *filp,
				  unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = compat_ptr(arg);
	int r;

	if (vcpu->kvm->mm != current->mm)
		return -EIO;

	switch (ioctl) {
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset;

		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
3048
					   sizeof(kvm_sigmask)))
3049 3050
				goto out;
			r = -EINVAL;
A
Al Viro 已提交
3051
			if (kvm_sigmask.len != sizeof(compat_sigset_t))
3052 3053
				goto out;
			r = -EFAULT;
A
Al Viro 已提交
3054
			if (get_compat_sigset(&sigset, (void *)sigmask_arg->sigset))
3055
				goto out;
3056 3057 3058
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		} else
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069
		break;
	}
	default:
		r = kvm_vcpu_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

3070 3071 3072 3073 3074 3075 3076 3077 3078 3079
static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct kvm_device *dev = filp->private_data;

	if (dev->ops->mmap)
		return dev->ops->mmap(dev, vma);

	return -ENODEV;
}

S
Scott Wood 已提交
3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100
static int kvm_device_ioctl_attr(struct kvm_device *dev,
				 int (*accessor)(struct kvm_device *dev,
						 struct kvm_device_attr *attr),
				 unsigned long arg)
{
	struct kvm_device_attr attr;

	if (!accessor)
		return -EPERM;

	if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
		return -EFAULT;

	return accessor(dev, &attr);
}

static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
			     unsigned long arg)
{
	struct kvm_device *dev = filp->private_data;

3101 3102 3103
	if (dev->kvm->mm != current->mm)
		return -EIO;

S
Scott Wood 已提交
3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
	switch (ioctl) {
	case KVM_SET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
	case KVM_GET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
	case KVM_HAS_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
	default:
		if (dev->ops->ioctl)
			return dev->ops->ioctl(dev, ioctl, arg);

		return -ENOTTY;
	}
}

static int kvm_device_release(struct inode *inode, struct file *filp)
{
	struct kvm_device *dev = filp->private_data;
	struct kvm *kvm = dev->kvm;

3124 3125 3126 3127 3128 3129 3130
	if (dev->ops->release) {
		mutex_lock(&kvm->lock);
		list_del(&dev->vm_node);
		dev->ops->release(dev);
		mutex_unlock(&kvm->lock);
	}

S
Scott Wood 已提交
3131 3132 3133 3134 3135 3136 3137
	kvm_put_kvm(kvm);
	return 0;
}

static const struct file_operations kvm_device_fops = {
	.unlocked_ioctl = kvm_device_ioctl,
	.release = kvm_device_release,
3138
	KVM_COMPAT(kvm_device_ioctl),
3139
	.mmap = kvm_device_mmap,
S
Scott Wood 已提交
3140 3141 3142 3143 3144 3145 3146 3147 3148 3149
};

struct kvm_device *kvm_device_from_filp(struct file *filp)
{
	if (filp->f_op != &kvm_device_fops)
		return NULL;

	return filp->private_data;
}

3150
static const struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
3151
#ifdef CONFIG_KVM_MPIC
3152 3153
	[KVM_DEV_TYPE_FSL_MPIC_20]	= &kvm_mpic_ops,
	[KVM_DEV_TYPE_FSL_MPIC_42]	= &kvm_mpic_ops,
3154
#endif
3155 3156
};

3157
int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type)
3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168
{
	if (type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENOSPC;

	if (kvm_device_ops_table[type] != NULL)
		return -EEXIST;

	kvm_device_ops_table[type] = ops;
	return 0;
}

3169 3170 3171 3172 3173 3174
void kvm_unregister_device_ops(u32 type)
{
	if (kvm_device_ops_table[type] != NULL)
		kvm_device_ops_table[type] = NULL;
}

S
Scott Wood 已提交
3175 3176 3177
static int kvm_ioctl_create_device(struct kvm *kvm,
				   struct kvm_create_device *cd)
{
3178
	const struct kvm_device_ops *ops = NULL;
S
Scott Wood 已提交
3179 3180
	struct kvm_device *dev;
	bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
P
Paolo Bonzini 已提交
3181
	int type;
S
Scott Wood 已提交
3182 3183
	int ret;

3184 3185 3186
	if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENODEV;

P
Paolo Bonzini 已提交
3187 3188
	type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table));
	ops = kvm_device_ops_table[type];
3189
	if (ops == NULL)
S
Scott Wood 已提交
3190 3191 3192 3193 3194
		return -ENODEV;

	if (test)
		return 0;

3195
	dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
S
Scott Wood 已提交
3196 3197 3198 3199 3200 3201
	if (!dev)
		return -ENOMEM;

	dev->ops = ops;
	dev->kvm = kvm;

3202
	mutex_lock(&kvm->lock);
P
Paolo Bonzini 已提交
3203
	ret = ops->create(dev, type);
S
Scott Wood 已提交
3204
	if (ret < 0) {
3205
		mutex_unlock(&kvm->lock);
S
Scott Wood 已提交
3206 3207 3208
		kfree(dev);
		return ret;
	}
3209 3210
	list_add(&dev->vm_node, &kvm->devices);
	mutex_unlock(&kvm->lock);
S
Scott Wood 已提交
3211

3212 3213 3214
	if (ops->init)
		ops->init(dev);

3215
	kvm_get_kvm(kvm);
3216
	ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
S
Scott Wood 已提交
3217
	if (ret < 0) {
3218
		kvm_put_kvm_no_destroy(kvm);
3219 3220 3221
		mutex_lock(&kvm->lock);
		list_del(&dev->vm_node);
		mutex_unlock(&kvm->lock);
3222
		ops->destroy(dev);
S
Scott Wood 已提交
3223 3224 3225 3226 3227 3228 3229
		return ret;
	}

	cd->fd = ret;
	return 0;
}

3230 3231 3232 3233 3234 3235 3236 3237 3238 3239
static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
{
	switch (arg) {
	case KVM_CAP_USER_MEMORY:
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
	case KVM_CAP_INTERNAL_ERROR_DATA:
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_CAP_SIGNAL_MSI:
#endif
3240
#ifdef CONFIG_HAVE_KVM_IRQFD
3241
	case KVM_CAP_IRQFD:
3242 3243
	case KVM_CAP_IRQFD_RESAMPLE:
#endif
3244
	case KVM_CAP_IOEVENTFD_ANY_LENGTH:
3245
	case KVM_CAP_CHECK_EXTENSION_VM:
3246
	case KVM_CAP_ENABLE_CAP_VM:
3247
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3248
	case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
3249
#endif
3250
		return 1;
3251
#ifdef CONFIG_KVM_MMIO
3252 3253
	case KVM_CAP_COALESCED_MMIO:
		return KVM_COALESCED_MMIO_PAGE_OFFSET;
P
Peng Hao 已提交
3254 3255
	case KVM_CAP_COALESCED_PIO:
		return 1;
3256
#endif
3257 3258 3259
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_CAP_IRQ_ROUTING:
		return KVM_MAX_IRQ_ROUTES;
3260 3261 3262 3263
#endif
#if KVM_ADDRESS_SPACE_NUM > 1
	case KVM_CAP_MULTI_ADDRESS_SPACE:
		return KVM_ADDRESS_SPACE_NUM;
3264
#endif
3265 3266
	case KVM_CAP_NR_MEMSLOTS:
		return KVM_USER_MEM_SLOTS;
3267 3268 3269 3270 3271 3272
	default:
		break;
	}
	return kvm_vm_ioctl_check_extension(kvm, arg);
}

3273 3274 3275 3276 3277 3278 3279 3280 3281 3282
int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
						  struct kvm_enable_cap *cap)
{
	return -EINVAL;
}

static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
					   struct kvm_enable_cap *cap)
{
	switch (cap->cap) {
3283
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3284
	case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
3285 3286 3287 3288 3289
		if (cap->flags || (cap->args[0] & ~1))
			return -EINVAL;
		kvm->manual_dirty_log_protect = cap->args[0];
		return 0;
#endif
3290 3291 3292 3293 3294
	default:
		return kvm_vm_ioctl_enable_cap(kvm, cap);
	}
}

A
Avi Kivity 已提交
3295 3296 3297 3298 3299
static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
3300
	int r;
A
Avi Kivity 已提交
3301

3302 3303
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
3304 3305 3306 3307
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		break;
3308 3309 3310 3311 3312 3313 3314 3315 3316
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		r = -EFAULT;
		if (copy_from_user(&cap, argp, sizeof(cap)))
			goto out;
		r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
		break;
	}
3317 3318 3319 3320 3321
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
3322
						sizeof(kvm_userspace_mem)))
3323 3324
			goto out;

3325
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
A
Avi Kivity 已提交
3326 3327 3328 3329 3330 3331
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
3332
		if (copy_from_user(&log, argp, sizeof(log)))
A
Avi Kivity 已提交
3333
			goto out;
3334
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
3335 3336
		break;
	}
3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
	case KVM_CLEAR_DIRTY_LOG: {
		struct kvm_clear_dirty_log log;

		r = -EFAULT;
		if (copy_from_user(&log, argp, sizeof(log)))
			goto out;
		r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
		break;
	}
#endif
3348
#ifdef CONFIG_KVM_MMIO
3349 3350
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
3351

3352
		r = -EFAULT;
3353
		if (copy_from_user(&zone, argp, sizeof(zone)))
3354 3355 3356 3357 3358 3359
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
3360

3361
		r = -EFAULT;
3362
		if (copy_from_user(&zone, argp, sizeof(zone)))
3363 3364 3365 3366 3367
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		break;
	}
#endif
G
Gregory Haskins 已提交
3368 3369 3370 3371
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
3372
		if (copy_from_user(&data, argp, sizeof(data)))
G
Gregory Haskins 已提交
3373
			goto out;
3374
		r = kvm_irqfd(kvm, &data);
G
Gregory Haskins 已提交
3375 3376
		break;
	}
G
Gregory Haskins 已提交
3377 3378 3379 3380
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
3381
		if (copy_from_user(&data, argp, sizeof(data)))
G
Gregory Haskins 已提交
3382 3383 3384 3385
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
3386 3387 3388 3389 3390
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_SIGNAL_MSI: {
		struct kvm_msi msi;

		r = -EFAULT;
3391
		if (copy_from_user(&msi, argp, sizeof(msi)))
3392 3393 3394 3395
			goto out;
		r = kvm_send_userspace_msi(kvm, &msi);
		break;
	}
3396 3397 3398 3399 3400 3401 3402
#endif
#ifdef __KVM_HAVE_IRQ_LINE
	case KVM_IRQ_LINE_STATUS:
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
3403
		if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
3404 3405
			goto out;

3406 3407
		r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
					ioctl == KVM_IRQ_LINE_STATUS);
3408 3409 3410 3411 3412
		if (r)
			goto out;

		r = -EFAULT;
		if (ioctl == KVM_IRQ_LINE_STATUS) {
3413
			if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
3414 3415 3416 3417 3418 3419
				goto out;
		}

		r = 0;
		break;
	}
3420
#endif
3421 3422 3423 3424
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_SET_GSI_ROUTING: {
		struct kvm_irq_routing routing;
		struct kvm_irq_routing __user *urouting;
3425
		struct kvm_irq_routing_entry *entries = NULL;
3426 3427 3428 3429 3430

		r = -EFAULT;
		if (copy_from_user(&routing, argp, sizeof(routing)))
			goto out;
		r = -EINVAL;
3431 3432
		if (!kvm_arch_can_set_irq_routing(kvm))
			goto out;
3433
		if (routing.nr > KVM_MAX_IRQ_ROUTES)
3434 3435 3436
			goto out;
		if (routing.flags)
			goto out;
3437 3438
		if (routing.nr) {
			r = -ENOMEM;
3439 3440
			entries = vmalloc(array_size(sizeof(*entries),
						     routing.nr));
3441 3442 3443 3444 3445 3446 3447 3448
			if (!entries)
				goto out;
			r = -EFAULT;
			urouting = argp;
			if (copy_from_user(entries, urouting->entries,
					   routing.nr * sizeof(*entries)))
				goto out_free_irq_routing;
		}
3449 3450
		r = kvm_set_irq_routing(kvm, entries, routing.nr,
					routing.flags);
3451
out_free_irq_routing:
3452 3453 3454 3455
		vfree(entries);
		break;
	}
#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
S
Scott Wood 已提交
3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473
	case KVM_CREATE_DEVICE: {
		struct kvm_create_device cd;

		r = -EFAULT;
		if (copy_from_user(&cd, argp, sizeof(cd)))
			goto out;

		r = kvm_ioctl_create_device(kvm, &cd);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(argp, &cd, sizeof(cd)))
			goto out;

		r = 0;
		break;
	}
3474 3475 3476
	case KVM_CHECK_EXTENSION:
		r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
		break;
3477
	default:
3478
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
3479 3480 3481 3482 3483
	}
out:
	return r;
}

3484
#ifdef CONFIG_KVM_COMPAT
3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
3509
			return -EFAULT;
3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}
	return r;
}
#endif

3525
static struct file_operations kvm_vm_fops = {
3526 3527
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
3528
	.llseek		= noop_llseek,
3529
	KVM_COMPAT(kvm_vm_compat_ioctl),
3530 3531
};

3532
static int kvm_dev_ioctl_create_vm(unsigned long type)
3533
{
3534
	int r;
3535
	struct kvm *kvm;
3536
	struct file *file;
3537

3538
	kvm = kvm_create_vm(type);
3539 3540
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
3541
#ifdef CONFIG_KVM_MMIO
3542
	r = kvm_coalesced_mmio_init(kvm);
3543 3544
	if (r < 0)
		goto put_kvm;
3545
#endif
3546
	r = get_unused_fd_flags(O_CLOEXEC);
3547 3548 3549
	if (r < 0)
		goto put_kvm;

3550 3551 3552
	file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
	if (IS_ERR(file)) {
		put_unused_fd(r);
3553 3554
		r = PTR_ERR(file);
		goto put_kvm;
3555
	}
3556

3557 3558 3559 3560 3561 3562
	/*
	 * Don't call kvm_put_kvm anymore at this point; file->f_op is
	 * already set, with ->release() being kvm_vm_release().  In error
	 * cases it will be called by the final fput(file) and will take
	 * care of doing kvm_put_kvm(kvm).
	 */
3563
	if (kvm_create_vm_debugfs(kvm, r) < 0) {
3564 3565
		put_unused_fd(r);
		fput(file);
3566 3567
		return -ENOMEM;
	}
3568
	kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
3569

3570
	fd_install(r, file);
3571
	return r;
3572 3573 3574 3575

put_kvm:
	kvm_put_kvm(kvm);
	return r;
3576 3577 3578 3579 3580
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
3581
	long r = -EINVAL;
3582 3583 3584

	switch (ioctl) {
	case KVM_GET_API_VERSION:
3585 3586
		if (arg)
			goto out;
3587 3588 3589
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
3590
		r = kvm_dev_ioctl_create_vm(arg);
3591
		break;
3592
	case KVM_CHECK_EXTENSION:
3593
		r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3594
		break;
3595 3596 3597
	case KVM_GET_VCPU_MMAP_SIZE:
		if (arg)
			goto out;
3598 3599 3600
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
3601
#endif
3602
#ifdef CONFIG_KVM_MMIO
3603
		r += PAGE_SIZE;    /* coalesced mmio ring page */
3604
#endif
3605
		break;
3606 3607 3608
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
3609
		r = -EOPNOTSUPP;
3610
		break;
A
Avi Kivity 已提交
3611
	default:
3612
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
3613 3614 3615 3616 3617 3618 3619
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
3620
	.llseek		= noop_llseek,
3621
	KVM_COMPAT(kvm_dev_ioctl),
A
Avi Kivity 已提交
3622 3623 3624
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
3625
	KVM_MINOR,
A
Avi Kivity 已提交
3626 3627 3628 3629
	"kvm",
	&kvm_chardev_ops,
};

3630
static void hardware_enable_nolock(void *junk)
3631 3632
{
	int cpu = raw_smp_processor_id();
3633
	int r;
3634

3635
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3636
		return;
3637

3638
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
3639

3640
	r = kvm_arch_hardware_enable();
3641 3642 3643 3644

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
X
Xiubo Li 已提交
3645
		pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3646
	}
3647 3648
}

3649
static int kvm_starting_cpu(unsigned int cpu)
3650
{
3651
	raw_spin_lock(&kvm_count_lock);
3652 3653
	if (kvm_usage_count)
		hardware_enable_nolock(NULL);
3654
	raw_spin_unlock(&kvm_count_lock);
3655
	return 0;
3656 3657 3658
}

static void hardware_disable_nolock(void *junk)
3659 3660 3661
{
	int cpu = raw_smp_processor_id();

3662
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3663
		return;
3664
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3665
	kvm_arch_hardware_disable();
3666 3667
}

3668
static int kvm_dying_cpu(unsigned int cpu)
3669
{
3670
	raw_spin_lock(&kvm_count_lock);
3671 3672
	if (kvm_usage_count)
		hardware_disable_nolock(NULL);
3673
	raw_spin_unlock(&kvm_count_lock);
3674
	return 0;
3675 3676
}

3677 3678 3679 3680 3681 3682
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
3683
		on_each_cpu(hardware_disable_nolock, NULL, 1);
3684 3685 3686 3687
}

static void hardware_disable_all(void)
{
3688
	raw_spin_lock(&kvm_count_lock);
3689
	hardware_disable_all_nolock();
3690
	raw_spin_unlock(&kvm_count_lock);
3691 3692 3693 3694 3695 3696
}

static int hardware_enable_all(void)
{
	int r = 0;

3697
	raw_spin_lock(&kvm_count_lock);
3698 3699 3700 3701

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
3702
		on_each_cpu(hardware_enable_nolock, NULL, 1);
3703 3704 3705 3706 3707 3708 3709

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

3710
	raw_spin_unlock(&kvm_count_lock);
3711 3712 3713 3714

	return r;
}

3715
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
3716
		      void *v)
3717
{
3718 3719 3720 3721 3722 3723
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
X
Xiubo Li 已提交
3724
	pr_info("kvm: exiting hardware virtualization\n");
3725
	kvm_rebooting = true;
3726
	on_each_cpu(hardware_disable_nolock, NULL, 1);
3727 3728 3729 3730 3731 3732 3733 3734
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
3735
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3736 3737 3738 3739
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
3740
		struct kvm_io_device *pos = bus->range[i].dev;
3741 3742 3743

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
3744
	kfree(bus);
3745 3746
}

3747
static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
X
Xiubo Li 已提交
3748
				 const struct kvm_io_range *r2)
3749
{
J
Jason Wang 已提交
3750 3751 3752 3753
	gpa_t addr1 = r1->addr;
	gpa_t addr2 = r2->addr;

	if (addr1 < addr2)
3754
		return -1;
J
Jason Wang 已提交
3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766

	/* If r2->len == 0, match the exact address.  If r2->len != 0,
	 * accept any overlapping write.  Any order is acceptable for
	 * overlapping ranges, because kvm_io_bus_get_first_dev ensures
	 * we process all of them.
	 */
	if (r2->len) {
		addr1 += r1->len;
		addr2 += r2->len;
	}

	if (addr1 > addr2)
3767
		return 1;
J
Jason Wang 已提交
3768

3769 3770 3771
	return 0;
}

3772 3773
static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
{
3774
	return kvm_io_bus_cmp(p1, p2);
3775 3776
}

G
Geoff Levand 已提交
3777
static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794
			     gpa_t addr, int len)
{
	struct kvm_io_range *range, key;
	int off;

	key = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	range = bsearch(&key, bus->range, bus->dev_count,
			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
	if (range == NULL)
		return -ENOENT;

	off = range - bus->range;

3795
	while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3796 3797 3798 3799 3800
		off--;

	return off;
}

3801
static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
C
Cornelia Huck 已提交
3802 3803 3804 3805 3806 3807 3808 3809 3810
			      struct kvm_io_range *range, const void *val)
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
3811
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3812
		if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
C
Cornelia Huck 已提交
3813 3814 3815 3816 3817 3818 3819 3820
					range->len, val))
			return idx;
		idx++;
	}

	return -EOPNOTSUPP;
}

3821
/* kvm_io_bus_write - called under kvm->slots_lock */
3822
int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3823
		     int len, const void *val)
3824
{
3825
	struct kvm_io_bus *bus;
3826
	struct kvm_io_range range;
C
Cornelia Huck 已提交
3827
	int r;
3828 3829 3830 3831 3832

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
3833

3834
	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3835 3836
	if (!bus)
		return -ENOMEM;
3837
	r = __kvm_io_bus_write(vcpu, bus, &range, val);
C
Cornelia Huck 已提交
3838 3839
	return r < 0 ? r : 0;
}
L
Leo Yan 已提交
3840
EXPORT_SYMBOL_GPL(kvm_io_bus_write);
C
Cornelia Huck 已提交
3841 3842

/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3843 3844
int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
			    gpa_t addr, int len, const void *val, long cookie)
C
Cornelia Huck 已提交
3845 3846 3847 3848 3849 3850 3851 3852 3853
{
	struct kvm_io_bus *bus;
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

3854
	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3855 3856
	if (!bus)
		return -ENOMEM;
C
Cornelia Huck 已提交
3857 3858 3859

	/* First try the device referenced by cookie. */
	if ((cookie >= 0) && (cookie < bus->dev_count) &&
3860
	    (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3861
		if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
C
Cornelia Huck 已提交
3862 3863 3864 3865 3866 3867 3868
					val))
			return cookie;

	/*
	 * cookie contained garbage; fall back to search and return the
	 * correct cookie value.
	 */
3869
	return __kvm_io_bus_write(vcpu, bus, &range, val);
C
Cornelia Huck 已提交
3870 3871
}

3872 3873
static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
			     struct kvm_io_range *range, void *val)
C
Cornelia Huck 已提交
3874 3875 3876 3877
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3878 3879 3880 3881
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
3882
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3883
		if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
C
Cornelia Huck 已提交
3884 3885
				       range->len, val))
			return idx;
3886 3887 3888
		idx++;
	}

3889 3890
	return -EOPNOTSUPP;
}
3891

3892
/* kvm_io_bus_read - called under kvm->slots_lock */
3893
int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
M
Marcelo Tosatti 已提交
3894
		    int len, void *val)
3895
{
3896
	struct kvm_io_bus *bus;
3897
	struct kvm_io_range range;
C
Cornelia Huck 已提交
3898
	int r;
3899 3900 3901 3902 3903

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
M
Marcelo Tosatti 已提交
3904

3905
	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3906 3907
	if (!bus)
		return -ENOMEM;
3908
	r = __kvm_io_bus_read(vcpu, bus, &range, val);
C
Cornelia Huck 已提交
3909 3910
	return r < 0 ? r : 0;
}
3911

3912
/* Caller must hold slots_lock. */
3913 3914
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, struct kvm_io_device *dev)
3915
{
3916
	int i;
M
Marcelo Tosatti 已提交
3917
	struct kvm_io_bus *new_bus, *bus;
3918
	struct kvm_io_range range;
3919

3920
	bus = kvm_get_bus(kvm, bus_idx);
3921 3922 3923
	if (!bus)
		return -ENOMEM;

3924 3925
	/* exclude ioeventfd which is limited by maximum fd */
	if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3926
		return -ENOSPC;
3927

3928
	new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
3929
			  GFP_KERNEL_ACCOUNT);
M
Marcelo Tosatti 已提交
3930 3931
	if (!new_bus)
		return -ENOMEM;
3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
		.dev = dev,
	};

	for (i = 0; i < bus->dev_count; i++)
		if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
			break;

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count++;
	new_bus->range[i] = range;
	memcpy(new_bus->range + i + 1, bus->range + i,
		(bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
3948 3949 3950
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
3951 3952 3953 3954

	return 0;
}

3955
/* Caller must hold slots_lock. */
3956 3957
void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			       struct kvm_io_device *dev)
3958
{
3959
	int i;
M
Marcelo Tosatti 已提交
3960
	struct kvm_io_bus *new_bus, *bus;
3961

3962
	bus = kvm_get_bus(kvm, bus_idx);
3963
	if (!bus)
3964
		return;
3965

3966 3967
	for (i = 0; i < bus->dev_count; i++)
		if (bus->range[i].dev == dev) {
3968 3969
			break;
		}
M
Marcelo Tosatti 已提交
3970

3971 3972
	if (i == bus->dev_count)
		return;
3973

3974
	new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
3975
			  GFP_KERNEL_ACCOUNT);
3976 3977 3978 3979
	if (!new_bus)  {
		pr_err("kvm: failed to shrink bus, removing it completely\n");
		goto broken;
	}
3980 3981 3982 3983 3984

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count--;
	memcpy(new_bus->range + i, bus->range + i + 1,
	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
3985

3986
broken:
M
Marcelo Tosatti 已提交
3987 3988 3989
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
3990
	return;
3991 3992
}

3993 3994 3995 3996 3997 3998 3999 4000 4001 4002
struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
					 gpa_t addr)
{
	struct kvm_io_bus *bus;
	int dev_idx, srcu_idx;
	struct kvm_io_device *iodev = NULL;

	srcu_idx = srcu_read_lock(&kvm->srcu);

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
4003 4004
	if (!bus)
		goto out_unlock;
4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018

	dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
	if (dev_idx < 0)
		goto out_unlock;

	iodev = bus->range[dev_idx].dev;

out_unlock:
	srcu_read_unlock(&kvm->srcu, srcu_idx);

	return iodev;
}
EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);

4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030
static int kvm_debugfs_open(struct inode *inode, struct file *file,
			   int (*get)(void *, u64 *), int (*set)(void *, u64),
			   const char *fmt)
{
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
					  inode->i_private;

	/* The debugfs files are a reference to the kvm struct which
	 * is still valid when kvm_destroy_vm is called.
	 * To avoid the race between open and the removal of the debugfs
	 * directory we test against the users count.
	 */
4031
	if (!refcount_inc_not_zero(&stat_data->kvm->users_count))
4032 4033
		return -ENOENT;

4034
	if (simple_attr_open(inode, file, get,
4035 4036 4037
		    KVM_DBGFS_GET_MODE(stat_data->dbgfs_item) & 0222
		    ? set : NULL,
		    fmt)) {
4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055
		kvm_put_kvm(stat_data->kvm);
		return -ENOMEM;
	}

	return 0;
}

static int kvm_debugfs_release(struct inode *inode, struct file *file)
{
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
					  inode->i_private;

	simple_attr_release(inode, file);
	kvm_put_kvm(stat_data->kvm);

	return 0;
}

4056
static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val)
4057
{
4058
	*val = *(ulong *)((void *)kvm + offset);
4059

4060 4061 4062 4063 4064 4065
	return 0;
}

static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset)
{
	*(ulong *)((void *)kvm + offset) = 0;
4066 4067 4068 4069

	return 0;
}

4070
static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val)
4071
{
4072 4073
	int i;
	struct kvm_vcpu *vcpu;
4074

4075
	*val = 0;
4076

4077 4078
	kvm_for_each_vcpu(i, vcpu, kvm)
		*val += *(u64 *)((void *)vcpu + offset);
4079 4080 4081 4082

	return 0;
}

4083
static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset)
4084
{
4085 4086
	int i;
	struct kvm_vcpu *vcpu;
4087

4088 4089 4090 4091 4092
	kvm_for_each_vcpu(i, vcpu, kvm)
		*(u64 *)((void *)vcpu + offset) = 0;

	return 0;
}
4093

4094
static int kvm_stat_data_get(void *data, u64 *val)
4095
{
4096
	int r = -EFAULT;
4097 4098
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;

4099 4100 4101 4102 4103 4104 4105 4106 4107 4108
	switch (stat_data->dbgfs_item->kind) {
	case KVM_STAT_VM:
		r = kvm_get_stat_per_vm(stat_data->kvm,
					stat_data->dbgfs_item->offset, val);
		break;
	case KVM_STAT_VCPU:
		r = kvm_get_stat_per_vcpu(stat_data->kvm,
					  stat_data->dbgfs_item->offset, val);
		break;
	}
4109

4110
	return r;
4111 4112
}

4113
static int kvm_stat_data_clear(void *data, u64 val)
4114
{
4115
	int r = -EFAULT;
4116 4117 4118 4119 4120
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;

	if (val)
		return -EINVAL;

4121 4122 4123 4124 4125 4126 4127 4128 4129 4130
	switch (stat_data->dbgfs_item->kind) {
	case KVM_STAT_VM:
		r = kvm_clear_stat_per_vm(stat_data->kvm,
					  stat_data->dbgfs_item->offset);
		break;
	case KVM_STAT_VCPU:
		r = kvm_clear_stat_per_vcpu(stat_data->kvm,
					    stat_data->dbgfs_item->offset);
		break;
	}
4131

4132
	return r;
4133 4134
}

4135
static int kvm_stat_data_open(struct inode *inode, struct file *file)
4136 4137
{
	__simple_attr_check_format("%llu\n", 0ull);
4138 4139
	return kvm_debugfs_open(inode, file, kvm_stat_data_get,
				kvm_stat_data_clear, "%llu\n");
4140 4141
}

4142 4143 4144
static const struct file_operations stat_fops_per_vm = {
	.owner = THIS_MODULE,
	.open = kvm_stat_data_open,
4145
	.release = kvm_debugfs_release,
4146 4147 4148
	.read = simple_attr_read,
	.write = simple_attr_write,
	.llseek = no_llseek,
4149 4150
};

4151
static int vm_stat_get(void *_offset, u64 *val)
4152 4153 4154
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
4155
	u64 tmp_val;
4156

4157
	*val = 0;
J
Junaid Shahid 已提交
4158
	mutex_lock(&kvm_lock);
4159
	list_for_each_entry(kvm, &vm_list, vm_list) {
4160
		kvm_get_stat_per_vm(kvm, offset, &tmp_val);
4161 4162
		*val += tmp_val;
	}
J
Junaid Shahid 已提交
4163
	mutex_unlock(&kvm_lock);
4164
	return 0;
4165 4166
}

4167 4168 4169 4170 4171 4172 4173 4174
static int vm_stat_clear(void *_offset, u64 val)
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

	if (val)
		return -EINVAL;

J
Junaid Shahid 已提交
4175
	mutex_lock(&kvm_lock);
4176
	list_for_each_entry(kvm, &vm_list, vm_list) {
4177
		kvm_clear_stat_per_vm(kvm, offset);
4178
	}
J
Junaid Shahid 已提交
4179
	mutex_unlock(&kvm_lock);
4180 4181 4182 4183 4184

	return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
4185

4186
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
4187 4188 4189
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
4190
	u64 tmp_val;
A
Avi Kivity 已提交
4191

4192
	*val = 0;
J
Junaid Shahid 已提交
4193
	mutex_lock(&kvm_lock);
4194
	list_for_each_entry(kvm, &vm_list, vm_list) {
4195
		kvm_get_stat_per_vcpu(kvm, offset, &tmp_val);
4196 4197
		*val += tmp_val;
	}
J
Junaid Shahid 已提交
4198
	mutex_unlock(&kvm_lock);
4199
	return 0;
A
Avi Kivity 已提交
4200 4201
}

4202 4203 4204 4205 4206 4207 4208 4209
static int vcpu_stat_clear(void *_offset, u64 val)
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

	if (val)
		return -EINVAL;

J
Junaid Shahid 已提交
4210
	mutex_lock(&kvm_lock);
4211
	list_for_each_entry(kvm, &vm_list, vm_list) {
4212
		kvm_clear_stat_per_vcpu(kvm, offset);
4213
	}
J
Junaid Shahid 已提交
4214
	mutex_unlock(&kvm_lock);
4215 4216 4217 4218 4219 4220

	return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
			"%llu\n");
4221

4222
static const struct file_operations *stat_fops[] = {
4223 4224 4225
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
4226

4227 4228 4229 4230 4231 4232 4233 4234
static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
{
	struct kobj_uevent_env *env;
	unsigned long long created, active;

	if (!kvm_dev.this_device || !kvm)
		return;

J
Junaid Shahid 已提交
4235
	mutex_lock(&kvm_lock);
4236 4237 4238 4239 4240 4241 4242 4243
	if (type == KVM_EVENT_CREATE_VM) {
		kvm_createvm_count++;
		kvm_active_vms++;
	} else if (type == KVM_EVENT_DESTROY_VM) {
		kvm_active_vms--;
	}
	created = kvm_createvm_count;
	active = kvm_active_vms;
J
Junaid Shahid 已提交
4244
	mutex_unlock(&kvm_lock);
4245

4246
	env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT);
4247 4248 4249 4250 4251 4252
	if (!env)
		return;

	add_uevent_var(env, "CREATED=%llu", created);
	add_uevent_var(env, "COUNT=%llu", active);

4253
	if (type == KVM_EVENT_CREATE_VM) {
4254
		add_uevent_var(env, "EVENT=create");
4255 4256
		kvm->userspace_pid = task_pid_nr(current);
	} else if (type == KVM_EVENT_DESTROY_VM) {
4257
		add_uevent_var(env, "EVENT=destroy");
4258 4259
	}
	add_uevent_var(env, "PID=%d", kvm->userspace_pid);
4260

4261
	if (!IS_ERR_OR_NULL(kvm->debugfs_dentry)) {
4262
		char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT);
4263 4264 4265 4266 4267 4268

		if (p) {
			tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
			if (!IS_ERR(tmp))
				add_uevent_var(env, "STATS_PATH=%s", tmp);
			kfree(p);
4269 4270 4271 4272 4273 4274 4275 4276
		}
	}
	/* no need for checks, since we are adding at most only 5 keys */
	env->envp[env->envp_idx++] = NULL;
	kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
	kfree(env);
}

4277
static void kvm_init_debug(void)
A
Avi Kivity 已提交
4278 4279 4280
{
	struct kvm_stats_debugfs_item *p;

4281
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
4282

4283 4284
	kvm_debugfs_num_entries = 0;
	for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
4285 4286
		debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p),
				    kvm_debugfs_dir, (void *)(long)p->offset,
4287
				    stat_fops[p->kind]);
4288
	}
A
Avi Kivity 已提交
4289 4290
}

4291
static int kvm_suspend(void)
4292
{
4293
	if (kvm_usage_count)
4294
		hardware_disable_nolock(NULL);
4295 4296 4297
	return 0;
}

4298
static void kvm_resume(void)
4299
{
4300
	if (kvm_usage_count) {
4301 4302 4303
#ifdef CONFIG_LOCKDEP
		WARN_ON(lockdep_is_held(&kvm_count_lock));
#endif
4304
		hardware_enable_nolock(NULL);
4305
	}
4306 4307
}

4308
static struct syscore_ops kvm_syscore_ops = {
4309 4310 4311 4312
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

4313 4314 4315 4316 4317 4318 4319 4320 4321
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
4322

4323
	WRITE_ONCE(vcpu->preempted, false);
4324
	WRITE_ONCE(vcpu->ready, false);
4325

R
Radim Krčmář 已提交
4326 4327
	kvm_arch_sched_in(vcpu, cpu);

4328
	kvm_arch_vcpu_load(vcpu, cpu);
4329 4330 4331 4332 4333 4334 4335
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

4336
	if (current->state == TASK_RUNNING) {
4337
		WRITE_ONCE(vcpu->preempted, true);
4338 4339
		WRITE_ONCE(vcpu->ready, true);
	}
4340
	kvm_arch_vcpu_put(vcpu);
4341 4342
}

4343 4344 4345 4346 4347
static void check_processor_compat(void *rtn)
{
	*(int *)rtn = kvm_arch_check_processor_compat();
}

4348
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
4349
		  struct module *module)
A
Avi Kivity 已提交
4350 4351
{
	int r;
Y
Yang, Sheng 已提交
4352
	int cpu;
A
Avi Kivity 已提交
4353

4354 4355
	r = kvm_arch_init(opaque);
	if (r)
4356
		goto out_fail;
4357

4358 4359 4360 4361
	/*
	 * kvm_arch_init makes sure there's at most one caller
	 * for architectures that support multiple implementations,
	 * like intel and amd on x86.
P
Paolo Bonzini 已提交
4362 4363
	 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
	 * conflicts in case kvm is already setup for another implementation.
4364
	 */
P
Paolo Bonzini 已提交
4365 4366 4367
	r = kvm_irqfd_init();
	if (r)
		goto out_irqfd;
4368

4369
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
4370 4371 4372 4373
		r = -ENOMEM;
		goto out_free_0;
	}

4374
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
4375
	if (r < 0)
4376
		goto out_free_1;
A
Avi Kivity 已提交
4377

Y
Yang, Sheng 已提交
4378
	for_each_online_cpu(cpu) {
4379
		smp_call_function_single(cpu, check_processor_compat, &r, 1);
Y
Yang, Sheng 已提交
4380
		if (r < 0)
4381
			goto out_free_2;
Y
Yang, Sheng 已提交
4382 4383
	}

T
Thomas Gleixner 已提交
4384
	r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
4385
				      kvm_starting_cpu, kvm_dying_cpu);
A
Avi Kivity 已提交
4386
	if (r)
4387
		goto out_free_2;
A
Avi Kivity 已提交
4388 4389
	register_reboot_notifier(&kvm_reboot_notifier);

4390
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
4391 4392
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
4393 4394 4395 4396 4397 4398
	kvm_vcpu_cache =
		kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
					   SLAB_ACCOUNT,
					   offsetof(struct kvm_vcpu, arch),
					   sizeof_field(struct kvm_vcpu, arch),
					   NULL);
4399 4400
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
4401
		goto out_free_3;
4402 4403
	}

4404 4405 4406 4407
	r = kvm_async_pf_init();
	if (r)
		goto out_free;

A
Avi Kivity 已提交
4408
	kvm_chardev_ops.owner = module;
4409 4410
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
4411 4412 4413

	r = misc_register(&kvm_dev);
	if (r) {
X
Xiubo Li 已提交
4414
		pr_err("kvm: misc device register failed\n");
4415
		goto out_unreg;
A
Avi Kivity 已提交
4416 4417
	}

4418 4419
	register_syscore_ops(&kvm_syscore_ops);

4420 4421 4422
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

4423
	kvm_init_debug();
4424

P
Paolo Bonzini 已提交
4425 4426 4427
	r = kvm_vfio_ops_init();
	WARN_ON(r);

4428
	return 0;
A
Avi Kivity 已提交
4429

4430 4431
out_unreg:
	kvm_async_pf_deinit();
A
Avi Kivity 已提交
4432
out_free:
4433
	kmem_cache_destroy(kvm_vcpu_cache);
4434
out_free_3:
A
Avi Kivity 已提交
4435
	unregister_reboot_notifier(&kvm_reboot_notifier);
4436
	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4437
out_free_2:
4438
	kvm_arch_hardware_unsetup();
4439
out_free_1:
4440
	free_cpumask_var(cpus_hardware_enabled);
4441
out_free_0:
4442
	kvm_irqfd_exit();
P
Paolo Bonzini 已提交
4443
out_irqfd:
4444 4445
	kvm_arch_exit();
out_fail:
A
Avi Kivity 已提交
4446 4447
	return r;
}
4448
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
4449

4450
void kvm_exit(void)
A
Avi Kivity 已提交
4451
{
4452
	debugfs_remove_recursive(kvm_debugfs_dir);
A
Avi Kivity 已提交
4453
	misc_deregister(&kvm_dev);
4454
	kmem_cache_destroy(kvm_vcpu_cache);
4455
	kvm_async_pf_deinit();
4456
	unregister_syscore_ops(&kvm_syscore_ops);
A
Avi Kivity 已提交
4457
	unregister_reboot_notifier(&kvm_reboot_notifier);
4458
	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4459
	on_each_cpu(hardware_disable_nolock, NULL, 1);
4460
	kvm_arch_hardware_unsetup();
4461
	kvm_arch_exit();
4462
	kvm_irqfd_exit();
4463
	free_cpumask_var(cpus_hardware_enabled);
4464
	kvm_vfio_ops_exit();
A
Avi Kivity 已提交
4465
}
4466
EXPORT_SYMBOL_GPL(kvm_exit);
4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549

struct kvm_vm_worker_thread_context {
	struct kvm *kvm;
	struct task_struct *parent;
	struct completion init_done;
	kvm_vm_thread_fn_t thread_fn;
	uintptr_t data;
	int err;
};

static int kvm_vm_worker_thread(void *context)
{
	/*
	 * The init_context is allocated on the stack of the parent thread, so
	 * we have to locally copy anything that is needed beyond initialization
	 */
	struct kvm_vm_worker_thread_context *init_context = context;
	struct kvm *kvm = init_context->kvm;
	kvm_vm_thread_fn_t thread_fn = init_context->thread_fn;
	uintptr_t data = init_context->data;
	int err;

	err = kthread_park(current);
	/* kthread_park(current) is never supposed to return an error */
	WARN_ON(err != 0);
	if (err)
		goto init_complete;

	err = cgroup_attach_task_all(init_context->parent, current);
	if (err) {
		kvm_err("%s: cgroup_attach_task_all failed with err %d\n",
			__func__, err);
		goto init_complete;
	}

	set_user_nice(current, task_nice(init_context->parent));

init_complete:
	init_context->err = err;
	complete(&init_context->init_done);
	init_context = NULL;

	if (err)
		return err;

	/* Wait to be woken up by the spawner before proceeding. */
	kthread_parkme();

	if (!kthread_should_stop())
		err = thread_fn(kvm, data);

	return err;
}

int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn,
				uintptr_t data, const char *name,
				struct task_struct **thread_ptr)
{
	struct kvm_vm_worker_thread_context init_context = {};
	struct task_struct *thread;

	*thread_ptr = NULL;
	init_context.kvm = kvm;
	init_context.parent = current;
	init_context.thread_fn = thread_fn;
	init_context.data = data;
	init_completion(&init_context.init_done);

	thread = kthread_run(kvm_vm_worker_thread, &init_context,
			     "%s-%d", name, task_pid_nr(current));
	if (IS_ERR(thread))
		return PTR_ERR(thread);

	/* kthread_run is never supposed to return NULL */
	WARN_ON(thread == NULL);

	wait_for_completion(&init_context.init_done);

	if (!init_context.err)
		*thread_ptr = thread;

	return init_context.err;
}