kvm_main.c 109.7 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
A
Avi Kivity 已提交
2 3 4 5 6 7 8
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
9
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
10 11 12 13 14 15
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 */

16
#include <kvm/iodev.h>
A
Avi Kivity 已提交
17

18
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
19 20 21 22 23 24 25 26 27 28 29
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
30
#include <linux/syscore_ops.h>
A
Avi Kivity 已提交
31
#include <linux/cpu.h>
32
#include <linux/sched/signal.h>
33
#include <linux/sched/mm.h>
34
#include <linux/sched/stat.h>
35 36
#include <linux/cpumask.h>
#include <linux/smp.h>
37
#include <linux/anon_inodes.h>
38
#include <linux/profile.h>
39
#include <linux/kvm_para.h>
40
#include <linux/pagemap.h>
41
#include <linux/mman.h>
42
#include <linux/swap.h>
43
#include <linux/bitops.h>
44
#include <linux/spinlock.h>
45
#include <linux/compat.h>
46
#include <linux/srcu.h>
J
Joerg Roedel 已提交
47
#include <linux/hugetlb.h>
48
#include <linux/slab.h>
49 50
#include <linux/sort.h>
#include <linux/bsearch.h>
P
Paolo Bonzini 已提交
51
#include <linux/io.h>
52
#include <linux/lockdep.h>
53
#include <linux/kthread.h>
A
Avi Kivity 已提交
54

A
Avi Kivity 已提交
55
#include <asm/processor.h>
56
#include <asm/ioctl.h>
57
#include <linux/uaccess.h>
58
#include <asm/pgtable.h>
A
Avi Kivity 已提交
59

60
#include "coalesced_mmio.h"
61
#include "async_pf.h"
P
Paolo Bonzini 已提交
62
#include "vfio.h"
63

64 65 66
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

67 68 69
/* Worst case buffer size needed for holding an integer. */
#define ITOA_MAX_LEN 12

A
Avi Kivity 已提交
70 71 72
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

73
/* Architectures should define their poll value according to the halt latency */
74
unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
75
module_param(halt_poll_ns, uint, 0644);
76
EXPORT_SYMBOL_GPL(halt_poll_ns);
77

W
Wanpeng Li 已提交
78
/* Default doubles per-vcpu halt_poll_ns. */
79
unsigned int halt_poll_ns_grow = 2;
80
module_param(halt_poll_ns_grow, uint, 0644);
81
EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
W
Wanpeng Li 已提交
82

83 84 85 86 87
/* The start value to grow halt_poll_ns from */
unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
module_param(halt_poll_ns_grow_start, uint, 0644);
EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);

W
Wanpeng Li 已提交
88
/* Default resets per-vcpu halt_poll_ns . */
89
unsigned int halt_poll_ns_shrink;
90
module_param(halt_poll_ns_shrink, uint, 0644);
91
EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
W
Wanpeng Li 已提交
92

93 94 95
/*
 * Ordering of locks:
 *
96
 *	kvm->lock --> kvm->slots_lock --> kvm->irq_lock
97 98
 */

J
Junaid Shahid 已提交
99
DEFINE_MUTEX(kvm_lock);
100
static DEFINE_RAW_SPINLOCK(kvm_count_lock);
101
LIST_HEAD(vm_list);
102

103
static cpumask_var_t cpus_hardware_enabled;
104
static int kvm_usage_count;
105
static atomic_t hardware_enable_failed;
106

107
static struct kmem_cache *kvm_vcpu_cache;
A
Avi Kivity 已提交
108

109
static __read_mostly struct preempt_ops kvm_preempt_ops;
110
static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu);
111

112
struct dentry *kvm_debugfs_dir;
113
EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
A
Avi Kivity 已提交
114

115
static int kvm_debugfs_num_entries;
116
static const struct file_operations stat_fops_per_vm;
117

A
Avi Kivity 已提交
118 119
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
120
#ifdef CONFIG_KVM_COMPAT
121 122
static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
				  unsigned long arg);
123 124
#define KVM_COMPAT(c)	.compat_ioctl	= (c)
#else
125 126 127 128 129 130 131
/*
 * For architectures that don't implement a compat infrastructure,
 * adopt a double line of defense:
 * - Prevent a compat task from opening /dev/kvm
 * - If the open has been done by a 64bit task, and the KVM fd
 *   passed to a compat task, let the ioctls fail.
 */
132 133
static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
				unsigned long arg) { return -EINVAL; }
134 135 136 137 138 139 140

static int kvm_no_compat_open(struct inode *inode, struct file *file)
{
	return is_compat_task() ? -ENODEV : 0;
}
#define KVM_COMPAT(c)	.compat_ioctl	= kvm_no_compat_ioctl,	\
			.open		= kvm_no_compat_open
141
#endif
142 143
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
144

M
Marcelo Tosatti 已提交
145
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
146

147
static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
M
Marcelo Tosatti 已提交
148

149
__visible bool kvm_rebooting;
150
EXPORT_SYMBOL_GPL(kvm_rebooting);
151

152 153
static bool largepages_enabled = true;

154 155 156 157 158 159
#define KVM_EVENT_CREATE_VM 0
#define KVM_EVENT_DESTROY_VM 1
static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
static unsigned long long kvm_createvm_count;
static unsigned long long kvm_active_vms;

160 161
__weak int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
		unsigned long start, unsigned long end, bool blockable)
162
{
163
	return 0;
164 165
}

166 167 168 169 170 171 172 173 174 175 176 177 178 179
bool kvm_is_zone_device_pfn(kvm_pfn_t pfn)
{
	/*
	 * The metadata used by is_zone_device_page() to determine whether or
	 * not a page is ZONE_DEVICE is guaranteed to be valid if and only if
	 * the device has been pinned, e.g. by get_user_pages().  WARN if the
	 * page_count() is zero to help detect bad usage of this helper.
	 */
	if (!pfn_valid(pfn) || WARN_ON_ONCE(!page_count(pfn_to_page(pfn))))
		return false;

	return is_zone_device_page(pfn_to_page(pfn));
}

D
Dan Williams 已提交
180
bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
B
Ben-Ami Yassour 已提交
181
{
182 183 184 185 186
	/*
	 * ZONE_DEVICE pages currently set PG_reserved, but from a refcounting
	 * perspective they are "normal" pages, albeit with slightly different
	 * usage rules.
	 */
187
	if (pfn_valid(pfn))
188
		return PageReserved(pfn_to_page(pfn)) &&
189
		       !is_zero_pfn(pfn) &&
190
		       !kvm_is_zone_device_pfn(pfn);
B
Ben-Ami Yassour 已提交
191 192 193 194

	return true;
}

195 196 197 198 199 200 201 202 203 204
bool kvm_is_transparent_hugepage(kvm_pfn_t pfn)
{
	struct page *page = pfn_to_page(pfn);

	if (!PageTransCompoundMap(page))
		return false;

	return is_transparent_hugepage(compound_head(page));
}

A
Avi Kivity 已提交
205 206 207
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
208
void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
209
{
210
	int cpu = get_cpu();
211 212

	__this_cpu_write(kvm_running_vcpu, vcpu);
213
	preempt_notifier_register(&vcpu->preempt_notifier);
214
	kvm_arch_vcpu_load(vcpu, cpu);
215
	put_cpu();
A
Avi Kivity 已提交
216
}
217
EXPORT_SYMBOL_GPL(vcpu_load);
A
Avi Kivity 已提交
218

219
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
220
{
221
	preempt_disable();
222
	kvm_arch_vcpu_put(vcpu);
223
	preempt_notifier_unregister(&vcpu->preempt_notifier);
224
	__this_cpu_write(kvm_running_vcpu, NULL);
225
	preempt_enable();
A
Avi Kivity 已提交
226
}
227
EXPORT_SYMBOL_GPL(vcpu_put);
A
Avi Kivity 已提交
228

229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
/* TODO: merge with kvm_arch_vcpu_should_kick */
static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
{
	int mode = kvm_vcpu_exiting_guest_mode(vcpu);

	/*
	 * We need to wait for the VCPU to reenable interrupts and get out of
	 * READING_SHADOW_PAGE_TABLES mode.
	 */
	if (req & KVM_REQUEST_WAIT)
		return mode != OUTSIDE_GUEST_MODE;

	/*
	 * Need to kick a running VCPU, but otherwise there is nothing to do.
	 */
	return mode == IN_GUEST_MODE;
}

247 248 249 250
static void ack_flush(void *_completed)
{
}

251 252 253 254 255 256 257 258 259 260 261 262
static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
{
	if (unlikely(!cpus))
		cpus = cpu_online_mask;

	if (cpumask_empty(cpus))
		return false;

	smp_call_function_many(cpus, ack_flush, NULL, wait);
	return true;
}

263 264
bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
				 unsigned long *vcpu_bitmap, cpumask_var_t tmp)
265
{
266
	int i, cpu, me;
267
	struct kvm_vcpu *vcpu;
268
	bool called;
269

270
	me = get_cpu();
271

272
	kvm_for_each_vcpu(i, vcpu, kvm) {
273
		if (vcpu_bitmap && !test_bit(i, vcpu_bitmap))
274 275
			continue;

276
		kvm_make_request(req, vcpu);
277
		cpu = vcpu->cpu;
278

279 280
		if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
			continue;
281

282
		if (tmp != NULL && cpu != -1 && cpu != me &&
283
		    kvm_request_needs_ipi(vcpu, req))
284
			__cpumask_set_cpu(cpu, tmp);
285
	}
286 287

	called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT));
288
	put_cpu();
289 290 291 292 293 294 295 296 297 298 299

	return called;
}

bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
{
	cpumask_var_t cpus;
	bool called;

	zalloc_cpumask_var(&cpus, GFP_ATOMIC);

300
	called = kvm_make_vcpus_request_mask(kvm, req, NULL, cpus);
301

302
	free_cpumask_var(cpus);
303
	return called;
304 305
}

306
#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
307
void kvm_flush_remote_tlbs(struct kvm *kvm)
308
{
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
	/*
	 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
	 * kvm_make_all_cpus_request.
	 */
	long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);

	/*
	 * We want to publish modifications to the page tables before reading
	 * mode. Pairs with a memory barrier in arch-specific code.
	 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
	 * and smp_mb in walk_shadow_page_lockless_begin/end.
	 * - powerpc: smp_mb in kvmppc_prepare_to_enter.
	 *
	 * There is already an smp_mb__after_atomic() before
	 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
	 * barrier here.
	 */
326 327
	if (!kvm_arch_flush_remote_tlb(kvm)
	    || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
328
		++kvm->stat.remote_tlb_flush;
329
	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
330
}
331
EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
332
#endif
333

334 335
void kvm_reload_remote_mmus(struct kvm *kvm)
{
336
	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
337
}
338

339
static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
R
Rusty Russell 已提交
340 341 342 343 344
{
	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
345
	vcpu->pid = NULL;
346
	init_swait_queue_head(&vcpu->wq);
347
	kvm_async_pf_vcpu_init(vcpu);
R
Rusty Russell 已提交
348

349 350 351
	vcpu->pre_pcpu = -1;
	INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);

352 353
	kvm_vcpu_set_in_spin_loop(vcpu, false);
	kvm_vcpu_set_dy_eligible(vcpu, false);
354
	vcpu->preempted = false;
355
	vcpu->ready = false;
356
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
R
Rusty Russell 已提交
357 358
}

359 360 361
void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
{
	kvm_arch_vcpu_destroy(vcpu);
362

363 364 365 366 367 368 369
	/*
	 * No need for rcu_read_lock as VCPU_RUN is the only place that changes
	 * the vcpu->pid pointer, and at destruction time all file descriptors
	 * are already gone.
	 */
	put_pid(rcu_dereference_protected(vcpu->pid, 1));

370
	free_page((unsigned long)vcpu->run);
371
	kmem_cache_free(kvm_vcpu_cache, vcpu);
372 373 374
}
EXPORT_SYMBOL_GPL(kvm_vcpu_destroy);

375 376 377 378 379 380
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

381 382 383 384 385 386
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
387
	int idx;
388

389
	idx = srcu_read_lock(&kvm->srcu);
390 391
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
392 393 394 395

	if (kvm_set_spte_hva(kvm, address, pte))
		kvm_flush_remote_tlbs(kvm);

396
	spin_unlock(&kvm->mmu_lock);
397
	srcu_read_unlock(&kvm->srcu, idx);
398 399
}

400
static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
401
					const struct mmu_notifier_range *range)
402 403
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
404
	int need_tlb_flush = 0, idx;
405
	int ret;
406

407
	idx = srcu_read_lock(&kvm->srcu);
408 409 410 411 412 413 414
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
415
	need_tlb_flush = kvm_unmap_hva_range(kvm, range->start, range->end);
416
	need_tlb_flush |= kvm->tlbs_dirty;
417 418 419
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
420 421

	spin_unlock(&kvm->mmu_lock);
422

423
	ret = kvm_arch_mmu_notifier_invalidate_range(kvm, range->start,
424 425
					range->end,
					mmu_notifier_range_blockable(range));
426

427
	srcu_read_unlock(&kvm->srcu, idx);
428 429

	return ret;
430 431 432
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
433
					const struct mmu_notifier_range *range)
434 435 436 437 438 439 440 441 442 443
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
444
	smp_wmb();
445 446
	/*
	 * The above sequence increase must be visible before the
447 448
	 * below count decrease, which is ensured by the smp_wmb above
	 * in conjunction with the smp_rmb in mmu_notifier_retry().
449 450 451 452 453 454 455 456 457
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
A
Andres Lagar-Cavilla 已提交
458 459
					      unsigned long start,
					      unsigned long end)
460 461
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
462
	int young, idx;
463

464
	idx = srcu_read_lock(&kvm->srcu);
465 466
	spin_lock(&kvm->mmu_lock);

A
Andres Lagar-Cavilla 已提交
467
	young = kvm_age_hva(kvm, start, end);
468 469 470
	if (young)
		kvm_flush_remote_tlbs(kvm);

471 472 473
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

474 475 476
	return young;
}

477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long start,
					unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	/*
	 * Even though we do not flush TLB, this will still adversely
	 * affect performance on pre-Haswell Intel EPT, where there is
	 * no EPT Access Bit to clear so that we have to tear down EPT
	 * tables instead. If we find this unacceptable, we can always
	 * add a parameter to kvm_age_hva so that it effectively doesn't
	 * do anything on clear_young.
	 *
	 * Also note that currently we never issue secondary TLB flushes
	 * from clear_young, leaving this job up to the regular system
	 * cadence. If we find this inaccurate, we might come up with a
	 * more sophisticated heuristic later.
	 */
	young = kvm_age_hva(kvm, start, end);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

A
Andrea Arcangeli 已提交
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
				       struct mm_struct *mm,
				       unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	young = kvm_test_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

523 524 525 526
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
527 528 529
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
530
	kvm_arch_flush_shadow_all(kvm);
531
	srcu_read_unlock(&kvm->srcu, idx);
532 533
}

534 535 536 537
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
538
	.clear_young		= kvm_mmu_notifier_clear_young,
A
Andrea Arcangeli 已提交
539
	.test_young		= kvm_mmu_notifier_test_young,
540
	.change_pte		= kvm_mmu_notifier_change_pte,
541
	.release		= kvm_mmu_notifier_release,
542
};
543 544 545 546 547 548 549 550 551 552 553 554 555 556

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

557 558
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

559
static struct kvm_memslots *kvm_alloc_memslots(void)
560 561
{
	int i;
562
	struct kvm_memslots *slots;
563

564
	slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
565 566 567
	if (!slots)
		return NULL;

568
	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
569
		slots->id_to_index[i] = slots->memslots[i].id = i;
570 571 572 573 574 575 576 577 578 579 580 581 582

	return slots;
}

static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
	if (!memslot->dirty_bitmap)
		return;

	kvfree(memslot->dirty_bitmap);
	memslot->dirty_bitmap = NULL;
}

583
static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
584
{
585
	kvm_destroy_dirty_bitmap(slot);
586

587
	kvm_arch_free_memslot(kvm, slot);
588

589 590
	slot->flags = 0;
	slot->npages = 0;
591 592 593 594 595 596 597 598 599 600
}

static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
{
	struct kvm_memory_slot *memslot;

	if (!slots)
		return;

	kvm_for_each_memslot(memslot, slots)
601
		kvm_free_memslot(kvm, memslot);
602 603

	kvfree(slots);
604 605
}

606 607 608 609 610 611 612 613 614
static void kvm_destroy_vm_debugfs(struct kvm *kvm)
{
	int i;

	if (!kvm->debugfs_dentry)
		return;

	debugfs_remove_recursive(kvm->debugfs_dentry);

615 616 617 618 619
	if (kvm->debugfs_stat_data) {
		for (i = 0; i < kvm_debugfs_num_entries; i++)
			kfree(kvm->debugfs_stat_data[i]);
		kfree(kvm->debugfs_stat_data);
	}
620 621 622 623 624 625 626 627 628 629 630 631
}

static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
{
	char dir_name[ITOA_MAX_LEN * 2];
	struct kvm_stat_data *stat_data;
	struct kvm_stats_debugfs_item *p;

	if (!debugfs_initialized())
		return 0;

	snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
632
	kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir);
633 634 635

	kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
					 sizeof(*kvm->debugfs_stat_data),
636
					 GFP_KERNEL_ACCOUNT);
637 638 639 640
	if (!kvm->debugfs_stat_data)
		return -ENOMEM;

	for (p = debugfs_entries; p->name; p++) {
641
		stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
642 643 644 645
		if (!stat_data)
			return -ENOMEM;

		stat_data->kvm = kvm;
646
		stat_data->dbgfs_item = p;
647
		kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
648 649 650
		debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p),
				    kvm->debugfs_dentry, stat_data,
				    &stat_fops_per_vm);
651 652 653 654
	}
	return 0;
}

655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
/*
 * Called after the VM is otherwise initialized, but just before adding it to
 * the vm_list.
 */
int __weak kvm_arch_post_init_vm(struct kvm *kvm)
{
	return 0;
}

/*
 * Called just after removing the VM from the vm_list, but before doing any
 * other destruction.
 */
void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm)
{
}

672
static struct kvm *kvm_create_vm(unsigned long type)
A
Avi Kivity 已提交
673
{
674
	struct kvm *kvm = kvm_arch_alloc_vm();
675 676
	int r = -ENOMEM;
	int i;
A
Avi Kivity 已提交
677

678 679 680
	if (!kvm)
		return ERR_PTR(-ENOMEM);

681
	spin_lock_init(&kvm->mmu_lock);
V
Vegard Nossum 已提交
682
	mmgrab(current->mm);
683 684 685 686 687 688 689
	kvm->mm = current->mm;
	kvm_eventfd_init(kvm);
	mutex_init(&kvm->lock);
	mutex_init(&kvm->irq_lock);
	mutex_init(&kvm->slots_lock);
	INIT_LIST_HEAD(&kvm->devices);

690 691
	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);

692 693 694 695 696
	if (init_srcu_struct(&kvm->srcu))
		goto out_err_no_srcu;
	if (init_srcu_struct(&kvm->irq_srcu))
		goto out_err_no_irq_srcu;

697
	refcount_set(&kvm->users_count, 1);
698
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
699
		struct kvm_memslots *slots = kvm_alloc_memslots();
700

701
		if (!slots)
702
			goto out_err_no_arch_destroy_vm;
703
		/* Generations must be different for each address space. */
704
		slots->generation = i;
705
		rcu_assign_pointer(kvm->memslots[i], slots);
706
	}
707

M
Marcelo Tosatti 已提交
708
	for (i = 0; i < KVM_NR_BUSES; i++) {
709
		rcu_assign_pointer(kvm->buses[i],
710
			kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
711
		if (!kvm->buses[i])
712
			goto out_err_no_arch_destroy_vm;
M
Marcelo Tosatti 已提交
713
	}
714

715
	r = kvm_arch_init_vm(kvm, type);
716
	if (r)
717
		goto out_err_no_arch_destroy_vm;
718 719 720

	r = hardware_enable_all();
	if (r)
721
		goto out_err_no_disable;
722

723
#ifdef CONFIG_HAVE_KVM_IRQFD
724
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
725
#endif
A
Avi Kivity 已提交
726

727
	r = kvm_init_mmu_notifier(kvm);
728 729 730 731
	if (r)
		goto out_err_no_mmu_notifier;

	r = kvm_arch_post_init_vm(kvm);
732 733 734
	if (r)
		goto out_err;

J
Junaid Shahid 已提交
735
	mutex_lock(&kvm_lock);
736
	list_add(&kvm->vm_list, &vm_list);
J
Junaid Shahid 已提交
737
	mutex_unlock(&kvm_lock);
738

739 740
	preempt_notifier_inc();

741
	return kvm;
742 743

out_err:
744 745 746 747 748
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	if (kvm->mmu_notifier.ops)
		mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
#endif
out_err_no_mmu_notifier:
749
	hardware_disable_all();
750
out_err_no_disable:
751 752
	kvm_arch_destroy_vm(kvm);
out_err_no_arch_destroy_vm:
753
	WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
M
Marcelo Tosatti 已提交
754
	for (i = 0; i < KVM_NR_BUSES; i++)
755
		kfree(kvm_get_bus(kvm, i));
756
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
757
		kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
758 759 760 761
	cleanup_srcu_struct(&kvm->irq_srcu);
out_err_no_irq_srcu:
	cleanup_srcu_struct(&kvm->srcu);
out_err_no_srcu:
762
	kvm_arch_free_vm(kvm);
763
	mmdrop(current->mm);
764
	return ERR_PTR(r);
765 766
}

767 768
static void kvm_destroy_devices(struct kvm *kvm)
{
G
Geliang Tang 已提交
769
	struct kvm_device *dev, *tmp;
770

771 772 773 774 775
	/*
	 * We do not need to take the kvm->lock here, because nobody else
	 * has a reference to the struct kvm at this point and therefore
	 * cannot access the devices list anyhow.
	 */
G
Geliang Tang 已提交
776 777
	list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
		list_del(&dev->vm_node);
778 779 780 781
		dev->ops->destroy(dev);
	}
}

782 783
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
784
	int i;
785 786
	struct mm_struct *mm = kvm->mm;

787
	kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
788
	kvm_destroy_vm_debugfs(kvm);
789
	kvm_arch_sync_events(kvm);
J
Junaid Shahid 已提交
790
	mutex_lock(&kvm_lock);
791
	list_del(&kvm->vm_list);
J
Junaid Shahid 已提交
792
	mutex_unlock(&kvm_lock);
793 794
	kvm_arch_pre_destroy_vm(kvm);

795
	kvm_free_irq_routing(kvm);
796
	for (i = 0; i < KVM_NR_BUSES; i++) {
797
		struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
798 799 800

		if (bus)
			kvm_io_bus_destroy(bus);
801 802
		kvm->buses[i] = NULL;
	}
803
	kvm_coalesced_mmio_free(kvm);
804 805
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
806
#else
807
	kvm_arch_flush_shadow_all(kvm);
808
#endif
809
	kvm_arch_destroy_vm(kvm);
810
	kvm_destroy_devices(kvm);
811
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
812
		kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
813
	cleanup_srcu_struct(&kvm->irq_srcu);
814 815
	cleanup_srcu_struct(&kvm->srcu);
	kvm_arch_free_vm(kvm);
816
	preempt_notifier_dec();
817
	hardware_disable_all();
818
	mmdrop(mm);
819 820
}

I
Izik Eidus 已提交
821 822
void kvm_get_kvm(struct kvm *kvm)
{
823
	refcount_inc(&kvm->users_count);
I
Izik Eidus 已提交
824 825 826 827 828
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
829
	if (refcount_dec_and_test(&kvm->users_count))
I
Izik Eidus 已提交
830 831 832 833
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);

834 835 836 837 838 839 840 841 842 843 844 845
/*
 * Used to put a reference that was taken on behalf of an object associated
 * with a user-visible file descriptor, e.g. a vcpu or device, if installation
 * of the new file descriptor fails and the reference cannot be transferred to
 * its final owner.  In such cases, the caller is still actively using @kvm and
 * will fail miserably if the refcount unexpectedly hits zero.
 */
void kvm_put_kvm_no_destroy(struct kvm *kvm)
{
	WARN_ON(refcount_dec_and_test(&kvm->users_count));
}
EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy);
I
Izik Eidus 已提交
846

847 848 849 850
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
851 852
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
853
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
854 855 856
	return 0;
}

857 858
/*
 * Allocation size is twice as large as the actual dirty bitmap size.
859
 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
860
 */
861 862
static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
{
863
	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
864

865
	memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);
866 867 868 869 870 871
	if (!memslot->dirty_bitmap)
		return -ENOMEM;

	return 0;
}

872
/*
873 874 875 876
 * Insert memslot and re-sort memslots based on their GFN,
 * so binary search could be used to lookup GFN.
 * Sorting algorithm takes advantage of having initially
 * sorted array and known changed memslot position.
877
 */
878
static void update_memslots(struct kvm_memslots *slots,
879 880
			    struct kvm_memory_slot *new,
			    enum kvm_mr_change change)
881
{
882 883
	int id = new->id;
	int i = slots->id_to_index[id];
884
	struct kvm_memory_slot *mslots = slots->memslots;
885

886
	WARN_ON(mslots[i].id != id);
887 888 889 890 891 892 893 894 895 896 897
	switch (change) {
	case KVM_MR_CREATE:
		slots->used_slots++;
		WARN_ON(mslots[i].npages || !new->npages);
		break;
	case KVM_MR_DELETE:
		slots->used_slots--;
		WARN_ON(new->npages || !mslots[i].npages);
		break;
	default:
		break;
898
	}
899

900
	while (i < KVM_MEM_SLOTS_NUM - 1 &&
901 902 903
	       new->base_gfn <= mslots[i + 1].base_gfn) {
		if (!mslots[i + 1].npages)
			break;
904 905 906 907
		mslots[i] = mslots[i + 1];
		slots->id_to_index[mslots[i].id] = i;
		i++;
	}
908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924

	/*
	 * The ">=" is needed when creating a slot with base_gfn == 0,
	 * so that it moves before all those with base_gfn == npages == 0.
	 *
	 * On the other hand, if new->npages is zero, the above loop has
	 * already left i pointing to the beginning of the empty part of
	 * mslots, and the ">=" would move the hole backwards in this
	 * case---which is wrong.  So skip the loop when deleting a slot.
	 */
	if (new->npages) {
		while (i > 0 &&
		       new->base_gfn >= mslots[i - 1].base_gfn) {
			mslots[i] = mslots[i - 1];
			slots->id_to_index[mslots[i].id] = i;
			i--;
		}
925 926
	} else
		WARN_ON_ONCE(i != slots->used_slots);
927

928 929
	mslots[i] = *new;
	slots->id_to_index[mslots[i].id] = i;
930 931
}

932
static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
933
{
X
Xiao Guangrong 已提交
934 935
	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;

936
#ifdef __KVM_HAVE_READONLY_MEM
X
Xiao Guangrong 已提交
937 938 939 940
	valid_flags |= KVM_MEM_READONLY;
#endif

	if (mem->flags & ~valid_flags)
941 942 943 944 945
		return -EINVAL;

	return 0;
}

946
static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
947
		int as_id, struct kvm_memslots *slots)
948
{
949
	struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
950
	u64 gen = old_memslots->generation;
951

952 953
	WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
	slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
954

955
	rcu_assign_pointer(kvm->memslots[as_id], slots);
956
	synchronize_srcu_expedited(&kvm->srcu);
957

958
	/*
959
	 * Increment the new memslot generation a second time, dropping the
M
Miaohe Lin 已提交
960
	 * update in-progress flag and incrementing the generation based on
961 962 963 964 965 966
	 * the number of address spaces.  This provides a unique and easily
	 * identifiable generation number while the memslots are in flux.
	 */
	gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;

	/*
967 968 969
	 * Generations must be unique even across address spaces.  We do not need
	 * a global counter for that, instead the generation space is evenly split
	 * across address spaces.  For example, with two address spaces, address
970 971
	 * space 0 will use generations 0, 2, 4, ... while address space 1 will
	 * use generations 1, 3, 5, ...
972
	 */
973
	gen += KVM_ADDRESS_SPACE_NUM;
974

975
	kvm_arch_memslots_updated(kvm, gen);
976

977
	slots->generation = gen;
978 979

	return old_memslots;
980 981
}

982 983
static int kvm_set_memslot(struct kvm *kvm,
			   const struct kvm_userspace_memory_region *mem,
984
			   struct kvm_memory_slot *old,
985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
			   struct kvm_memory_slot *new, int as_id,
			   enum kvm_mr_change change)
{
	struct kvm_memory_slot *slot;
	struct kvm_memslots *slots;
	int r;

	slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
	if (!slots)
		return -ENOMEM;
	memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));

	if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
		/*
		 * Note, the INVALID flag needs to be in the appropriate entry
		 * in the freshly allocated memslots, not in @old or @new.
		 */
		slot = id_to_memslot(slots, old->id);
		slot->flags |= KVM_MEMSLOT_INVALID;

		/*
		 * We can re-use the old memslots, the only difference from the
		 * newly installed memslots is the invalid flag, which will get
		 * dropped by update_memslots anyway.  We'll also revert to the
		 * old memslots if preparing the new memory region fails.
		 */
		slots = install_new_memslots(kvm, as_id, slots);

		/* From this point no new shadow pages pointing to a deleted,
		 * or moved, memslot will be created.
		 *
		 * validation of sp->gfn happens in:
		 *	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
		 *	- kvm_is_visible_gfn (mmu_check_root)
		 */
		kvm_arch_flush_shadow_memslot(kvm, slot);
	}

	r = kvm_arch_prepare_memory_region(kvm, new, mem, change);
	if (r)
		goto out_slots;

	update_memslots(slots, new, change);
	slots = install_new_memslots(kvm, as_id, slots);

	kvm_arch_commit_memory_region(kvm, mem, old, new, change);

	kvfree(slots);
	return 0;

out_slots:
	if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
		slots = install_new_memslots(kvm, as_id, slots);
	kvfree(slots);
	return r;
}

1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
static int kvm_delete_memslot(struct kvm *kvm,
			      const struct kvm_userspace_memory_region *mem,
			      struct kvm_memory_slot *old, int as_id)
{
	struct kvm_memory_slot new;
	int r;

	if (!old->npages)
		return -EINVAL;

	memset(&new, 0, sizeof(new));
	new.id = old->id;

	r = kvm_set_memslot(kvm, mem, old, &new, as_id, KVM_MR_DELETE);
	if (r)
		return r;

1059
	kvm_free_memslot(kvm, old);
1060 1061 1062
	return 0;
}

A
Avi Kivity 已提交
1063 1064 1065 1066 1067
/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
1068
 *
1069
 * Must be called holding kvm->slots_lock for write.
A
Avi Kivity 已提交
1070
 */
1071
int __kvm_set_memory_region(struct kvm *kvm,
1072
			    const struct kvm_userspace_memory_region *mem)
A
Avi Kivity 已提交
1073
{
1074
	int r;
A
Avi Kivity 已提交
1075
	gfn_t base_gfn;
1076
	unsigned long npages;
1077
	struct kvm_memory_slot *slot;
A
Avi Kivity 已提交
1078
	struct kvm_memory_slot old, new;
1079
	int as_id, id;
1080
	enum kvm_mr_change change;
A
Avi Kivity 已提交
1081

1082 1083
	r = check_memory_region_flags(mem);
	if (r)
1084
		return r;
1085

1086 1087 1088
	as_id = mem->slot >> 16;
	id = (u16)mem->slot;

A
Avi Kivity 已提交
1089 1090
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
1091
		return -EINVAL;
A
Avi Kivity 已提交
1092
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
1093
		return -EINVAL;
1094
	/* We can read the guest memory with __xxx_user() later on. */
1095
	if ((id < KVM_USER_MEM_SLOTS) &&
1096
	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
1097
	     !access_ok((void __user *)(unsigned long)mem->userspace_addr,
1098
			mem->memory_size)))
1099
		return -EINVAL;
1100
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
1101
		return -EINVAL;
A
Avi Kivity 已提交
1102
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
1103
		return -EINVAL;
A
Avi Kivity 已提交
1104

1105
	slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
A
Avi Kivity 已提交
1106 1107 1108
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

1109
	if (npages > KVM_MEM_MAX_NR_PAGES)
1110
		return -EINVAL;
1111

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
	/*
	 * Make a full copy of the old memslot, the pointer will become stale
	 * when the memslots are re-sorted by update_memslots(), and the old
	 * memslot needs to be referenced after calling update_memslots(), e.g.
	 * to free its resources and for arch specific behavior.
	 */
	old = *slot;
	if (!mem->memory_size)
		return kvm_delete_memslot(kvm, mem, &old, as_id);

	new = old;
A
Avi Kivity 已提交
1123

1124
	new.id = id;
A
Avi Kivity 已提交
1125 1126 1127
	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;
1128
	new.userspace_addr = mem->userspace_addr;
A
Avi Kivity 已提交
1129

1130 1131 1132 1133 1134 1135
	if (!old.npages) {
		change = KVM_MR_CREATE;
	} else { /* Modify an existing slot. */
		if ((new.userspace_addr != old.userspace_addr) ||
		    (npages != old.npages) ||
		    ((new.flags ^ old.flags) & KVM_MEM_READONLY))
1136
			return -EINVAL;
1137

1138 1139 1140 1141 1142 1143
		if (base_gfn != old.base_gfn)
			change = KVM_MR_MOVE;
		else if (new.flags != old.flags)
			change = KVM_MR_FLAGS_ONLY;
		else /* Nothing to change. */
			return 0;
1144
	}
A
Avi Kivity 已提交
1145

1146
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
1147
		/* Check for overlaps */
1148
		kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
1149
			if (slot->id == id)
1150 1151 1152
				continue;
			if (!((base_gfn + npages <= slot->base_gfn) ||
			      (base_gfn >= slot->base_gfn + slot->npages)))
1153
				return -EEXIST;
1154
		}
A
Avi Kivity 已提交
1155 1156
	}

1157 1158 1159 1160
	/* Allocate/free page dirty bitmap as needed */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
		new.dirty_bitmap = NULL;
	else if (!new.dirty_bitmap) {
1161 1162 1163
		r = kvm_create_dirty_bitmap(&new);
		if (r)
			return r;
A
Avi Kivity 已提交
1164 1165
	}

1166 1167 1168
	r = kvm_set_memslot(kvm, mem, &old, &new, as_id, change);
	if (r)
		goto out_bitmap;
1169

1170 1171
	if (old.dirty_bitmap && !new.dirty_bitmap)
		kvm_destroy_dirty_bitmap(&old);
A
Avi Kivity 已提交
1172 1173
	return 0;

1174 1175 1176
out_bitmap:
	if (new.dirty_bitmap && !old.dirty_bitmap)
		kvm_destroy_dirty_bitmap(&new);
A
Avi Kivity 已提交
1177
	return r;
1178
}
1179 1180 1181
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
1182
			  const struct kvm_userspace_memory_region *mem)
1183 1184 1185
{
	int r;

1186
	mutex_lock(&kvm->slots_lock);
1187
	r = __kvm_set_memory_region(kvm, mem);
1188
	mutex_unlock(&kvm->slots_lock);
1189 1190
	return r;
}
1191 1192
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

1193 1194
static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
					  struct kvm_userspace_memory_region *mem)
1195
{
1196
	if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1197
		return -EINVAL;
1198

1199
	return kvm_set_memory_region(kvm, mem);
A
Avi Kivity 已提交
1200 1201
}

1202 1203
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
1204
{
1205
	struct kvm_memslots *slots;
A
Avi Kivity 已提交
1206
	struct kvm_memory_slot *memslot;
1207
	int i, as_id, id;
1208
	unsigned long n;
A
Avi Kivity 已提交
1209 1210
	unsigned long any = 0;

1211 1212 1213
	as_id = log->slot >> 16;
	id = (u16)log->slot;
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1214
		return -EINVAL;
A
Avi Kivity 已提交
1215

1216 1217
	slots = __kvm_memslots(kvm, as_id);
	memslot = id_to_memslot(slots, id);
A
Avi Kivity 已提交
1218
	if (!memslot->dirty_bitmap)
1219
		return -ENOENT;
A
Avi Kivity 已提交
1220

1221
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
1222

1223
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
1224 1225 1226
		any = memslot->dirty_bitmap[i];

	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1227
		return -EFAULT;
A
Avi Kivity 已提交
1228

1229 1230
	if (any)
		*is_dirty = 1;
1231
	return 0;
A
Avi Kivity 已提交
1232
}
1233
EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
A
Avi Kivity 已提交
1234

1235 1236
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
/**
J
Jiang Biao 已提交
1237
 * kvm_get_dirty_log_protect - get a snapshot of dirty pages
1238
 *	and reenable dirty page tracking for the corresponding pages.
1239 1240
 * @kvm:	pointer to kvm instance
 * @log:	slot id and address to which we copy the log
J
Jiang Biao 已提交
1241
 * @flush:	true if TLB flush is needed by caller
1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
 *
 * We need to keep it in mind that VCPU threads can write to the bitmap
 * concurrently. So, to avoid losing track of dirty pages we keep the
 * following order:
 *
 *    1. Take a snapshot of the bit and clear it if needed.
 *    2. Write protect the corresponding page.
 *    3. Copy the snapshot to the userspace.
 *    4. Upon return caller flushes TLB's if needed.
 *
 * Between 2 and 4, the guest may write to the page using the remaining TLB
 * entry.  This is not a problem because the page is reported dirty using
 * the snapshot taken before and step 4 ensures that writes done after
 * exiting to userspace will be logged for the next call.
 *
 */
int kvm_get_dirty_log_protect(struct kvm *kvm,
1259
			struct kvm_dirty_log *log, bool *flush)
1260
{
1261
	struct kvm_memslots *slots;
1262
	struct kvm_memory_slot *memslot;
1263
	int i, as_id, id;
1264 1265 1266 1267
	unsigned long n;
	unsigned long *dirty_bitmap;
	unsigned long *dirty_bitmap_buffer;

1268 1269 1270
	as_id = log->slot >> 16;
	id = (u16)log->slot;
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1271
		return -EINVAL;
1272

1273 1274
	slots = __kvm_memslots(kvm, as_id);
	memslot = id_to_memslot(slots, id);
1275 1276 1277

	dirty_bitmap = memslot->dirty_bitmap;
	if (!dirty_bitmap)
1278
		return -ENOENT;
1279 1280

	n = kvm_dirty_bitmap_bytes(memslot);
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294
	*flush = false;
	if (kvm->manual_dirty_log_protect) {
		/*
		 * Unlike kvm_get_dirty_log, we always return false in *flush,
		 * because no flush is needed until KVM_CLEAR_DIRTY_LOG.  There
		 * is some code duplication between this function and
		 * kvm_get_dirty_log, but hopefully all architecture
		 * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
		 * can be eliminated.
		 */
		dirty_bitmap_buffer = dirty_bitmap;
	} else {
		dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
		memset(dirty_bitmap_buffer, 0, n);
1295

1296 1297 1298 1299
		spin_lock(&kvm->mmu_lock);
		for (i = 0; i < n / sizeof(long); i++) {
			unsigned long mask;
			gfn_t offset;
1300

1301 1302 1303 1304 1305 1306 1307
			if (!dirty_bitmap[i])
				continue;

			*flush = true;
			mask = xchg(&dirty_bitmap[i], 0);
			dirty_bitmap_buffer[i] = mask;

1308 1309 1310
			offset = i * BITS_PER_LONG;
			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
								offset, mask);
1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
		}
		spin_unlock(&kvm->mmu_lock);
	}

	if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);

/**
 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
 *	and reenable dirty page tracking for the corresponding pages.
 * @kvm:	pointer to kvm instance
 * @log:	slot id and address from which to fetch the bitmap of dirty pages
J
Jiang Biao 已提交
1326
 * @flush:	true if TLB flush is needed by caller
1327 1328 1329 1330 1331 1332
 */
int kvm_clear_dirty_log_protect(struct kvm *kvm,
				struct kvm_clear_dirty_log *log, bool *flush)
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
1333
	int as_id, id;
1334
	gfn_t offset;
1335
	unsigned long i, n;
1336 1337 1338 1339 1340 1341 1342 1343
	unsigned long *dirty_bitmap;
	unsigned long *dirty_bitmap_buffer;

	as_id = log->slot >> 16;
	id = (u16)log->slot;
	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
		return -EINVAL;

1344
	if (log->first_page & 63)
1345 1346 1347 1348 1349 1350 1351 1352 1353
		return -EINVAL;

	slots = __kvm_memslots(kvm, as_id);
	memslot = id_to_memslot(slots, id);

	dirty_bitmap = memslot->dirty_bitmap;
	if (!dirty_bitmap)
		return -ENOENT;

1354
	n = ALIGN(log->num_pages, BITS_PER_LONG) / 8;
1355 1356

	if (log->first_page > memslot->npages ||
1357 1358 1359
	    log->num_pages > memslot->npages - log->first_page ||
	    (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))
	    return -EINVAL;
1360

1361
	*flush = false;
1362 1363 1364
	dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
	if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
		return -EFAULT;
1365

1366
	spin_lock(&kvm->mmu_lock);
1367 1368
	for (offset = log->first_page, i = offset / BITS_PER_LONG,
		 n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
1369 1370 1371 1372
	     i++, offset += BITS_PER_LONG) {
		unsigned long mask = *dirty_bitmap_buffer++;
		atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
		if (!mask)
1373 1374
			continue;

1375
		mask &= atomic_long_fetch_andnot(mask, p);
1376

1377 1378 1379 1380 1381 1382
		/*
		 * mask contains the bits that really have been cleared.  This
		 * never includes any bits beyond the length of the memslot (if
		 * the length is not aligned to 64 pages), therefore it is not
		 * a problem if userspace sets them in log->dirty_bitmap.
		*/
1383
		if (mask) {
1384
			*flush = true;
1385 1386 1387
			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
								offset, mask);
		}
1388 1389
	}
	spin_unlock(&kvm->mmu_lock);
1390

1391
	return 0;
1392
}
1393
EXPORT_SYMBOL_GPL(kvm_clear_dirty_log_protect);
1394 1395
#endif

1396 1397 1398 1399 1400
bool kvm_largepages_enabled(void)
{
	return largepages_enabled;
}

1401 1402 1403 1404 1405 1406
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

1407 1408 1409 1410
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
A
Avi Kivity 已提交
1411
EXPORT_SYMBOL_GPL(gfn_to_memslot);
A
Avi Kivity 已提交
1412

1413 1414 1415 1416 1417
struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
}

1418
bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1419
{
1420
	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1421

1422
	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1423
	      memslot->flags & KVM_MEMSLOT_INVALID)
1424
		return false;
1425

1426
	return true;
1427 1428 1429
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

1430
unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn)
J
Joerg Roedel 已提交
1431 1432 1433 1434 1435 1436
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

1437
	addr = kvm_vcpu_gfn_to_hva_prot(vcpu, gfn, NULL);
J
Joerg Roedel 已提交
1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

X
Xiao Guangrong 已提交
1454 1455 1456 1457 1458 1459 1460
static bool memslot_is_readonly(struct kvm_memory_slot *slot)
{
	return slot->flags & KVM_MEM_READONLY;
}

static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				       gfn_t *nr_pages, bool write)
I
Izik Eidus 已提交
1461
{
1462
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
X
Xiao Guangrong 已提交
1463
		return KVM_HVA_ERR_BAD;
1464

X
Xiao Guangrong 已提交
1465 1466
	if (memslot_is_readonly(slot) && write)
		return KVM_HVA_ERR_RO_BAD;
1467 1468 1469 1470

	if (nr_pages)
		*nr_pages = slot->npages - (gfn - slot->base_gfn);

X
Xiao Guangrong 已提交
1471
	return __gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
1472
}
1473

X
Xiao Guangrong 已提交
1474 1475 1476 1477
static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				     gfn_t *nr_pages)
{
	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
I
Izik Eidus 已提交
1478
}
1479

X
Xiao Guangrong 已提交
1480
unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1481
					gfn_t gfn)
X
Xiao Guangrong 已提交
1482 1483 1484 1485 1486
{
	return gfn_to_hva_many(slot, gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);

1487 1488
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
1489
	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1490
}
1491
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1492

1493 1494 1495 1496 1497 1498
unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);

1499
/*
1500 1501 1502 1503 1504 1505
 * Return the hva of a @gfn and the R/W attribute if possible.
 *
 * @slot: the kvm_memory_slot which contains @gfn
 * @gfn: the gfn to be translated
 * @writable: used to return the read/write attribute of the @slot if the hva
 * is valid and @writable is not NULL
1506
 */
1507 1508
unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
				      gfn_t gfn, bool *writable)
1509
{
1510 1511 1512
	unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);

	if (!kvm_is_error_hva(hva) && writable)
1513 1514
		*writable = !memslot_is_readonly(slot);

1515
	return hva;
1516 1517
}

1518 1519 1520 1521 1522 1523 1524
unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

1525 1526 1527 1528 1529 1530 1531
unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
{
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

1532 1533
static inline int check_user_page_hwpoison(unsigned long addr)
{
L
Lorenzo Stoakes 已提交
1534
	int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
1535

L
Lorenzo Stoakes 已提交
1536
	rc = get_user_pages(addr, 1, flags, NULL, NULL);
1537 1538 1539
	return rc == -EHWPOISON;
}

X
Xiao Guangrong 已提交
1540
/*
1541 1542
 * The fast path to get the writable pfn which will be stored in @pfn,
 * true indicates success, otherwise false is returned.  It's also the
M
Miaohe Lin 已提交
1543
 * only part that runs if we can in atomic context.
X
Xiao Guangrong 已提交
1544
 */
1545 1546
static bool hva_to_pfn_fast(unsigned long addr, bool write_fault,
			    bool *writable, kvm_pfn_t *pfn)
A
Avi Kivity 已提交
1547
{
1548
	struct page *page[1];
X
Xiao Guangrong 已提交
1549
	int npages;
A
Avi Kivity 已提交
1550

1551 1552 1553 1554 1555 1556 1557
	/*
	 * Fast pin a writable pfn only if it is a write fault request
	 * or the caller allows to map a writable pfn for a read fault
	 * request.
	 */
	if (!(write_fault || writable))
		return false;
1558

X
Xiao Guangrong 已提交
1559 1560 1561
	npages = __get_user_pages_fast(addr, 1, 1, page);
	if (npages == 1) {
		*pfn = page_to_pfn(page[0]);
1562

X
Xiao Guangrong 已提交
1563 1564 1565 1566
		if (writable)
			*writable = true;
		return true;
	}
1567

X
Xiao Guangrong 已提交
1568 1569
	return false;
}
1570

X
Xiao Guangrong 已提交
1571 1572 1573 1574 1575
/*
 * The slow path to get the pfn of the specified host virtual address,
 * 1 indicates success, -errno is returned if error is detected.
 */
static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
D
Dan Williams 已提交
1576
			   bool *writable, kvm_pfn_t *pfn)
X
Xiao Guangrong 已提交
1577
{
1578 1579
	unsigned int flags = FOLL_HWPOISON;
	struct page *page;
X
Xiao Guangrong 已提交
1580
	int npages = 0;
1581

X
Xiao Guangrong 已提交
1582 1583 1584 1585 1586
	might_sleep();

	if (writable)
		*writable = write_fault;

1587 1588 1589 1590
	if (write_fault)
		flags |= FOLL_WRITE;
	if (async)
		flags |= FOLL_NOWAIT;
1591

1592
	npages = get_user_pages_unlocked(addr, 1, &page, flags);
X
Xiao Guangrong 已提交
1593 1594 1595 1596
	if (npages != 1)
		return npages;

	/* map read fault as writable if possible */
1597
	if (unlikely(!write_fault) && writable) {
1598
		struct page *wpage;
X
Xiao Guangrong 已提交
1599

1600
		if (__get_user_pages_fast(addr, 1, 1, &wpage) == 1) {
X
Xiao Guangrong 已提交
1601
			*writable = true;
1602 1603
			put_page(page);
			page = wpage;
1604
		}
1605
	}
1606
	*pfn = page_to_pfn(page);
X
Xiao Guangrong 已提交
1607 1608
	return npages;
}
I
Izik Eidus 已提交
1609

X
Xiao Guangrong 已提交
1610 1611 1612 1613
static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
{
	if (unlikely(!(vma->vm_flags & VM_READ)))
		return false;
1614

X
Xiao Guangrong 已提交
1615 1616
	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
		return false;
1617

X
Xiao Guangrong 已提交
1618 1619
	return true;
}
1620

1621 1622
static int hva_to_pfn_remapped(struct vm_area_struct *vma,
			       unsigned long addr, bool *async,
1623 1624
			       bool write_fault, bool *writable,
			       kvm_pfn_t *p_pfn)
1625
{
1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649
	unsigned long pfn;
	int r;

	r = follow_pfn(vma, addr, &pfn);
	if (r) {
		/*
		 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
		 * not call the fault handler, so do it here.
		 */
		bool unlocked = false;
		r = fixup_user_fault(current, current->mm, addr,
				     (write_fault ? FAULT_FLAG_WRITE : 0),
				     &unlocked);
		if (unlocked)
			return -EAGAIN;
		if (r)
			return r;

		r = follow_pfn(vma, addr, &pfn);
		if (r)
			return r;

	}

1650 1651
	if (writable)
		*writable = true;
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666

	/*
	 * Get a reference here because callers of *hva_to_pfn* and
	 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
	 * returned pfn.  This is only needed if the VMA has VM_MIXEDMAP
	 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
	 * simply do nothing for reserved pfns.
	 *
	 * Whoever called remap_pfn_range is also going to call e.g.
	 * unmap_mapping_range before the underlying pages are freed,
	 * causing a call to our MMU notifier.
	 */ 
	kvm_get_pfn(pfn);

	*p_pfn = pfn;
1667 1668 1669
	return 0;
}

1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683
/*
 * Pin guest page in memory and return its pfn.
 * @addr: host virtual address which maps memory to the guest
 * @atomic: whether this function can sleep
 * @async: whether this function need to wait IO complete if the
 *         host page is not in the memory
 * @write_fault: whether we should get a writable host page
 * @writable: whether it allows to map a writable host page for !@write_fault
 *
 * The function will map a writable host page for these two cases:
 * 1): @write_fault = true
 * 2): @write_fault = false && @writable, @writable will tell the caller
 *     whether the mapping is writable.
 */
D
Dan Williams 已提交
1684
static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
X
Xiao Guangrong 已提交
1685 1686 1687
			bool write_fault, bool *writable)
{
	struct vm_area_struct *vma;
D
Dan Williams 已提交
1688
	kvm_pfn_t pfn = 0;
1689
	int npages, r;
1690

X
Xiao Guangrong 已提交
1691 1692
	/* we can do it either atomically or asynchronously, not both */
	BUG_ON(atomic && async);
1693

1694
	if (hva_to_pfn_fast(addr, write_fault, writable, &pfn))
X
Xiao Guangrong 已提交
1695 1696 1697 1698 1699 1700 1701 1702
		return pfn;

	if (atomic)
		return KVM_PFN_ERR_FAULT;

	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
	if (npages == 1)
		return pfn;
1703

X
Xiao Guangrong 已提交
1704 1705 1706 1707 1708 1709 1710
	down_read(&current->mm->mmap_sem);
	if (npages == -EHWPOISON ||
	      (!async && check_user_page_hwpoison(addr))) {
		pfn = KVM_PFN_ERR_HWPOISON;
		goto exit;
	}

1711
retry:
X
Xiao Guangrong 已提交
1712 1713 1714 1715
	vma = find_vma_intersection(current->mm, addr, addr + 1);

	if (vma == NULL)
		pfn = KVM_PFN_ERR_FAULT;
1716
	else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
1717
		r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
1718 1719
		if (r == -EAGAIN)
			goto retry;
1720 1721
		if (r < 0)
			pfn = KVM_PFN_ERR_FAULT;
X
Xiao Guangrong 已提交
1722
	} else {
X
Xiao Guangrong 已提交
1723
		if (async && vma_is_valid(vma, write_fault))
X
Xiao Guangrong 已提交
1724 1725 1726 1727 1728
			*async = true;
		pfn = KVM_PFN_ERR_FAULT;
	}
exit:
	up_read(&current->mm->mmap_sem);
1729
	return pfn;
1730 1731
}

D
Dan Williams 已提交
1732 1733 1734
kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
			       bool atomic, bool *async, bool write_fault,
			       bool *writable)
1735
{
X
Xiao Guangrong 已提交
1736 1737
	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);

1738 1739 1740
	if (addr == KVM_HVA_ERR_RO_BAD) {
		if (writable)
			*writable = false;
X
Xiao Guangrong 已提交
1741
		return KVM_PFN_ERR_RO_FAULT;
1742
	}
X
Xiao Guangrong 已提交
1743

1744 1745 1746
	if (kvm_is_error_hva(addr)) {
		if (writable)
			*writable = false;
1747
		return KVM_PFN_NOSLOT;
1748
	}
X
Xiao Guangrong 已提交
1749 1750 1751 1752 1753 1754 1755 1756 1757

	/* Do not map writable pfn in the readonly memslot. */
	if (writable && memslot_is_readonly(slot)) {
		*writable = false;
		writable = NULL;
	}

	return hva_to_pfn(addr, atomic, async, write_fault,
			  writable);
1758
}
1759
EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1760

D
Dan Williams 已提交
1761
kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1762 1763
		      bool *writable)
{
P
Paolo Bonzini 已提交
1764 1765
	return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
				    write_fault, writable);
1766 1767 1768
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);

D
Dan Williams 已提交
1769
kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1770
{
X
Xiao Guangrong 已提交
1771
	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1772
}
P
Paolo Bonzini 已提交
1773
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1774

D
Dan Williams 已提交
1775
kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1776
{
X
Xiao Guangrong 已提交
1777
	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1778
}
1779
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1780

D
Dan Williams 已提交
1781
kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
P
Paolo Bonzini 已提交
1782 1783 1784 1785 1786
{
	return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);

D
Dan Williams 已提交
1787
kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1788 1789 1790 1791 1792
{
	return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);

D
Dan Williams 已提交
1793
kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
P
Paolo Bonzini 已提交
1794 1795 1796 1797 1798
{
	return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn);

D
Dan Williams 已提交
1799
kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1800 1801 1802 1803 1804
{
	return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);

1805 1806
int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
			    struct page **pages, int nr_pages)
1807 1808
{
	unsigned long addr;
1809
	gfn_t entry = 0;
1810

1811
	addr = gfn_to_hva_many(slot, gfn, &entry);
1812 1813 1814 1815 1816 1817 1818 1819 1820 1821
	if (kvm_is_error_hva(addr))
		return -1;

	if (entry < nr_pages)
		return 0;

	return __get_user_pages_fast(addr, nr_pages, 1, pages);
}
EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);

D
Dan Williams 已提交
1822
static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
X
Xiao Guangrong 已提交
1823
{
1824
	if (is_error_noslot_pfn(pfn))
1825
		return KVM_ERR_PTR_BAD_PAGE;
X
Xiao Guangrong 已提交
1826

1827
	if (kvm_is_reserved_pfn(pfn)) {
1828
		WARN_ON(1);
1829
		return KVM_ERR_PTR_BAD_PAGE;
1830
	}
X
Xiao Guangrong 已提交
1831 1832 1833 1834

	return pfn_to_page(pfn);
}

1835 1836
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
D
Dan Williams 已提交
1837
	kvm_pfn_t pfn;
1838 1839 1840

	pfn = gfn_to_pfn(kvm, gfn);

X
Xiao Guangrong 已提交
1841
	return kvm_pfn_to_page(pfn);
A
Avi Kivity 已提交
1842 1843 1844
}
EXPORT_SYMBOL_GPL(gfn_to_page);

1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
void kvm_release_pfn(kvm_pfn_t pfn, bool dirty, struct gfn_to_pfn_cache *cache)
{
	if (pfn == 0)
		return;

	if (cache)
		cache->pfn = cache->gfn = 0;

	if (dirty)
		kvm_release_pfn_dirty(pfn);
	else
		kvm_release_pfn_clean(pfn);
}

static void kvm_cache_gfn_to_pfn(struct kvm_memory_slot *slot, gfn_t gfn,
				 struct gfn_to_pfn_cache *cache, u64 gen)
{
	kvm_release_pfn(cache->pfn, cache->dirty, cache);

	cache->pfn = gfn_to_pfn_memslot(slot, gfn);
	cache->gfn = gfn;
	cache->dirty = false;
	cache->generation = gen;
}

1870
static int __kvm_map_gfn(struct kvm_memslots *slots, gfn_t gfn,
1871 1872 1873
			 struct kvm_host_map *map,
			 struct gfn_to_pfn_cache *cache,
			 bool atomic)
1874 1875 1876 1877
{
	kvm_pfn_t pfn;
	void *hva = NULL;
	struct page *page = KVM_UNMAPPED_PAGE;
1878
	struct kvm_memory_slot *slot = __gfn_to_memslot(slots, gfn);
1879
	u64 gen = slots->generation;
1880 1881 1882 1883

	if (!map)
		return -EINVAL;

1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896
	if (cache) {
		if (!cache->pfn || cache->gfn != gfn ||
			cache->generation != gen) {
			if (atomic)
				return -EAGAIN;
			kvm_cache_gfn_to_pfn(slot, gfn, cache, gen);
		}
		pfn = cache->pfn;
	} else {
		if (atomic)
			return -EAGAIN;
		pfn = gfn_to_pfn_memslot(slot, gfn);
	}
1897 1898 1899 1900 1901
	if (is_error_noslot_pfn(pfn))
		return -EINVAL;

	if (pfn_valid(pfn)) {
		page = pfn_to_page(pfn);
1902 1903 1904 1905
		if (atomic)
			hva = kmap_atomic(page);
		else
			hva = kmap(page);
P
Paolo Bonzini 已提交
1906
#ifdef CONFIG_HAS_IOMEM
1907
	} else if (!atomic) {
1908
		hva = memremap(pfn_to_hpa(pfn), PAGE_SIZE, MEMREMAP_WB);
1909 1910
	} else {
		return -EINVAL;
P
Paolo Bonzini 已提交
1911
#endif
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924
	}

	if (!hva)
		return -EFAULT;

	map->page = page;
	map->hva = hva;
	map->pfn = pfn;
	map->gfn = gfn;

	return 0;
}

1925 1926
int kvm_map_gfn(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map,
		struct gfn_to_pfn_cache *cache, bool atomic)
1927
{
1928 1929
	return __kvm_map_gfn(kvm_memslots(vcpu->kvm), gfn, map,
			cache, atomic);
1930 1931 1932
}
EXPORT_SYMBOL_GPL(kvm_map_gfn);

1933 1934
int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map)
{
1935 1936
	return __kvm_map_gfn(kvm_vcpu_memslots(vcpu), gfn, map,
		NULL, false);
1937 1938 1939
}
EXPORT_SYMBOL_GPL(kvm_vcpu_map);

1940
static void __kvm_unmap_gfn(struct kvm_memory_slot *memslot,
1941 1942 1943
			struct kvm_host_map *map,
			struct gfn_to_pfn_cache *cache,
			bool dirty, bool atomic)
1944 1945 1946 1947 1948 1949 1950
{
	if (!map)
		return;

	if (!map->hva)
		return;

1951 1952 1953 1954 1955 1956
	if (map->page != KVM_UNMAPPED_PAGE) {
		if (atomic)
			kunmap_atomic(map->hva);
		else
			kunmap(map->page);
	}
1957
#ifdef CONFIG_HAS_IOMEM
1958
	else if (!atomic)
1959
		memunmap(map->hva);
1960 1961
	else
		WARN_ONCE(1, "Unexpected unmapping in atomic context");
1962
#endif
1963

1964
	if (dirty)
1965
		mark_page_dirty_in_slot(memslot, map->gfn);
1966 1967 1968 1969 1970

	if (cache)
		cache->dirty |= dirty;
	else
		kvm_release_pfn(map->pfn, dirty, NULL);
1971 1972 1973 1974

	map->hva = NULL;
	map->page = NULL;
}
1975

1976 1977
int kvm_unmap_gfn(struct kvm_vcpu *vcpu, struct kvm_host_map *map, 
		  struct gfn_to_pfn_cache *cache, bool dirty, bool atomic)
1978
{
1979 1980
	__kvm_unmap_gfn(gfn_to_memslot(vcpu->kvm, map->gfn), map,
			cache, dirty, atomic);
1981 1982 1983 1984 1985 1986
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_unmap_gfn);

void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty)
{
1987 1988
	__kvm_unmap_gfn(kvm_vcpu_gfn_to_memslot(vcpu, map->gfn), map, NULL,
			dirty, false);
1989
}
1990 1991
EXPORT_SYMBOL_GPL(kvm_vcpu_unmap);

1992 1993
struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
{
D
Dan Williams 已提交
1994
	kvm_pfn_t pfn;
1995 1996 1997 1998 1999 2000 2001

	pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);

	return kvm_pfn_to_page(pfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);

2002 2003
void kvm_release_page_clean(struct page *page)
{
2004 2005
	WARN_ON(is_error_page(page));

2006
	kvm_release_pfn_clean(page_to_pfn(page));
2007 2008 2009
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

D
Dan Williams 已提交
2010
void kvm_release_pfn_clean(kvm_pfn_t pfn)
2011
{
2012
	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
2013
		put_page(pfn_to_page(pfn));
2014 2015 2016
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

2017
void kvm_release_page_dirty(struct page *page)
2018
{
X
Xiao Guangrong 已提交
2019 2020
	WARN_ON(is_error_page(page));

2021 2022 2023 2024
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

2025
void kvm_release_pfn_dirty(kvm_pfn_t pfn)
2026 2027 2028 2029
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
2030
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
2031

D
Dan Williams 已提交
2032
void kvm_set_pfn_dirty(kvm_pfn_t pfn)
2033
{
2034 2035
	if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn))
		SetPageDirty(pfn_to_page(pfn));
2036
}
2037 2038
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

D
Dan Williams 已提交
2039
void kvm_set_pfn_accessed(kvm_pfn_t pfn)
2040
{
2041
	if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn))
2042
		mark_page_accessed(pfn_to_page(pfn));
2043 2044 2045
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

D
Dan Williams 已提交
2046
void kvm_get_pfn(kvm_pfn_t pfn)
2047
{
2048
	if (!kvm_is_reserved_pfn(pfn))
2049
		get_page(pfn_to_page(pfn));
2050 2051
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
2052

2053 2054 2055 2056 2057 2058 2059 2060
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

2061 2062
static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
				 void *data, int offset, int len)
2063
{
2064 2065
	int r;
	unsigned long addr;
2066

2067
	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
2068 2069
	if (kvm_is_error_hva(addr))
		return -EFAULT;
2070
	r = __copy_from_user(data, (void __user *)addr + offset, len);
2071
	if (r)
2072 2073 2074
		return -EFAULT;
	return 0;
}
2075 2076 2077 2078 2079 2080 2081 2082

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return __kvm_read_guest_page(slot, gfn, data, offset, len);
}
2083 2084
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

2085 2086 2087 2088 2089 2090 2091 2092 2093
int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
			     int offset, int len)
{
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	return __kvm_read_guest_page(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);

2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113
int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

2114
int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
2115 2116
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
2117
	int seg;
2118
	int offset = offset_in_page(gpa);
2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
2133

2134 2135 2136 2137 2138 2139 2140
static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
			           void *data, int offset, unsigned long len)
{
	int r;
	unsigned long addr;

	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
2141 2142
	if (kvm_is_error_hva(addr))
		return -EFAULT;
2143
	pagefault_disable();
2144
	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
2145
	pagefault_enable();
2146 2147 2148 2149 2150
	if (r)
		return -EFAULT;
	return 0;
}

2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163
int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
			       void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
	int offset = offset_in_page(gpa);

	return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);

static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
			          const void *data, int offset, int len)
2164
{
2165 2166
	int r;
	unsigned long addr;
2167

2168
	addr = gfn_to_hva_memslot(memslot, gfn);
2169 2170
	if (kvm_is_error_hva(addr))
		return -EFAULT;
2171
	r = __copy_to_user((void __user *)addr + offset, data, len);
2172
	if (r)
2173
		return -EFAULT;
2174
	mark_page_dirty_in_slot(memslot, gfn);
2175 2176
	return 0;
}
2177 2178 2179 2180 2181 2182 2183 2184

int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
			 const void *data, int offset, int len)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return __kvm_write_guest_page(slot, gfn, data, offset, len);
}
2185 2186
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

2187 2188 2189 2190 2191 2192 2193 2194 2195
int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
			      const void *data, int offset, int len)
{
	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	return __kvm_write_guest_page(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);

2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214
int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
2215
EXPORT_SYMBOL_GPL(kvm_write_guest);
2216

2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237
int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
		         unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);

2238 2239 2240
static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
				       struct gfn_to_hva_cache *ghc,
				       gpa_t gpa, unsigned long len)
2241 2242
{
	int offset = offset_in_page(gpa);
2243 2244 2245 2246
	gfn_t start_gfn = gpa >> PAGE_SHIFT;
	gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
	gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
	gfn_t nr_pages_avail;
2247

2248
	/* Update ghc->generation before performing any error checks. */
2249
	ghc->generation = slots->generation;
2250 2251 2252 2253 2254

	if (start_gfn > end_gfn) {
		ghc->hva = KVM_HVA_ERR_BAD;
		return -EINVAL;
	}
2255 2256 2257 2258 2259

	/*
	 * If the requested region crosses two memslots, we still
	 * verify that the entire region is valid here.
	 */
2260
	for ( ; start_gfn <= end_gfn; start_gfn += nr_pages_avail) {
2261 2262 2263 2264
		ghc->memslot = __gfn_to_memslot(slots, start_gfn);
		ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
					   &nr_pages_avail);
		if (kvm_is_error_hva(ghc->hva))
2265
			return -EFAULT;
2266 2267 2268
	}

	/* Use the slow path for cross page reads and writes. */
2269
	if (nr_pages_needed == 1)
2270
		ghc->hva += offset;
2271
	else
2272
		ghc->memslot = NULL;
2273

2274 2275 2276
	ghc->gpa = gpa;
	ghc->len = len;
	return 0;
2277
}
2278

2279
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2280 2281
			      gpa_t gpa, unsigned long len)
{
2282
	struct kvm_memslots *slots = kvm_memslots(kvm);
2283 2284
	return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
}
2285
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
2286

2287
int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2288 2289
				  void *data, unsigned int offset,
				  unsigned long len)
2290
{
2291
	struct kvm_memslots *slots = kvm_memslots(kvm);
2292
	int r;
2293
	gpa_t gpa = ghc->gpa + offset;
2294

2295
	BUG_ON(len + offset > ghc->len);
2296

2297 2298 2299 2300
	if (slots->generation != ghc->generation) {
		if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
			return -EFAULT;
	}
2301

2302 2303 2304
	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

2305 2306 2307
	if (unlikely(!ghc->memslot))
		return kvm_write_guest(kvm, gpa, data, len);

2308
	r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
2309 2310
	if (r)
		return -EFAULT;
2311
	mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
2312 2313 2314

	return 0;
}
2315
EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
2316

2317 2318
int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
2319
{
2320
	return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
2321
}
2322
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
2323

2324 2325
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
2326
{
2327
	struct kvm_memslots *slots = kvm_memslots(kvm);
2328 2329
	int r;

2330 2331
	BUG_ON(len > ghc->len);

2332 2333 2334 2335
	if (slots->generation != ghc->generation) {
		if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
			return -EFAULT;
	}
2336

2337 2338 2339
	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

2340 2341 2342
	if (unlikely(!ghc->memslot))
		return kvm_read_guest(kvm, ghc->gpa, data, len);

2343 2344 2345 2346 2347 2348
	r = __copy_from_user(data, (void __user *)ghc->hva, len);
	if (r)
		return -EFAULT;

	return 0;
}
2349
EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
2350

2351 2352
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
2353 2354 2355
	const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));

	return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
2356 2357 2358 2359 2360 2361 2362 2363 2364 2365
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

2366
	while ((seg = next_segment(len, offset)) != 0) {
2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

2378
static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
2379
				    gfn_t gfn)
A
Avi Kivity 已提交
2380
{
R
Rusty Russell 已提交
2381 2382
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
2383

2384
		set_bit_le(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
2385 2386 2387
	}
}

2388 2389 2390 2391 2392
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = gfn_to_memslot(kvm, gfn);
2393
	mark_page_dirty_in_slot(memslot, gfn);
2394
}
2395
EXPORT_SYMBOL_GPL(mark_page_dirty);
2396

2397 2398 2399 2400 2401 2402 2403 2404 2405
void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
	mark_page_dirty_in_slot(memslot, gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);

2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428
void kvm_sigset_activate(struct kvm_vcpu *vcpu)
{
	if (!vcpu->sigset_active)
		return;

	/*
	 * This does a lockless modification of ->real_blocked, which is fine
	 * because, only current can change ->real_blocked and all readers of
	 * ->real_blocked don't care as long ->real_blocked is always a subset
	 * of ->blocked.
	 */
	sigprocmask(SIG_SETMASK, &vcpu->sigset, &current->real_blocked);
}

void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
{
	if (!vcpu->sigset_active)
		return;

	sigprocmask(SIG_SETMASK, &current->real_blocked, NULL);
	sigemptyset(&current->real_blocked);
}

W
Wanpeng Li 已提交
2429 2430
static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
{
2431
	unsigned int old, val, grow, grow_start;
W
Wanpeng Li 已提交
2432

2433
	old = val = vcpu->halt_poll_ns;
2434
	grow_start = READ_ONCE(halt_poll_ns_grow_start);
2435
	grow = READ_ONCE(halt_poll_ns_grow);
2436 2437 2438
	if (!grow)
		goto out;

2439 2440 2441
	val *= grow;
	if (val < grow_start)
		val = grow_start;
W
Wanpeng Li 已提交
2442

2443 2444 2445
	if (val > halt_poll_ns)
		val = halt_poll_ns;

W
Wanpeng Li 已提交
2446
	vcpu->halt_poll_ns = val;
2447
out:
2448
	trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
W
Wanpeng Li 已提交
2449 2450 2451 2452
}

static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
{
2453
	unsigned int old, val, shrink;
W
Wanpeng Li 已提交
2454

2455
	old = val = vcpu->halt_poll_ns;
2456 2457
	shrink = READ_ONCE(halt_poll_ns_shrink);
	if (shrink == 0)
W
Wanpeng Li 已提交
2458 2459
		val = 0;
	else
2460
		val /= shrink;
W
Wanpeng Li 已提交
2461 2462

	vcpu->halt_poll_ns = val;
2463
	trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
W
Wanpeng Li 已提交
2464 2465
}

2466 2467
static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
{
2468 2469 2470
	int ret = -EINTR;
	int idx = srcu_read_lock(&vcpu->kvm->srcu);

2471 2472
	if (kvm_arch_vcpu_runnable(vcpu)) {
		kvm_make_request(KVM_REQ_UNHALT, vcpu);
2473
		goto out;
2474 2475
	}
	if (kvm_cpu_has_pending_timer(vcpu))
2476
		goto out;
2477
	if (signal_pending(current))
2478
		goto out;
2479

2480 2481 2482 2483
	ret = 0;
out:
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	return ret;
2484 2485
}

E
Eddie Dong 已提交
2486 2487 2488
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
2489
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2490
{
2491
	ktime_t start, cur;
2492
	DECLARE_SWAITQUEUE(wait);
2493
	bool waited = false;
W
Wanpeng Li 已提交
2494
	u64 block_ns;
2495

2496 2497
	kvm_arch_vcpu_blocking(vcpu);

2498
	start = cur = ktime_get();
2499
	if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) {
W
Wanpeng Li 已提交
2500
		ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2501

2502
		++vcpu->stat.halt_attempted_poll;
2503 2504 2505 2506 2507 2508 2509
		do {
			/*
			 * This sets KVM_REQ_UNHALT if an interrupt
			 * arrives.
			 */
			if (kvm_vcpu_check_block(vcpu) < 0) {
				++vcpu->stat.halt_successful_poll;
2510 2511
				if (!vcpu_valid_wakeup(vcpu))
					++vcpu->stat.halt_poll_invalid;
2512 2513 2514 2515 2516
				goto out;
			}
			cur = ktime_get();
		} while (single_task_running() && ktime_before(cur, stop));
	}
2517 2518

	for (;;) {
2519
		prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2520

2521
		if (kvm_vcpu_check_block(vcpu) < 0)
2522 2523
			break;

2524
		waited = true;
E
Eddie Dong 已提交
2525 2526
		schedule();
	}
2527

2528
	finish_swait(&vcpu->wq, &wait);
2529 2530
	cur = ktime_get();
out:
2531
	kvm_arch_vcpu_unblocking(vcpu);
W
Wanpeng Li 已提交
2532 2533
	block_ns = ktime_to_ns(cur) - ktime_to_ns(start);

2534 2535
	if (!kvm_arch_no_poll(vcpu)) {
		if (!vcpu_valid_wakeup(vcpu)) {
W
Wanpeng Li 已提交
2536
			shrink_halt_poll_ns(vcpu);
2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550
		} else if (halt_poll_ns) {
			if (block_ns <= vcpu->halt_poll_ns)
				;
			/* we had a long block, shrink polling */
			else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
				shrink_halt_poll_ns(vcpu);
			/* we had a short halt and our poll time is too small */
			else if (vcpu->halt_poll_ns < halt_poll_ns &&
				block_ns < halt_poll_ns)
				grow_halt_poll_ns(vcpu);
		} else {
			vcpu->halt_poll_ns = 0;
		}
	}
W
Wanpeng Li 已提交
2551

2552 2553
	trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
	kvm_arch_vcpu_block_finish(vcpu);
E
Eddie Dong 已提交
2554
}
2555
EXPORT_SYMBOL_GPL(kvm_vcpu_block);
E
Eddie Dong 已提交
2556

2557
bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2558
{
2559
	struct swait_queue_head *wqp;
2560 2561

	wqp = kvm_arch_vcpu_wq(vcpu);
2562
	if (swq_has_sleeper(wqp)) {
2563
		swake_up_one(wqp);
2564
		WRITE_ONCE(vcpu->ready, true);
2565
		++vcpu->stat.halt_wakeup;
2566
		return true;
2567 2568
	}

2569
	return false;
2570 2571 2572
}
EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);

2573
#ifndef CONFIG_S390
2574 2575 2576 2577 2578 2579 2580 2581
/*
 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
 */
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
	int me;
	int cpu = vcpu->cpu;

2582 2583 2584
	if (kvm_vcpu_wake_up(vcpu))
		return;

2585 2586 2587 2588 2589 2590
	me = get_cpu();
	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
		if (kvm_arch_vcpu_should_kick(vcpu))
			smp_send_reschedule(cpu);
	put_cpu();
}
2591
EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2592
#endif /* !CONFIG_S390 */
2593

2594
int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2595 2596 2597
{
	struct pid *pid;
	struct task_struct *task = NULL;
2598
	int ret = 0;
2599 2600 2601 2602

	rcu_read_lock();
	pid = rcu_dereference(target->pid);
	if (pid)
2603
		task = get_pid_task(pid, PIDTYPE_PID);
2604 2605
	rcu_read_unlock();
	if (!task)
2606 2607
		return ret;
	ret = yield_to(task, 1);
2608
	put_task_struct(task);
2609 2610

	return ret;
2611 2612 2613
}
EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);

2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635
/*
 * Helper that checks whether a VCPU is eligible for directed yield.
 * Most eligible candidate to yield is decided by following heuristics:
 *
 *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
 *  (preempted lock holder), indicated by @in_spin_loop.
 *  Set at the beiginning and cleared at the end of interception/PLE handler.
 *
 *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
 *  chance last time (mostly it has become eligible now since we have probably
 *  yielded to lockholder in last iteration. This is done by toggling
 *  @dy_eligible each time a VCPU checked for eligibility.)
 *
 *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
 *  to preempted lock-holder could result in wrong VCPU selection and CPU
 *  burning. Giving priority for a potential lock-holder increases lock
 *  progress.
 *
 *  Since algorithm is based on heuristics, accessing another VCPU data without
 *  locking does not harm. It may result in trying to yield to  same VCPU, fail
 *  and continue with next VCPU and so on.
 */
2636
static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2637
{
2638
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2639 2640 2641
	bool eligible;

	eligible = !vcpu->spin_loop.in_spin_loop ||
2642
		    vcpu->spin_loop.dy_eligible;
2643 2644 2645 2646 2647

	if (vcpu->spin_loop.in_spin_loop)
		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);

	return eligible;
2648 2649
#else
	return true;
2650
#endif
2651
}
2652

2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675
/*
 * Unlike kvm_arch_vcpu_runnable, this function is called outside
 * a vcpu_load/vcpu_put pair.  However, for most architectures
 * kvm_arch_vcpu_runnable does not require vcpu_load.
 */
bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
{
	return kvm_arch_vcpu_runnable(vcpu);
}

static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu)
{
	if (kvm_arch_dy_runnable(vcpu))
		return true;

#ifdef CONFIG_KVM_ASYNC_PF
	if (!list_empty_careful(&vcpu->async_pf.done))
		return true;
#endif

	return false;
}

2676
void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
Z
Zhai, Edwin 已提交
2677
{
2678 2679 2680 2681
	struct kvm *kvm = me->kvm;
	struct kvm_vcpu *vcpu;
	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
	int yielded = 0;
2682
	int try = 3;
2683 2684
	int pass;
	int i;
Z
Zhai, Edwin 已提交
2685

2686
	kvm_vcpu_set_in_spin_loop(me, true);
2687 2688 2689 2690 2691 2692 2693
	/*
	 * We boost the priority of a VCPU that is runnable but not
	 * currently running, because it got preempted by something
	 * else and called schedule in __vcpu_run.  Hopefully that
	 * VCPU is holding the lock that we need and will release it.
	 * We approximate round-robin by starting at the last boosted VCPU.
	 */
2694
	for (pass = 0; pass < 2 && !yielded && try; pass++) {
2695
		kvm_for_each_vcpu(i, vcpu, kvm) {
2696
			if (!pass && i <= last_boosted_vcpu) {
2697 2698 2699 2700
				i = last_boosted_vcpu;
				continue;
			} else if (pass && i > last_boosted_vcpu)
				break;
2701
			if (!READ_ONCE(vcpu->ready))
2702
				continue;
2703 2704
			if (vcpu == me)
				continue;
2705
			if (swait_active(&vcpu->wq) && !vcpu_dy_runnable(vcpu))
2706
				continue;
2707 2708
			if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
				!kvm_arch_vcpu_in_kernel(vcpu))
2709
				continue;
2710 2711
			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
				continue;
2712 2713 2714

			yielded = kvm_vcpu_yield_to(vcpu);
			if (yielded > 0) {
2715 2716
				kvm->last_boosted_vcpu = i;
				break;
2717 2718 2719 2720
			} else if (yielded < 0) {
				try--;
				if (!try)
					break;
2721 2722 2723
			}
		}
	}
2724
	kvm_vcpu_set_in_spin_loop(me, false);
2725 2726 2727

	/* Ensure vcpu is not eligible during next spinloop */
	kvm_vcpu_set_dy_eligible(me, false);
Z
Zhai, Edwin 已提交
2728 2729 2730
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

2731
static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
2732
{
2733
	struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
2734 2735
	struct page *page;

2736
	if (vmf->pgoff == 0)
2737
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
2738
#ifdef CONFIG_X86
2739
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2740
		page = virt_to_page(vcpu->arch.pio_data);
2741
#endif
2742
#ifdef CONFIG_KVM_MMIO
2743 2744
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
2745
#endif
2746
	else
2747
		return kvm_arch_vcpu_fault(vcpu, vmf);
2748
	get_page(page);
2749 2750
	vmf->page = page;
	return 0;
2751 2752
}

2753
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2754
	.fault = kvm_vcpu_fault,
2755 2756 2757 2758 2759 2760 2761 2762
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
2763 2764 2765 2766
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

2767
	debugfs_remove_recursive(vcpu->debugfs_dentry);
A
Al Viro 已提交
2768
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
2769 2770 2771
	return 0;
}

2772
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
2773 2774
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
2775
	.mmap           = kvm_vcpu_mmap,
2776
	.llseek		= noop_llseek,
2777
	KVM_COMPAT(kvm_vcpu_compat_ioctl),
A
Avi Kivity 已提交
2778 2779 2780 2781 2782 2783 2784
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
2785 2786 2787 2788
	char name[8 + 1 + ITOA_MAX_LEN + 1];

	snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
	return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
A
Avi Kivity 已提交
2789 2790
}

2791
static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2792
{
2793
#ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS
2794 2795 2796
	char dir_name[ITOA_MAX_LEN * 2];

	if (!debugfs_initialized())
2797
		return;
2798 2799 2800

	snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
	vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
2801
						  vcpu->kvm->debugfs_dentry);
2802

2803
	kvm_arch_create_vcpu_debugfs(vcpu);
2804
#endif
2805 2806
}

2807 2808 2809
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
2810
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2811 2812
{
	int r;
2813
	struct kvm_vcpu *vcpu;
2814
	struct page *page;
2815

G
Greg Kurz 已提交
2816
	if (id >= KVM_MAX_VCPU_ID)
2817 2818
		return -EINVAL;

2819 2820 2821 2822 2823 2824 2825 2826 2827
	mutex_lock(&kvm->lock);
	if (kvm->created_vcpus == KVM_MAX_VCPUS) {
		mutex_unlock(&kvm->lock);
		return -EINVAL;
	}

	kvm->created_vcpus++;
	mutex_unlock(&kvm->lock);

2828 2829 2830 2831
	r = kvm_arch_vcpu_precreate(kvm, id);
	if (r)
		goto vcpu_decrement;

2832 2833 2834
	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
	if (!vcpu) {
		r = -ENOMEM;
2835 2836
		goto vcpu_decrement;
	}
2837

2838
	BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE);
2839 2840 2841
	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
2842
		goto vcpu_free;
2843 2844 2845 2846
	}
	vcpu->run = page_address(page);

	kvm_vcpu_init(vcpu, kvm, id);
2847 2848 2849

	r = kvm_arch_vcpu_create(vcpu);
	if (r)
2850
		goto vcpu_free_run_page;
2851

2852
	kvm_create_vcpu_debugfs(vcpu);
2853

S
Shaohua Li 已提交
2854
	mutex_lock(&kvm->lock);
2855 2856 2857 2858
	if (kvm_get_vcpu_by_id(kvm, id)) {
		r = -EEXIST;
		goto unlock_vcpu_destroy;
	}
2859

2860 2861
	vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
	BUG_ON(kvm->vcpus[vcpu->vcpu_idx]);
2862

R
Rusty Russell 已提交
2863
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
2864
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
2865
	r = create_vcpu_fd(vcpu);
2866
	if (r < 0) {
2867
		kvm_put_kvm_no_destroy(kvm);
2868
		goto unlock_vcpu_destroy;
2869 2870
	}

2871
	kvm->vcpus[vcpu->vcpu_idx] = vcpu;
2872 2873 2874 2875 2876

	/*
	 * Pairs with smp_rmb() in kvm_get_vcpu.  Write kvm->vcpus
	 * before kvm->online_vcpu's incremented value.
	 */
2877 2878 2879 2880
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

	mutex_unlock(&kvm->lock);
2881
	kvm_arch_vcpu_postcreate(vcpu);
R
Rusty Russell 已提交
2882
	return r;
2883

2884
unlock_vcpu_destroy:
2885
	mutex_unlock(&kvm->lock);
2886
	debugfs_remove_recursive(vcpu->debugfs_dentry);
2887
	kvm_arch_vcpu_destroy(vcpu);
2888 2889
vcpu_free_run_page:
	free_page((unsigned long)vcpu->run);
2890 2891
vcpu_free:
	kmem_cache_free(kvm_vcpu_cache, vcpu);
2892 2893 2894 2895
vcpu_decrement:
	mutex_lock(&kvm->lock);
	kvm->created_vcpus--;
	mutex_unlock(&kvm->lock);
2896 2897 2898
	return r;
}

A
Avi Kivity 已提交
2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
2910 2911
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
2912
{
A
Avi Kivity 已提交
2913
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
2914
	void __user *argp = (void __user *)arg;
2915
	int r;
2916 2917
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
2918

2919 2920
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
2921

2922 2923 2924
	if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
		return -EINVAL;

2925
	/*
2926 2927
	 * Some architectures have vcpu ioctls that are asynchronous to vcpu
	 * execution; mutex_lock() would break them.
2928
	 */
2929 2930
	r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
	if (r != -ENOIOCTLCMD)
2931
		return r;
2932

2933 2934
	if (mutex_lock_killable(&vcpu->mutex))
		return -EINTR;
A
Avi Kivity 已提交
2935
	switch (ioctl) {
2936 2937
	case KVM_RUN: {
		struct pid *oldpid;
2938 2939 2940
		r = -EINVAL;
		if (arg)
			goto out;
2941
		oldpid = rcu_access_pointer(vcpu->pid);
2942
		if (unlikely(oldpid != task_pid(current))) {
2943
			/* The thread running this VCPU changed. */
2944
			struct pid *newpid;
2945

2946 2947 2948 2949 2950
			r = kvm_arch_vcpu_run_pid_change(vcpu);
			if (r)
				break;

			newpid = get_task_pid(current, PIDTYPE_PID);
2951 2952 2953 2954 2955
			rcu_assign_pointer(vcpu->pid, newpid);
			if (oldpid)
				synchronize_rcu();
			put_pid(oldpid);
		}
2956
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2957
		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
A
Avi Kivity 已提交
2958
		break;
2959
	}
A
Avi Kivity 已提交
2960
	case KVM_GET_REGS: {
2961
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2962

2963
		r = -ENOMEM;
2964
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT);
2965
		if (!kvm_regs)
A
Avi Kivity 已提交
2966
			goto out;
2967 2968 2969
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
2970
		r = -EFAULT;
2971 2972
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
2973
		r = 0;
2974 2975
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
2976 2977 2978
		break;
	}
	case KVM_SET_REGS: {
2979
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2980

2981
		r = -ENOMEM;
2982 2983 2984
		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
		if (IS_ERR(kvm_regs)) {
			r = PTR_ERR(kvm_regs);
A
Avi Kivity 已提交
2985
			goto out;
2986
		}
2987 2988
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
		kfree(kvm_regs);
A
Avi Kivity 已提交
2989 2990 2991
		break;
	}
	case KVM_GET_SREGS: {
2992 2993
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs),
				    GFP_KERNEL_ACCOUNT);
2994 2995 2996 2997
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2998 2999 3000
		if (r)
			goto out;
		r = -EFAULT;
3001
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
3002 3003 3004 3005 3006
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
3007 3008 3009
		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
		if (IS_ERR(kvm_sregs)) {
			r = PTR_ERR(kvm_sregs);
G
Guo Chao 已提交
3010
			kvm_sregs = NULL;
A
Avi Kivity 已提交
3011
			goto out;
3012
		}
3013
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
3014 3015
		break;
	}
3016 3017 3018 3019 3020 3021 3022
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
3023
		if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
3024 3025 3026 3027 3028 3029 3030 3031
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
3032
		if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
3033 3034 3035 3036
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		break;
	}
A
Avi Kivity 已提交
3037 3038 3039 3040
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
3041
		if (copy_from_user(&tr, argp, sizeof(tr)))
A
Avi Kivity 已提交
3042
			goto out;
3043
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
3044 3045 3046
		if (r)
			goto out;
		r = -EFAULT;
3047
		if (copy_to_user(argp, &tr, sizeof(tr)))
A
Avi Kivity 已提交
3048 3049 3050 3051
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
3052 3053
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
3054 3055

		r = -EFAULT;
3056
		if (copy_from_user(&dbg, argp, sizeof(dbg)))
A
Avi Kivity 已提交
3057
			goto out;
J
Jan Kiszka 已提交
3058
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
3059 3060
		break;
	}
A
Avi Kivity 已提交
3061 3062 3063 3064 3065 3066 3067 3068 3069
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
3070
					   sizeof(kvm_sigmask)))
A
Avi Kivity 已提交
3071 3072
				goto out;
			r = -EINVAL;
3073
			if (kvm_sigmask.len != sizeof(sigset))
A
Avi Kivity 已提交
3074 3075 3076
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
3077
					   sizeof(sigset)))
A
Avi Kivity 已提交
3078 3079 3080
				goto out;
			p = &sigset;
		}
3081
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
A
Avi Kivity 已提交
3082 3083
		break;
	}
A
Avi Kivity 已提交
3084
	case KVM_GET_FPU: {
3085
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT);
3086 3087 3088 3089
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
3090 3091 3092
		if (r)
			goto out;
		r = -EFAULT;
3093
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
3094 3095 3096 3097 3098
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
3099 3100 3101
		fpu = memdup_user(argp, sizeof(*fpu));
		if (IS_ERR(fpu)) {
			r = PTR_ERR(fpu);
G
Guo Chao 已提交
3102
			fpu = NULL;
A
Avi Kivity 已提交
3103
			goto out;
3104
		}
3105
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
3106 3107
		break;
	}
A
Avi Kivity 已提交
3108
	default:
3109
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
3110 3111
	}
out:
3112
	mutex_unlock(&vcpu->mutex);
3113 3114
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
3115 3116 3117
	return r;
}

3118
#ifdef CONFIG_KVM_COMPAT
3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137
static long kvm_vcpu_compat_ioctl(struct file *filp,
				  unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = compat_ptr(arg);
	int r;

	if (vcpu->kvm->mm != current->mm)
		return -EIO;

	switch (ioctl) {
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset;

		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
3138
					   sizeof(kvm_sigmask)))
3139 3140
				goto out;
			r = -EINVAL;
A
Al Viro 已提交
3141
			if (kvm_sigmask.len != sizeof(compat_sigset_t))
3142 3143
				goto out;
			r = -EFAULT;
A
Al Viro 已提交
3144
			if (get_compat_sigset(&sigset, (void *)sigmask_arg->sigset))
3145
				goto out;
3146 3147 3148
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		} else
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159
		break;
	}
	default:
		r = kvm_vcpu_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct kvm_device *dev = filp->private_data;

	if (dev->ops->mmap)
		return dev->ops->mmap(dev, vma);

	return -ENODEV;
}

S
Scott Wood 已提交
3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190
static int kvm_device_ioctl_attr(struct kvm_device *dev,
				 int (*accessor)(struct kvm_device *dev,
						 struct kvm_device_attr *attr),
				 unsigned long arg)
{
	struct kvm_device_attr attr;

	if (!accessor)
		return -EPERM;

	if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
		return -EFAULT;

	return accessor(dev, &attr);
}

static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
			     unsigned long arg)
{
	struct kvm_device *dev = filp->private_data;

3191 3192 3193
	if (dev->kvm->mm != current->mm)
		return -EIO;

S
Scott Wood 已提交
3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213
	switch (ioctl) {
	case KVM_SET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
	case KVM_GET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
	case KVM_HAS_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
	default:
		if (dev->ops->ioctl)
			return dev->ops->ioctl(dev, ioctl, arg);

		return -ENOTTY;
	}
}

static int kvm_device_release(struct inode *inode, struct file *filp)
{
	struct kvm_device *dev = filp->private_data;
	struct kvm *kvm = dev->kvm;

3214 3215 3216 3217 3218 3219 3220
	if (dev->ops->release) {
		mutex_lock(&kvm->lock);
		list_del(&dev->vm_node);
		dev->ops->release(dev);
		mutex_unlock(&kvm->lock);
	}

S
Scott Wood 已提交
3221 3222 3223 3224 3225 3226 3227
	kvm_put_kvm(kvm);
	return 0;
}

static const struct file_operations kvm_device_fops = {
	.unlocked_ioctl = kvm_device_ioctl,
	.release = kvm_device_release,
3228
	KVM_COMPAT(kvm_device_ioctl),
3229
	.mmap = kvm_device_mmap,
S
Scott Wood 已提交
3230 3231 3232 3233 3234 3235 3236 3237 3238 3239
};

struct kvm_device *kvm_device_from_filp(struct file *filp)
{
	if (filp->f_op != &kvm_device_fops)
		return NULL;

	return filp->private_data;
}

3240
static const struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
3241
#ifdef CONFIG_KVM_MPIC
3242 3243
	[KVM_DEV_TYPE_FSL_MPIC_20]	= &kvm_mpic_ops,
	[KVM_DEV_TYPE_FSL_MPIC_42]	= &kvm_mpic_ops,
3244
#endif
3245 3246
};

3247
int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type)
3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258
{
	if (type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENOSPC;

	if (kvm_device_ops_table[type] != NULL)
		return -EEXIST;

	kvm_device_ops_table[type] = ops;
	return 0;
}

3259 3260 3261 3262 3263 3264
void kvm_unregister_device_ops(u32 type)
{
	if (kvm_device_ops_table[type] != NULL)
		kvm_device_ops_table[type] = NULL;
}

S
Scott Wood 已提交
3265 3266 3267
static int kvm_ioctl_create_device(struct kvm *kvm,
				   struct kvm_create_device *cd)
{
3268
	const struct kvm_device_ops *ops = NULL;
S
Scott Wood 已提交
3269 3270
	struct kvm_device *dev;
	bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
P
Paolo Bonzini 已提交
3271
	int type;
S
Scott Wood 已提交
3272 3273
	int ret;

3274 3275 3276
	if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENODEV;

P
Paolo Bonzini 已提交
3277 3278
	type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table));
	ops = kvm_device_ops_table[type];
3279
	if (ops == NULL)
S
Scott Wood 已提交
3280 3281 3282 3283 3284
		return -ENODEV;

	if (test)
		return 0;

3285
	dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
S
Scott Wood 已提交
3286 3287 3288 3289 3290 3291
	if (!dev)
		return -ENOMEM;

	dev->ops = ops;
	dev->kvm = kvm;

3292
	mutex_lock(&kvm->lock);
P
Paolo Bonzini 已提交
3293
	ret = ops->create(dev, type);
S
Scott Wood 已提交
3294
	if (ret < 0) {
3295
		mutex_unlock(&kvm->lock);
S
Scott Wood 已提交
3296 3297 3298
		kfree(dev);
		return ret;
	}
3299 3300
	list_add(&dev->vm_node, &kvm->devices);
	mutex_unlock(&kvm->lock);
S
Scott Wood 已提交
3301

3302 3303 3304
	if (ops->init)
		ops->init(dev);

3305
	kvm_get_kvm(kvm);
3306
	ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
S
Scott Wood 已提交
3307
	if (ret < 0) {
3308
		kvm_put_kvm_no_destroy(kvm);
3309 3310 3311
		mutex_lock(&kvm->lock);
		list_del(&dev->vm_node);
		mutex_unlock(&kvm->lock);
3312
		ops->destroy(dev);
S
Scott Wood 已提交
3313 3314 3315 3316 3317 3318 3319
		return ret;
	}

	cd->fd = ret;
	return 0;
}

3320 3321 3322 3323 3324 3325 3326 3327 3328 3329
static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
{
	switch (arg) {
	case KVM_CAP_USER_MEMORY:
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
	case KVM_CAP_INTERNAL_ERROR_DATA:
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_CAP_SIGNAL_MSI:
#endif
3330
#ifdef CONFIG_HAVE_KVM_IRQFD
3331
	case KVM_CAP_IRQFD:
3332 3333
	case KVM_CAP_IRQFD_RESAMPLE:
#endif
3334
	case KVM_CAP_IOEVENTFD_ANY_LENGTH:
3335
	case KVM_CAP_CHECK_EXTENSION_VM:
3336
	case KVM_CAP_ENABLE_CAP_VM:
3337
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3338
	case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
3339
#endif
3340
		return 1;
3341
#ifdef CONFIG_KVM_MMIO
3342 3343
	case KVM_CAP_COALESCED_MMIO:
		return KVM_COALESCED_MMIO_PAGE_OFFSET;
P
Peng Hao 已提交
3344 3345
	case KVM_CAP_COALESCED_PIO:
		return 1;
3346
#endif
3347 3348 3349
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_CAP_IRQ_ROUTING:
		return KVM_MAX_IRQ_ROUTES;
3350 3351 3352 3353
#endif
#if KVM_ADDRESS_SPACE_NUM > 1
	case KVM_CAP_MULTI_ADDRESS_SPACE:
		return KVM_ADDRESS_SPACE_NUM;
3354
#endif
3355 3356
	case KVM_CAP_NR_MEMSLOTS:
		return KVM_USER_MEM_SLOTS;
3357 3358 3359 3360 3361 3362
	default:
		break;
	}
	return kvm_vm_ioctl_check_extension(kvm, arg);
}

3363 3364 3365 3366 3367 3368 3369 3370 3371 3372
int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
						  struct kvm_enable_cap *cap)
{
	return -EINVAL;
}

static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
					   struct kvm_enable_cap *cap)
{
	switch (cap->cap) {
3373
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3374
	case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
3375 3376 3377 3378 3379
		if (cap->flags || (cap->args[0] & ~1))
			return -EINVAL;
		kvm->manual_dirty_log_protect = cap->args[0];
		return 0;
#endif
3380 3381 3382 3383 3384
	default:
		return kvm_vm_ioctl_enable_cap(kvm, cap);
	}
}

A
Avi Kivity 已提交
3385 3386 3387 3388 3389
static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
3390
	int r;
A
Avi Kivity 已提交
3391

3392 3393
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
3394 3395 3396 3397
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		break;
3398 3399 3400 3401 3402 3403 3404 3405 3406
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		r = -EFAULT;
		if (copy_from_user(&cap, argp, sizeof(cap)))
			goto out;
		r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
		break;
	}
3407 3408 3409 3410 3411
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
3412
						sizeof(kvm_userspace_mem)))
3413 3414
			goto out;

3415
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
A
Avi Kivity 已提交
3416 3417 3418 3419 3420 3421
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
3422
		if (copy_from_user(&log, argp, sizeof(log)))
A
Avi Kivity 已提交
3423
			goto out;
3424
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
3425 3426
		break;
	}
3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
	case KVM_CLEAR_DIRTY_LOG: {
		struct kvm_clear_dirty_log log;

		r = -EFAULT;
		if (copy_from_user(&log, argp, sizeof(log)))
			goto out;
		r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
		break;
	}
#endif
3438
#ifdef CONFIG_KVM_MMIO
3439 3440
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
3441

3442
		r = -EFAULT;
3443
		if (copy_from_user(&zone, argp, sizeof(zone)))
3444 3445 3446 3447 3448 3449
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
3450

3451
		r = -EFAULT;
3452
		if (copy_from_user(&zone, argp, sizeof(zone)))
3453 3454 3455 3456 3457
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		break;
	}
#endif
G
Gregory Haskins 已提交
3458 3459 3460 3461
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
3462
		if (copy_from_user(&data, argp, sizeof(data)))
G
Gregory Haskins 已提交
3463
			goto out;
3464
		r = kvm_irqfd(kvm, &data);
G
Gregory Haskins 已提交
3465 3466
		break;
	}
G
Gregory Haskins 已提交
3467 3468 3469 3470
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
3471
		if (copy_from_user(&data, argp, sizeof(data)))
G
Gregory Haskins 已提交
3472 3473 3474 3475
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
3476 3477 3478 3479 3480
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_SIGNAL_MSI: {
		struct kvm_msi msi;

		r = -EFAULT;
3481
		if (copy_from_user(&msi, argp, sizeof(msi)))
3482 3483 3484 3485
			goto out;
		r = kvm_send_userspace_msi(kvm, &msi);
		break;
	}
3486 3487 3488 3489 3490 3491 3492
#endif
#ifdef __KVM_HAVE_IRQ_LINE
	case KVM_IRQ_LINE_STATUS:
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
3493
		if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
3494 3495
			goto out;

3496 3497
		r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
					ioctl == KVM_IRQ_LINE_STATUS);
3498 3499 3500 3501 3502
		if (r)
			goto out;

		r = -EFAULT;
		if (ioctl == KVM_IRQ_LINE_STATUS) {
3503
			if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
3504 3505 3506 3507 3508 3509
				goto out;
		}

		r = 0;
		break;
	}
3510
#endif
3511 3512 3513 3514
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_SET_GSI_ROUTING: {
		struct kvm_irq_routing routing;
		struct kvm_irq_routing __user *urouting;
3515
		struct kvm_irq_routing_entry *entries = NULL;
3516 3517 3518 3519 3520

		r = -EFAULT;
		if (copy_from_user(&routing, argp, sizeof(routing)))
			goto out;
		r = -EINVAL;
3521 3522
		if (!kvm_arch_can_set_irq_routing(kvm))
			goto out;
3523
		if (routing.nr > KVM_MAX_IRQ_ROUTES)
3524 3525 3526
			goto out;
		if (routing.flags)
			goto out;
3527 3528
		if (routing.nr) {
			r = -ENOMEM;
3529 3530
			entries = vmalloc(array_size(sizeof(*entries),
						     routing.nr));
3531 3532 3533 3534 3535 3536 3537 3538
			if (!entries)
				goto out;
			r = -EFAULT;
			urouting = argp;
			if (copy_from_user(entries, urouting->entries,
					   routing.nr * sizeof(*entries)))
				goto out_free_irq_routing;
		}
3539 3540
		r = kvm_set_irq_routing(kvm, entries, routing.nr,
					routing.flags);
3541
out_free_irq_routing:
3542 3543 3544 3545
		vfree(entries);
		break;
	}
#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
S
Scott Wood 已提交
3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563
	case KVM_CREATE_DEVICE: {
		struct kvm_create_device cd;

		r = -EFAULT;
		if (copy_from_user(&cd, argp, sizeof(cd)))
			goto out;

		r = kvm_ioctl_create_device(kvm, &cd);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(argp, &cd, sizeof(cd)))
			goto out;

		r = 0;
		break;
	}
3564 3565 3566
	case KVM_CHECK_EXTENSION:
		r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
		break;
3567
	default:
3568
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
3569 3570 3571 3572 3573
	}
out:
	return r;
}

3574
#ifdef CONFIG_KVM_COMPAT
3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
3599
			return -EFAULT;
3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}
	return r;
}
#endif

3615
static struct file_operations kvm_vm_fops = {
3616 3617
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
3618
	.llseek		= noop_llseek,
3619
	KVM_COMPAT(kvm_vm_compat_ioctl),
3620 3621
};

3622
static int kvm_dev_ioctl_create_vm(unsigned long type)
3623
{
3624
	int r;
3625
	struct kvm *kvm;
3626
	struct file *file;
3627

3628
	kvm = kvm_create_vm(type);
3629 3630
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
3631
#ifdef CONFIG_KVM_MMIO
3632
	r = kvm_coalesced_mmio_init(kvm);
3633 3634
	if (r < 0)
		goto put_kvm;
3635
#endif
3636
	r = get_unused_fd_flags(O_CLOEXEC);
3637 3638 3639
	if (r < 0)
		goto put_kvm;

3640 3641 3642
	file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
	if (IS_ERR(file)) {
		put_unused_fd(r);
3643 3644
		r = PTR_ERR(file);
		goto put_kvm;
3645
	}
3646

3647 3648 3649 3650 3651 3652
	/*
	 * Don't call kvm_put_kvm anymore at this point; file->f_op is
	 * already set, with ->release() being kvm_vm_release().  In error
	 * cases it will be called by the final fput(file) and will take
	 * care of doing kvm_put_kvm(kvm).
	 */
3653
	if (kvm_create_vm_debugfs(kvm, r) < 0) {
3654 3655
		put_unused_fd(r);
		fput(file);
3656 3657
		return -ENOMEM;
	}
3658
	kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
3659

3660
	fd_install(r, file);
3661
	return r;
3662 3663 3664 3665

put_kvm:
	kvm_put_kvm(kvm);
	return r;
3666 3667 3668 3669 3670
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
3671
	long r = -EINVAL;
3672 3673 3674

	switch (ioctl) {
	case KVM_GET_API_VERSION:
3675 3676
		if (arg)
			goto out;
3677 3678 3679
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
3680
		r = kvm_dev_ioctl_create_vm(arg);
3681
		break;
3682
	case KVM_CHECK_EXTENSION:
3683
		r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3684
		break;
3685 3686 3687
	case KVM_GET_VCPU_MMAP_SIZE:
		if (arg)
			goto out;
3688 3689 3690
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
3691
#endif
3692
#ifdef CONFIG_KVM_MMIO
3693
		r += PAGE_SIZE;    /* coalesced mmio ring page */
3694
#endif
3695
		break;
3696 3697 3698
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
3699
		r = -EOPNOTSUPP;
3700
		break;
A
Avi Kivity 已提交
3701
	default:
3702
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
3703 3704 3705 3706 3707 3708 3709
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
3710
	.llseek		= noop_llseek,
3711
	KVM_COMPAT(kvm_dev_ioctl),
A
Avi Kivity 已提交
3712 3713 3714
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
3715
	KVM_MINOR,
A
Avi Kivity 已提交
3716 3717 3718 3719
	"kvm",
	&kvm_chardev_ops,
};

3720
static void hardware_enable_nolock(void *junk)
3721 3722
{
	int cpu = raw_smp_processor_id();
3723
	int r;
3724

3725
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3726
		return;
3727

3728
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
3729

3730
	r = kvm_arch_hardware_enable();
3731 3732 3733 3734

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
X
Xiubo Li 已提交
3735
		pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3736
	}
3737 3738
}

3739
static int kvm_starting_cpu(unsigned int cpu)
3740
{
3741
	raw_spin_lock(&kvm_count_lock);
3742 3743
	if (kvm_usage_count)
		hardware_enable_nolock(NULL);
3744
	raw_spin_unlock(&kvm_count_lock);
3745
	return 0;
3746 3747 3748
}

static void hardware_disable_nolock(void *junk)
3749 3750 3751
{
	int cpu = raw_smp_processor_id();

3752
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3753
		return;
3754
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3755
	kvm_arch_hardware_disable();
3756 3757
}

3758
static int kvm_dying_cpu(unsigned int cpu)
3759
{
3760
	raw_spin_lock(&kvm_count_lock);
3761 3762
	if (kvm_usage_count)
		hardware_disable_nolock(NULL);
3763
	raw_spin_unlock(&kvm_count_lock);
3764
	return 0;
3765 3766
}

3767 3768 3769 3770 3771 3772
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
3773
		on_each_cpu(hardware_disable_nolock, NULL, 1);
3774 3775 3776 3777
}

static void hardware_disable_all(void)
{
3778
	raw_spin_lock(&kvm_count_lock);
3779
	hardware_disable_all_nolock();
3780
	raw_spin_unlock(&kvm_count_lock);
3781 3782 3783 3784 3785 3786
}

static int hardware_enable_all(void)
{
	int r = 0;

3787
	raw_spin_lock(&kvm_count_lock);
3788 3789 3790 3791

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
3792
		on_each_cpu(hardware_enable_nolock, NULL, 1);
3793 3794 3795 3796 3797 3798 3799

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

3800
	raw_spin_unlock(&kvm_count_lock);
3801 3802 3803 3804

	return r;
}

3805
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
3806
		      void *v)
3807
{
3808 3809 3810 3811 3812 3813
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
X
Xiubo Li 已提交
3814
	pr_info("kvm: exiting hardware virtualization\n");
3815
	kvm_rebooting = true;
3816
	on_each_cpu(hardware_disable_nolock, NULL, 1);
3817 3818 3819 3820 3821 3822 3823 3824
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
3825
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3826 3827 3828 3829
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
3830
		struct kvm_io_device *pos = bus->range[i].dev;
3831 3832 3833

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
3834
	kfree(bus);
3835 3836
}

3837
static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
X
Xiubo Li 已提交
3838
				 const struct kvm_io_range *r2)
3839
{
J
Jason Wang 已提交
3840 3841 3842 3843
	gpa_t addr1 = r1->addr;
	gpa_t addr2 = r2->addr;

	if (addr1 < addr2)
3844
		return -1;
J
Jason Wang 已提交
3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856

	/* If r2->len == 0, match the exact address.  If r2->len != 0,
	 * accept any overlapping write.  Any order is acceptable for
	 * overlapping ranges, because kvm_io_bus_get_first_dev ensures
	 * we process all of them.
	 */
	if (r2->len) {
		addr1 += r1->len;
		addr2 += r2->len;
	}

	if (addr1 > addr2)
3857
		return 1;
J
Jason Wang 已提交
3858

3859 3860 3861
	return 0;
}

3862 3863
static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
{
3864
	return kvm_io_bus_cmp(p1, p2);
3865 3866
}

G
Geoff Levand 已提交
3867
static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884
			     gpa_t addr, int len)
{
	struct kvm_io_range *range, key;
	int off;

	key = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	range = bsearch(&key, bus->range, bus->dev_count,
			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
	if (range == NULL)
		return -ENOENT;

	off = range - bus->range;

3885
	while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3886 3887 3888 3889 3890
		off--;

	return off;
}

3891
static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
C
Cornelia Huck 已提交
3892 3893 3894 3895 3896 3897 3898 3899 3900
			      struct kvm_io_range *range, const void *val)
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
3901
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3902
		if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
C
Cornelia Huck 已提交
3903 3904 3905 3906 3907 3908 3909 3910
					range->len, val))
			return idx;
		idx++;
	}

	return -EOPNOTSUPP;
}

3911
/* kvm_io_bus_write - called under kvm->slots_lock */
3912
int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3913
		     int len, const void *val)
3914
{
3915
	struct kvm_io_bus *bus;
3916
	struct kvm_io_range range;
C
Cornelia Huck 已提交
3917
	int r;
3918 3919 3920 3921 3922

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
3923

3924
	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3925 3926
	if (!bus)
		return -ENOMEM;
3927
	r = __kvm_io_bus_write(vcpu, bus, &range, val);
C
Cornelia Huck 已提交
3928 3929
	return r < 0 ? r : 0;
}
L
Leo Yan 已提交
3930
EXPORT_SYMBOL_GPL(kvm_io_bus_write);
C
Cornelia Huck 已提交
3931 3932

/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3933 3934
int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
			    gpa_t addr, int len, const void *val, long cookie)
C
Cornelia Huck 已提交
3935 3936 3937 3938 3939 3940 3941 3942 3943
{
	struct kvm_io_bus *bus;
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

3944
	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3945 3946
	if (!bus)
		return -ENOMEM;
C
Cornelia Huck 已提交
3947 3948 3949

	/* First try the device referenced by cookie. */
	if ((cookie >= 0) && (cookie < bus->dev_count) &&
3950
	    (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3951
		if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
C
Cornelia Huck 已提交
3952 3953 3954 3955 3956 3957 3958
					val))
			return cookie;

	/*
	 * cookie contained garbage; fall back to search and return the
	 * correct cookie value.
	 */
3959
	return __kvm_io_bus_write(vcpu, bus, &range, val);
C
Cornelia Huck 已提交
3960 3961
}

3962 3963
static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
			     struct kvm_io_range *range, void *val)
C
Cornelia Huck 已提交
3964 3965 3966 3967
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3968 3969 3970 3971
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
3972
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3973
		if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
C
Cornelia Huck 已提交
3974 3975
				       range->len, val))
			return idx;
3976 3977 3978
		idx++;
	}

3979 3980
	return -EOPNOTSUPP;
}
3981

3982
/* kvm_io_bus_read - called under kvm->slots_lock */
3983
int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
M
Marcelo Tosatti 已提交
3984
		    int len, void *val)
3985
{
3986
	struct kvm_io_bus *bus;
3987
	struct kvm_io_range range;
C
Cornelia Huck 已提交
3988
	int r;
3989 3990 3991 3992 3993

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
M
Marcelo Tosatti 已提交
3994

3995
	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3996 3997
	if (!bus)
		return -ENOMEM;
3998
	r = __kvm_io_bus_read(vcpu, bus, &range, val);
C
Cornelia Huck 已提交
3999 4000
	return r < 0 ? r : 0;
}
4001

4002
/* Caller must hold slots_lock. */
4003 4004
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, struct kvm_io_device *dev)
4005
{
4006
	int i;
M
Marcelo Tosatti 已提交
4007
	struct kvm_io_bus *new_bus, *bus;
4008
	struct kvm_io_range range;
4009

4010
	bus = kvm_get_bus(kvm, bus_idx);
4011 4012 4013
	if (!bus)
		return -ENOMEM;

4014 4015
	/* exclude ioeventfd which is limited by maximum fd */
	if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
4016
		return -ENOSPC;
4017

4018
	new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
4019
			  GFP_KERNEL_ACCOUNT);
M
Marcelo Tosatti 已提交
4020 4021
	if (!new_bus)
		return -ENOMEM;
4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
		.dev = dev,
	};

	for (i = 0; i < bus->dev_count; i++)
		if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
			break;

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count++;
	new_bus->range[i] = range;
	memcpy(new_bus->range + i + 1, bus->range + i,
		(bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
4038 4039 4040
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
4041 4042 4043 4044

	return 0;
}

4045
/* Caller must hold slots_lock. */
4046 4047
void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			       struct kvm_io_device *dev)
4048
{
4049
	int i;
M
Marcelo Tosatti 已提交
4050
	struct kvm_io_bus *new_bus, *bus;
4051

4052
	bus = kvm_get_bus(kvm, bus_idx);
4053
	if (!bus)
4054
		return;
4055

4056 4057
	for (i = 0; i < bus->dev_count; i++)
		if (bus->range[i].dev == dev) {
4058 4059
			break;
		}
M
Marcelo Tosatti 已提交
4060

4061 4062
	if (i == bus->dev_count)
		return;
4063

4064
	new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
4065
			  GFP_KERNEL_ACCOUNT);
4066 4067 4068 4069
	if (!new_bus)  {
		pr_err("kvm: failed to shrink bus, removing it completely\n");
		goto broken;
	}
4070 4071 4072 4073 4074

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count--;
	memcpy(new_bus->range + i, bus->range + i + 1,
	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
4075

4076
broken:
M
Marcelo Tosatti 已提交
4077 4078 4079
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
4080
	return;
4081 4082
}

4083 4084 4085 4086 4087 4088 4089 4090 4091 4092
struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
					 gpa_t addr)
{
	struct kvm_io_bus *bus;
	int dev_idx, srcu_idx;
	struct kvm_io_device *iodev = NULL;

	srcu_idx = srcu_read_lock(&kvm->srcu);

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
4093 4094
	if (!bus)
		goto out_unlock;
4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108

	dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
	if (dev_idx < 0)
		goto out_unlock;

	iodev = bus->range[dev_idx].dev;

out_unlock:
	srcu_read_unlock(&kvm->srcu, srcu_idx);

	return iodev;
}
EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);

4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120
static int kvm_debugfs_open(struct inode *inode, struct file *file,
			   int (*get)(void *, u64 *), int (*set)(void *, u64),
			   const char *fmt)
{
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
					  inode->i_private;

	/* The debugfs files are a reference to the kvm struct which
	 * is still valid when kvm_destroy_vm is called.
	 * To avoid the race between open and the removal of the debugfs
	 * directory we test against the users count.
	 */
4121
	if (!refcount_inc_not_zero(&stat_data->kvm->users_count))
4122 4123
		return -ENOENT;

4124
	if (simple_attr_open(inode, file, get,
4125 4126 4127
		    KVM_DBGFS_GET_MODE(stat_data->dbgfs_item) & 0222
		    ? set : NULL,
		    fmt)) {
4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145
		kvm_put_kvm(stat_data->kvm);
		return -ENOMEM;
	}

	return 0;
}

static int kvm_debugfs_release(struct inode *inode, struct file *file)
{
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
					  inode->i_private;

	simple_attr_release(inode, file);
	kvm_put_kvm(stat_data->kvm);

	return 0;
}

4146
static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val)
4147
{
4148
	*val = *(ulong *)((void *)kvm + offset);
4149

4150 4151 4152 4153 4154 4155
	return 0;
}

static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset)
{
	*(ulong *)((void *)kvm + offset) = 0;
4156 4157 4158 4159

	return 0;
}

4160
static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val)
4161
{
4162 4163
	int i;
	struct kvm_vcpu *vcpu;
4164

4165
	*val = 0;
4166

4167 4168
	kvm_for_each_vcpu(i, vcpu, kvm)
		*val += *(u64 *)((void *)vcpu + offset);
4169 4170 4171 4172

	return 0;
}

4173
static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset)
4174
{
4175 4176
	int i;
	struct kvm_vcpu *vcpu;
4177

4178 4179 4180 4181 4182
	kvm_for_each_vcpu(i, vcpu, kvm)
		*(u64 *)((void *)vcpu + offset) = 0;

	return 0;
}
4183

4184
static int kvm_stat_data_get(void *data, u64 *val)
4185
{
4186
	int r = -EFAULT;
4187 4188
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;

4189 4190 4191 4192 4193 4194 4195 4196 4197 4198
	switch (stat_data->dbgfs_item->kind) {
	case KVM_STAT_VM:
		r = kvm_get_stat_per_vm(stat_data->kvm,
					stat_data->dbgfs_item->offset, val);
		break;
	case KVM_STAT_VCPU:
		r = kvm_get_stat_per_vcpu(stat_data->kvm,
					  stat_data->dbgfs_item->offset, val);
		break;
	}
4199

4200
	return r;
4201 4202
}

4203
static int kvm_stat_data_clear(void *data, u64 val)
4204
{
4205
	int r = -EFAULT;
4206 4207 4208 4209 4210
	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;

	if (val)
		return -EINVAL;

4211 4212 4213 4214 4215 4216 4217 4218 4219 4220
	switch (stat_data->dbgfs_item->kind) {
	case KVM_STAT_VM:
		r = kvm_clear_stat_per_vm(stat_data->kvm,
					  stat_data->dbgfs_item->offset);
		break;
	case KVM_STAT_VCPU:
		r = kvm_clear_stat_per_vcpu(stat_data->kvm,
					    stat_data->dbgfs_item->offset);
		break;
	}
4221

4222
	return r;
4223 4224
}

4225
static int kvm_stat_data_open(struct inode *inode, struct file *file)
4226 4227
{
	__simple_attr_check_format("%llu\n", 0ull);
4228 4229
	return kvm_debugfs_open(inode, file, kvm_stat_data_get,
				kvm_stat_data_clear, "%llu\n");
4230 4231
}

4232 4233 4234
static const struct file_operations stat_fops_per_vm = {
	.owner = THIS_MODULE,
	.open = kvm_stat_data_open,
4235
	.release = kvm_debugfs_release,
4236 4237 4238
	.read = simple_attr_read,
	.write = simple_attr_write,
	.llseek = no_llseek,
4239 4240
};

4241
static int vm_stat_get(void *_offset, u64 *val)
4242 4243 4244
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
4245
	u64 tmp_val;
4246

4247
	*val = 0;
J
Junaid Shahid 已提交
4248
	mutex_lock(&kvm_lock);
4249
	list_for_each_entry(kvm, &vm_list, vm_list) {
4250
		kvm_get_stat_per_vm(kvm, offset, &tmp_val);
4251 4252
		*val += tmp_val;
	}
J
Junaid Shahid 已提交
4253
	mutex_unlock(&kvm_lock);
4254
	return 0;
4255 4256
}

4257 4258 4259 4260 4261 4262 4263 4264
static int vm_stat_clear(void *_offset, u64 val)
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

	if (val)
		return -EINVAL;

J
Junaid Shahid 已提交
4265
	mutex_lock(&kvm_lock);
4266
	list_for_each_entry(kvm, &vm_list, vm_list) {
4267
		kvm_clear_stat_per_vm(kvm, offset);
4268
	}
J
Junaid Shahid 已提交
4269
	mutex_unlock(&kvm_lock);
4270 4271 4272 4273 4274

	return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
4275

4276
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
4277 4278 4279
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
4280
	u64 tmp_val;
A
Avi Kivity 已提交
4281

4282
	*val = 0;
J
Junaid Shahid 已提交
4283
	mutex_lock(&kvm_lock);
4284
	list_for_each_entry(kvm, &vm_list, vm_list) {
4285
		kvm_get_stat_per_vcpu(kvm, offset, &tmp_val);
4286 4287
		*val += tmp_val;
	}
J
Junaid Shahid 已提交
4288
	mutex_unlock(&kvm_lock);
4289
	return 0;
A
Avi Kivity 已提交
4290 4291
}

4292 4293 4294 4295 4296 4297 4298 4299
static int vcpu_stat_clear(void *_offset, u64 val)
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

	if (val)
		return -EINVAL;

J
Junaid Shahid 已提交
4300
	mutex_lock(&kvm_lock);
4301
	list_for_each_entry(kvm, &vm_list, vm_list) {
4302
		kvm_clear_stat_per_vcpu(kvm, offset);
4303
	}
J
Junaid Shahid 已提交
4304
	mutex_unlock(&kvm_lock);
4305 4306 4307 4308 4309 4310

	return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
			"%llu\n");
4311

4312
static const struct file_operations *stat_fops[] = {
4313 4314 4315
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
4316

4317 4318 4319 4320 4321 4322 4323 4324
static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
{
	struct kobj_uevent_env *env;
	unsigned long long created, active;

	if (!kvm_dev.this_device || !kvm)
		return;

J
Junaid Shahid 已提交
4325
	mutex_lock(&kvm_lock);
4326 4327 4328 4329 4330 4331 4332 4333
	if (type == KVM_EVENT_CREATE_VM) {
		kvm_createvm_count++;
		kvm_active_vms++;
	} else if (type == KVM_EVENT_DESTROY_VM) {
		kvm_active_vms--;
	}
	created = kvm_createvm_count;
	active = kvm_active_vms;
J
Junaid Shahid 已提交
4334
	mutex_unlock(&kvm_lock);
4335

4336
	env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT);
4337 4338 4339 4340 4341 4342
	if (!env)
		return;

	add_uevent_var(env, "CREATED=%llu", created);
	add_uevent_var(env, "COUNT=%llu", active);

4343
	if (type == KVM_EVENT_CREATE_VM) {
4344
		add_uevent_var(env, "EVENT=create");
4345 4346
		kvm->userspace_pid = task_pid_nr(current);
	} else if (type == KVM_EVENT_DESTROY_VM) {
4347
		add_uevent_var(env, "EVENT=destroy");
4348 4349
	}
	add_uevent_var(env, "PID=%d", kvm->userspace_pid);
4350

4351
	if (!IS_ERR_OR_NULL(kvm->debugfs_dentry)) {
4352
		char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT);
4353 4354 4355 4356 4357 4358

		if (p) {
			tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
			if (!IS_ERR(tmp))
				add_uevent_var(env, "STATS_PATH=%s", tmp);
			kfree(p);
4359 4360 4361 4362 4363 4364 4365 4366
		}
	}
	/* no need for checks, since we are adding at most only 5 keys */
	env->envp[env->envp_idx++] = NULL;
	kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
	kfree(env);
}

4367
static void kvm_init_debug(void)
A
Avi Kivity 已提交
4368 4369 4370
{
	struct kvm_stats_debugfs_item *p;

4371
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
4372

4373 4374
	kvm_debugfs_num_entries = 0;
	for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
4375 4376
		debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p),
				    kvm_debugfs_dir, (void *)(long)p->offset,
4377
				    stat_fops[p->kind]);
4378
	}
A
Avi Kivity 已提交
4379 4380
}

4381
static int kvm_suspend(void)
4382
{
4383
	if (kvm_usage_count)
4384
		hardware_disable_nolock(NULL);
4385 4386 4387
	return 0;
}

4388
static void kvm_resume(void)
4389
{
4390
	if (kvm_usage_count) {
4391 4392 4393
#ifdef CONFIG_LOCKDEP
		WARN_ON(lockdep_is_held(&kvm_count_lock));
#endif
4394
		hardware_enable_nolock(NULL);
4395
	}
4396 4397
}

4398
static struct syscore_ops kvm_syscore_ops = {
4399 4400 4401 4402
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

4403 4404 4405 4406 4407 4408 4409 4410 4411
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
4412

4413
	WRITE_ONCE(vcpu->preempted, false);
4414
	WRITE_ONCE(vcpu->ready, false);
4415

4416
	__this_cpu_write(kvm_running_vcpu, vcpu);
R
Radim Krčmář 已提交
4417
	kvm_arch_sched_in(vcpu, cpu);
4418
	kvm_arch_vcpu_load(vcpu, cpu);
4419 4420 4421 4422 4423 4424 4425
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

4426
	if (current->state == TASK_RUNNING) {
4427
		WRITE_ONCE(vcpu->preempted, true);
4428 4429
		WRITE_ONCE(vcpu->ready, true);
	}
4430
	kvm_arch_vcpu_put(vcpu);
4431 4432 4433 4434 4435
	__this_cpu_write(kvm_running_vcpu, NULL);
}

/**
 * kvm_get_running_vcpu - get the vcpu running on the current CPU.
4436 4437 4438 4439 4440 4441
 *
 * We can disable preemption locally around accessing the per-CPU variable,
 * and use the resolved vcpu pointer after enabling preemption again,
 * because even if the current thread is migrated to another CPU, reading
 * the per-CPU value later will give us the same value as we update the
 * per-CPU variable in the preempt notifier handlers.
4442 4443 4444
 */
struct kvm_vcpu *kvm_get_running_vcpu(void)
{
4445 4446 4447 4448 4449 4450 4451
	struct kvm_vcpu *vcpu;

	preempt_disable();
	vcpu = __this_cpu_read(kvm_running_vcpu);
	preempt_enable();

	return vcpu;
4452 4453 4454 4455 4456 4457 4458 4459
}

/**
 * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus.
 */
struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
{
        return &kvm_running_vcpu;
4460 4461
}

4462 4463 4464 4465 4466
static void check_processor_compat(void *rtn)
{
	*(int *)rtn = kvm_arch_check_processor_compat();
}

4467
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
4468
		  struct module *module)
A
Avi Kivity 已提交
4469 4470
{
	int r;
Y
Yang, Sheng 已提交
4471
	int cpu;
A
Avi Kivity 已提交
4472

4473 4474
	r = kvm_arch_init(opaque);
	if (r)
4475
		goto out_fail;
4476

4477 4478 4479 4480
	/*
	 * kvm_arch_init makes sure there's at most one caller
	 * for architectures that support multiple implementations,
	 * like intel and amd on x86.
P
Paolo Bonzini 已提交
4481 4482
	 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
	 * conflicts in case kvm is already setup for another implementation.
4483
	 */
P
Paolo Bonzini 已提交
4484 4485 4486
	r = kvm_irqfd_init();
	if (r)
		goto out_irqfd;
4487

4488
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
4489 4490 4491 4492
		r = -ENOMEM;
		goto out_free_0;
	}

4493
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
4494
	if (r < 0)
4495
		goto out_free_1;
A
Avi Kivity 已提交
4496

Y
Yang, Sheng 已提交
4497
	for_each_online_cpu(cpu) {
4498
		smp_call_function_single(cpu, check_processor_compat, &r, 1);
Y
Yang, Sheng 已提交
4499
		if (r < 0)
4500
			goto out_free_2;
Y
Yang, Sheng 已提交
4501 4502
	}

T
Thomas Gleixner 已提交
4503
	r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
4504
				      kvm_starting_cpu, kvm_dying_cpu);
A
Avi Kivity 已提交
4505
	if (r)
4506
		goto out_free_2;
A
Avi Kivity 已提交
4507 4508
	register_reboot_notifier(&kvm_reboot_notifier);

4509
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
4510 4511
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
4512 4513 4514 4515 4516 4517
	kvm_vcpu_cache =
		kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
					   SLAB_ACCOUNT,
					   offsetof(struct kvm_vcpu, arch),
					   sizeof_field(struct kvm_vcpu, arch),
					   NULL);
4518 4519
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
4520
		goto out_free_3;
4521 4522
	}

4523 4524 4525 4526
	r = kvm_async_pf_init();
	if (r)
		goto out_free;

A
Avi Kivity 已提交
4527
	kvm_chardev_ops.owner = module;
4528 4529
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
4530 4531 4532

	r = misc_register(&kvm_dev);
	if (r) {
X
Xiubo Li 已提交
4533
		pr_err("kvm: misc device register failed\n");
4534
		goto out_unreg;
A
Avi Kivity 已提交
4535 4536
	}

4537 4538
	register_syscore_ops(&kvm_syscore_ops);

4539 4540 4541
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

4542
	kvm_init_debug();
4543

P
Paolo Bonzini 已提交
4544 4545 4546
	r = kvm_vfio_ops_init();
	WARN_ON(r);

4547
	return 0;
A
Avi Kivity 已提交
4548

4549 4550
out_unreg:
	kvm_async_pf_deinit();
A
Avi Kivity 已提交
4551
out_free:
4552
	kmem_cache_destroy(kvm_vcpu_cache);
4553
out_free_3:
A
Avi Kivity 已提交
4554
	unregister_reboot_notifier(&kvm_reboot_notifier);
4555
	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4556
out_free_2:
4557
	kvm_arch_hardware_unsetup();
4558
out_free_1:
4559
	free_cpumask_var(cpus_hardware_enabled);
4560
out_free_0:
4561
	kvm_irqfd_exit();
P
Paolo Bonzini 已提交
4562
out_irqfd:
4563 4564
	kvm_arch_exit();
out_fail:
A
Avi Kivity 已提交
4565 4566
	return r;
}
4567
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
4568

4569
void kvm_exit(void)
A
Avi Kivity 已提交
4570
{
4571
	debugfs_remove_recursive(kvm_debugfs_dir);
A
Avi Kivity 已提交
4572
	misc_deregister(&kvm_dev);
4573
	kmem_cache_destroy(kvm_vcpu_cache);
4574
	kvm_async_pf_deinit();
4575
	unregister_syscore_ops(&kvm_syscore_ops);
A
Avi Kivity 已提交
4576
	unregister_reboot_notifier(&kvm_reboot_notifier);
4577
	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4578
	on_each_cpu(hardware_disable_nolock, NULL, 1);
4579
	kvm_arch_hardware_unsetup();
4580
	kvm_arch_exit();
4581
	kvm_irqfd_exit();
4582
	free_cpumask_var(cpus_hardware_enabled);
4583
	kvm_vfio_ops_exit();
A
Avi Kivity 已提交
4584
}
4585
EXPORT_SYMBOL_GPL(kvm_exit);
4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668

struct kvm_vm_worker_thread_context {
	struct kvm *kvm;
	struct task_struct *parent;
	struct completion init_done;
	kvm_vm_thread_fn_t thread_fn;
	uintptr_t data;
	int err;
};

static int kvm_vm_worker_thread(void *context)
{
	/*
	 * The init_context is allocated on the stack of the parent thread, so
	 * we have to locally copy anything that is needed beyond initialization
	 */
	struct kvm_vm_worker_thread_context *init_context = context;
	struct kvm *kvm = init_context->kvm;
	kvm_vm_thread_fn_t thread_fn = init_context->thread_fn;
	uintptr_t data = init_context->data;
	int err;

	err = kthread_park(current);
	/* kthread_park(current) is never supposed to return an error */
	WARN_ON(err != 0);
	if (err)
		goto init_complete;

	err = cgroup_attach_task_all(init_context->parent, current);
	if (err) {
		kvm_err("%s: cgroup_attach_task_all failed with err %d\n",
			__func__, err);
		goto init_complete;
	}

	set_user_nice(current, task_nice(init_context->parent));

init_complete:
	init_context->err = err;
	complete(&init_context->init_done);
	init_context = NULL;

	if (err)
		return err;

	/* Wait to be woken up by the spawner before proceeding. */
	kthread_parkme();

	if (!kthread_should_stop())
		err = thread_fn(kvm, data);

	return err;
}

int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn,
				uintptr_t data, const char *name,
				struct task_struct **thread_ptr)
{
	struct kvm_vm_worker_thread_context init_context = {};
	struct task_struct *thread;

	*thread_ptr = NULL;
	init_context.kvm = kvm;
	init_context.parent = current;
	init_context.thread_fn = thread_fn;
	init_context.data = data;
	init_completion(&init_context.init_done);

	thread = kthread_run(kvm_vm_worker_thread, &init_context,
			     "%s-%d", name, task_pid_nr(current));
	if (IS_ERR(thread))
		return PTR_ERR(thread);

	/* kthread_run is never supposed to return NULL */
	WARN_ON(thread == NULL);

	wait_for_completion(&init_context.init_done);

	if (!init_context.err)
		*thread_ptr = thread;

	return init_context.err;
}