kvm_main.c 77.7 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "iodev.h"
A
Avi Kivity 已提交
20

21
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/syscore_ops.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
35
#include <linux/sched.h>
36 37
#include <linux/cpumask.h>
#include <linux/smp.h>
38
#include <linux/anon_inodes.h>
39
#include <linux/profile.h>
40
#include <linux/kvm_para.h>
41
#include <linux/pagemap.h>
42
#include <linux/mman.h>
43
#include <linux/swap.h>
44
#include <linux/bitops.h>
45
#include <linux/spinlock.h>
46
#include <linux/compat.h>
47
#include <linux/srcu.h>
J
Joerg Roedel 已提交
48
#include <linux/hugetlb.h>
49
#include <linux/slab.h>
50 51
#include <linux/sort.h>
#include <linux/bsearch.h>
A
Avi Kivity 已提交
52

A
Avi Kivity 已提交
53 54
#include <asm/processor.h>
#include <asm/io.h>
55
#include <asm/ioctl.h>
A
Avi Kivity 已提交
56
#include <asm/uaccess.h>
57
#include <asm/pgtable.h>
A
Avi Kivity 已提交
58

59
#include "coalesced_mmio.h"
60
#include "async_pf.h"
P
Paolo Bonzini 已提交
61
#include "vfio.h"
62

63 64 65
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

A
Avi Kivity 已提交
66 67 68
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

69
static unsigned int halt_poll_ns;
70 71
module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);

72 73 74
/*
 * Ordering of locks:
 *
75
 * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
76 77
 */

78
DEFINE_SPINLOCK(kvm_lock);
79
static DEFINE_RAW_SPINLOCK(kvm_count_lock);
80
LIST_HEAD(vm_list);
81

82
static cpumask_var_t cpus_hardware_enabled;
83 84
static int kvm_usage_count = 0;
static atomic_t hardware_enable_failed;
85

86 87
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
88

89 90
static __read_mostly struct preempt_ops kvm_preempt_ops;

91
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
92

A
Avi Kivity 已提交
93 94
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
95
#ifdef CONFIG_KVM_COMPAT
96 97 98
static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
				  unsigned long arg);
#endif
99 100
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
101

M
Marcelo Tosatti 已提交
102
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
103 104 105 106

static void kvm_release_pfn_dirty(pfn_t pfn);
static void mark_page_dirty_in_slot(struct kvm *kvm,
				    struct kvm_memory_slot *memslot, gfn_t gfn);
M
Marcelo Tosatti 已提交
107

108
__visible bool kvm_rebooting;
109
EXPORT_SYMBOL_GPL(kvm_rebooting);
110

111 112
static bool largepages_enabled = true;

113
bool kvm_is_reserved_pfn(pfn_t pfn)
B
Ben-Ami Yassour 已提交
114
{
115
	if (pfn_valid(pfn))
116
		return PageReserved(pfn_to_page(pfn));
B
Ben-Ami Yassour 已提交
117 118 119 120

	return true;
}

A
Avi Kivity 已提交
121 122 123
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
124
int vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
125
{
126 127
	int cpu;

128 129
	if (mutex_lock_killable(&vcpu->mutex))
		return -EINTR;
130 131
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
132
	kvm_arch_vcpu_load(vcpu, cpu);
133
	put_cpu();
134
	return 0;
A
Avi Kivity 已提交
135 136
}

137
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
138
{
139
	preempt_disable();
140
	kvm_arch_vcpu_put(vcpu);
141 142
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
143 144 145
	mutex_unlock(&vcpu->mutex);
}

146 147 148 149
static void ack_flush(void *_completed)
{
}

150
bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
151
{
152
	int i, cpu, me;
153 154
	cpumask_var_t cpus;
	bool called = true;
155 156
	struct kvm_vcpu *vcpu;

157
	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
158

159
	me = get_cpu();
160
	kvm_for_each_vcpu(i, vcpu, kvm) {
161
		kvm_make_request(req, vcpu);
162
		cpu = vcpu->cpu;
163 164 165 166 167 168

		/* Set ->requests bit before we read ->mode */
		smp_mb();

		if (cpus != NULL && cpu != -1 && cpu != me &&
		      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
169
			cpumask_set_cpu(cpu, cpus);
170
	}
171 172 173 174 175 176
	if (unlikely(cpus == NULL))
		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
	else if (!cpumask_empty(cpus))
		smp_call_function_many(cpus, ack_flush, NULL, 1);
	else
		called = false;
177
	put_cpu();
178
	free_cpumask_var(cpus);
179
	return called;
180 181
}

182
#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
183
void kvm_flush_remote_tlbs(struct kvm *kvm)
184
{
185 186 187
	long dirty_count = kvm->tlbs_dirty;

	smp_mb();
188
	if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
189
		++kvm->stat.remote_tlb_flush;
190
	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
191
}
192
EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
193
#endif
194

195 196
void kvm_reload_remote_mmus(struct kvm *kvm)
{
197
	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
198
}
199

200 201
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
202
	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
203 204
}

205
void kvm_make_scan_ioapic_request(struct kvm *kvm)
206
{
207
	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
208 209
}

R
Rusty Russell 已提交
210 211 212 213 214 215 216 217 218
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
219
	vcpu->pid = NULL;
E
Eddie Dong 已提交
220
	init_waitqueue_head(&vcpu->wq);
221
	kvm_async_pf_vcpu_init(vcpu);
R
Rusty Russell 已提交
222 223 224 225 226 227 228 229

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

230 231
	kvm_vcpu_set_in_spin_loop(vcpu, false);
	kvm_vcpu_set_dy_eligible(vcpu, false);
232
	vcpu->preempted = false;
233

234
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
235
	if (r < 0)
236
		goto fail_free_run;
R
Rusty Russell 已提交
237 238 239 240 241
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
242
	return r;
R
Rusty Russell 已提交
243 244 245 246 247
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
248
	put_pid(vcpu->pid);
249
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
250 251 252 253
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

254 255 256 257 258 259 260 261 262 263 264
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
265
	int need_tlb_flush, idx;
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
285
	idx = srcu_read_lock(&kvm->srcu);
286
	spin_lock(&kvm->mmu_lock);
287

288
	kvm->mmu_notifier_seq++;
289
	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
290 291 292 293
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

294
	spin_unlock(&kvm->mmu_lock);
295 296 297

	kvm_arch_mmu_notifier_invalidate_page(kvm, address);

298
	srcu_read_unlock(&kvm->srcu, idx);
299 300
}

301 302 303 304 305 306
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
307
	int idx;
308

309
	idx = srcu_read_lock(&kvm->srcu);
310 311 312 313
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	kvm_set_spte_hva(kvm, address, pte);
	spin_unlock(&kvm->mmu_lock);
314
	srcu_read_unlock(&kvm->srcu, idx);
315 316
}

317 318 319 320 321 322
static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
323
	int need_tlb_flush = 0, idx;
324

325
	idx = srcu_read_lock(&kvm->srcu);
326 327 328 329 330 331 332
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
333
	need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
334
	need_tlb_flush |= kvm->tlbs_dirty;
335 336 337
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
338 339 340

	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
357
	smp_wmb();
358 359
	/*
	 * The above sequence increase must be visible before the
360 361
	 * below count decrease, which is ensured by the smp_wmb above
	 * in conjunction with the smp_rmb in mmu_notifier_retry().
362 363 364 365 366 367 368 369 370
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
A
Andres Lagar-Cavilla 已提交
371 372
					      unsigned long start,
					      unsigned long end)
373 374
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
375
	int young, idx;
376

377
	idx = srcu_read_lock(&kvm->srcu);
378 379
	spin_lock(&kvm->mmu_lock);

A
Andres Lagar-Cavilla 已提交
380
	young = kvm_age_hva(kvm, start, end);
381 382 383
	if (young)
		kvm_flush_remote_tlbs(kvm);

384 385 386
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

387 388 389
	return young;
}

A
Andrea Arcangeli 已提交
390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
				       struct mm_struct *mm,
				       unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	young = kvm_test_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

406 407 408 409
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
410 411 412
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
413
	kvm_arch_flush_shadow_all(kvm);
414
	srcu_read_unlock(&kvm->srcu, idx);
415 416
}

417 418 419 420 421
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
A
Andrea Arcangeli 已提交
422
	.test_young		= kvm_mmu_notifier_test_young,
423
	.change_pte		= kvm_mmu_notifier_change_pte,
424
	.release		= kvm_mmu_notifier_release,
425
};
426 427 428 429 430 431 432 433 434 435 436 437 438 439

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

440 441
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

442 443 444 445 446 447
static void kvm_init_memslots_id(struct kvm *kvm)
{
	int i;
	struct kvm_memslots *slots = kvm->memslots;

	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
448
		slots->id_to_index[i] = slots->memslots[i].id = i;
449 450
}

451
static struct kvm *kvm_create_vm(unsigned long type)
A
Avi Kivity 已提交
452
{
453 454
	int r, i;
	struct kvm *kvm = kvm_arch_alloc_vm();
A
Avi Kivity 已提交
455

456 457 458
	if (!kvm)
		return ERR_PTR(-ENOMEM);

459
	r = kvm_arch_init_vm(kvm, type);
460
	if (r)
461
		goto out_err_no_disable;
462 463 464

	r = hardware_enable_all();
	if (r)
465
		goto out_err_no_disable;
466

467
#ifdef CONFIG_HAVE_KVM_IRQFD
468
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
469
#endif
A
Avi Kivity 已提交
470

471 472
	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);

473 474 475
	r = -ENOMEM;
	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
	if (!kvm->memslots)
476
		goto out_err_no_srcu;
477 478 479 480 481 482 483

	/*
	 * Init kvm generation close to the maximum to easily test the
	 * code of handling generation number wrap-around.
	 */
	kvm->memslots->generation = -150;

484
	kvm_init_memslots_id(kvm);
485
	if (init_srcu_struct(&kvm->srcu))
486 487 488
		goto out_err_no_srcu;
	if (init_srcu_struct(&kvm->irq_srcu))
		goto out_err_no_irq_srcu;
M
Marcelo Tosatti 已提交
489 490 491
	for (i = 0; i < KVM_NR_BUSES; i++) {
		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
					GFP_KERNEL);
492
		if (!kvm->buses[i])
M
Marcelo Tosatti 已提交
493 494
			goto out_err;
	}
495

496
	spin_lock_init(&kvm->mmu_lock);
497 498
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
G
Gregory Haskins 已提交
499
	kvm_eventfd_init(kvm);
S
Shaohua Li 已提交
500
	mutex_init(&kvm->lock);
501
	mutex_init(&kvm->irq_lock);
502
	mutex_init(&kvm->slots_lock);
I
Izik Eidus 已提交
503
	atomic_set(&kvm->users_count, 1);
504
	INIT_LIST_HEAD(&kvm->devices);
505 506 507 508 509

	r = kvm_init_mmu_notifier(kvm);
	if (r)
		goto out_err;

510
	spin_lock(&kvm_lock);
511
	list_add(&kvm->vm_list, &vm_list);
512
	spin_unlock(&kvm_lock);
513

514
	return kvm;
515 516

out_err:
517 518
	cleanup_srcu_struct(&kvm->irq_srcu);
out_err_no_irq_srcu:
519
	cleanup_srcu_struct(&kvm->srcu);
520
out_err_no_srcu:
521
	hardware_disable_all();
522
out_err_no_disable:
M
Marcelo Tosatti 已提交
523 524
	for (i = 0; i < KVM_NR_BUSES; i++)
		kfree(kvm->buses[i]);
525
	kfree(kvm->memslots);
526
	kvm_arch_free_vm(kvm);
527
	return ERR_PTR(r);
528 529
}

530 531 532 533
/*
 * Avoid using vmalloc for a small buffer.
 * Should not be used when the size is statically known.
 */
534
void *kvm_kvzalloc(unsigned long size)
535 536 537 538 539 540 541
{
	if (size > PAGE_SIZE)
		return vzalloc(size);
	else
		return kzalloc(size, GFP_KERNEL);
}

542
void kvm_kvfree(const void *addr)
543 544 545 546 547 548 549
{
	if (is_vmalloc_addr(addr))
		vfree(addr);
	else
		kfree(addr);
}

550 551 552 553 554
static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
	if (!memslot->dirty_bitmap)
		return;

555
	kvm_kvfree(memslot->dirty_bitmap);
556 557 558
	memslot->dirty_bitmap = NULL;
}

A
Avi Kivity 已提交
559 560 561
/*
 * Free any memory in @free but not in @dont.
 */
562
static void kvm_free_physmem_slot(struct kvm *kvm, struct kvm_memory_slot *free,
A
Avi Kivity 已提交
563 564 565
				  struct kvm_memory_slot *dont)
{
	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
566
		kvm_destroy_dirty_bitmap(free);
A
Avi Kivity 已提交
567

568
	kvm_arch_free_memslot(kvm, free, dont);
M
Marcelo Tosatti 已提交
569

A
Avi Kivity 已提交
570 571 572
	free->npages = 0;
}

573
static void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
574
{
575
	struct kvm_memslots *slots = kvm->memslots;
576
	struct kvm_memory_slot *memslot;
577

578
	kvm_for_each_memslot(memslot, slots)
579
		kvm_free_physmem_slot(kvm, memslot, NULL);
A
Avi Kivity 已提交
580

581
	kfree(kvm->memslots);
A
Avi Kivity 已提交
582 583
}

584 585 586 587 588 589 590 591 592 593 594 595 596
static void kvm_destroy_devices(struct kvm *kvm)
{
	struct list_head *node, *tmp;

	list_for_each_safe(node, tmp, &kvm->devices) {
		struct kvm_device *dev =
			list_entry(node, struct kvm_device, vm_node);

		list_del(node);
		dev->ops->destroy(dev);
	}
}

597 598
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
599
	int i;
600 601
	struct mm_struct *mm = kvm->mm;

602
	kvm_arch_sync_events(kvm);
603
	spin_lock(&kvm_lock);
604
	list_del(&kvm->vm_list);
605
	spin_unlock(&kvm_lock);
606
	kvm_free_irq_routing(kvm);
M
Marcelo Tosatti 已提交
607 608
	for (i = 0; i < KVM_NR_BUSES; i++)
		kvm_io_bus_destroy(kvm->buses[i]);
609
	kvm_coalesced_mmio_free(kvm);
610 611
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
612
#else
613
	kvm_arch_flush_shadow_all(kvm);
614
#endif
615
	kvm_arch_destroy_vm(kvm);
616
	kvm_destroy_devices(kvm);
617
	kvm_free_physmem(kvm);
618
	cleanup_srcu_struct(&kvm->irq_srcu);
619 620
	cleanup_srcu_struct(&kvm->srcu);
	kvm_arch_free_vm(kvm);
621
	hardware_disable_all();
622
	mmdrop(mm);
623 624
}

I
Izik Eidus 已提交
625 626 627 628 629 630 631 632 633 634 635 636 637 638
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


639 640 641 642
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
643 644
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
645
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
646 647 648
	return 0;
}

649 650
/*
 * Allocation size is twice as large as the actual dirty bitmap size.
651
 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
652
 */
653 654
static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
{
655
	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
656

657
	memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
658 659 660 661 662 663
	if (!memslot->dirty_bitmap)
		return -ENOMEM;

	return 0;
}

664
/*
665 666 667 668
 * Insert memslot and re-sort memslots based on their GFN,
 * so binary search could be used to lookup GFN.
 * Sorting algorithm takes advantage of having initially
 * sorted array and known changed memslot position.
669
 */
670 671
static void update_memslots(struct kvm_memslots *slots,
			    struct kvm_memory_slot *new)
672
{
673 674
	int id = new->id;
	int i = slots->id_to_index[id];
675
	struct kvm_memory_slot *mslots = slots->memslots;
676

677
	WARN_ON(mslots[i].id != id);
678
	if (!new->npages) {
679
		WARN_ON(!mslots[i].npages);
680
		new->base_gfn = 0;
681
		new->flags = 0;
682 683 684 685 686 687
		if (mslots[i].npages)
			slots->used_slots--;
	} else {
		if (!mslots[i].npages)
			slots->used_slots++;
	}
688

689
	while (i < KVM_MEM_SLOTS_NUM - 1 &&
690 691 692
	       new->base_gfn <= mslots[i + 1].base_gfn) {
		if (!mslots[i + 1].npages)
			break;
693 694 695 696
		mslots[i] = mslots[i + 1];
		slots->id_to_index[mslots[i].id] = i;
		i++;
	}
697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713

	/*
	 * The ">=" is needed when creating a slot with base_gfn == 0,
	 * so that it moves before all those with base_gfn == npages == 0.
	 *
	 * On the other hand, if new->npages is zero, the above loop has
	 * already left i pointing to the beginning of the empty part of
	 * mslots, and the ">=" would move the hole backwards in this
	 * case---which is wrong.  So skip the loop when deleting a slot.
	 */
	if (new->npages) {
		while (i > 0 &&
		       new->base_gfn >= mslots[i - 1].base_gfn) {
			mslots[i] = mslots[i - 1];
			slots->id_to_index[mslots[i].id] = i;
			i--;
		}
714 715
	} else
		WARN_ON_ONCE(i != slots->used_slots);
716

717 718
	mslots[i] = *new;
	slots->id_to_index[mslots[i].id] = i;
719 720
}

721 722
static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
{
X
Xiao Guangrong 已提交
723 724
	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;

725
#ifdef __KVM_HAVE_READONLY_MEM
X
Xiao Guangrong 已提交
726 727 728 729
	valid_flags |= KVM_MEM_READONLY;
#endif

	if (mem->flags & ~valid_flags)
730 731 732 733 734
		return -EINVAL;

	return 0;
}

735
static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
736
		struct kvm_memslots *slots)
737 738 739
{
	struct kvm_memslots *old_memslots = kvm->memslots;

740 741 742 743 744 745 746
	/*
	 * Set the low bit in the generation, which disables SPTE caching
	 * until the end of synchronize_srcu_expedited.
	 */
	WARN_ON(old_memslots->generation & 1);
	slots->generation = old_memslots->generation + 1;

747 748
	rcu_assign_pointer(kvm->memslots, slots);
	synchronize_srcu_expedited(&kvm->srcu);
749

750 751 752 753 754 755 756
	/*
	 * Increment the new memslot generation a second time. This prevents
	 * vm exits that race with memslot updates from caching a memslot
	 * generation that will (potentially) be valid forever.
	 */
	slots->generation++;

757 758 759
	kvm_arch_memslots_updated(kvm);

	return old_memslots;
760 761
}

A
Avi Kivity 已提交
762 763 764 765 766
/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
767
 *
768
 * Must be called holding kvm->slots_lock for write.
A
Avi Kivity 已提交
769
 */
770
int __kvm_set_memory_region(struct kvm *kvm,
771
			    struct kvm_userspace_memory_region *mem)
A
Avi Kivity 已提交
772
{
773
	int r;
A
Avi Kivity 已提交
774
	gfn_t base_gfn;
775
	unsigned long npages;
776
	struct kvm_memory_slot *slot;
A
Avi Kivity 已提交
777
	struct kvm_memory_slot old, new;
778
	struct kvm_memslots *slots = NULL, *old_memslots;
779
	enum kvm_mr_change change;
A
Avi Kivity 已提交
780

781 782 783 784
	r = check_memory_region_flags(mem);
	if (r)
		goto out;

A
Avi Kivity 已提交
785 786 787 788 789 790
	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
791
	/* We can read the guest memory with __xxx_user() later on. */
792
	if ((mem->slot < KVM_USER_MEM_SLOTS) &&
793
	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
794 795 796
	     !access_ok(VERIFY_WRITE,
			(void __user *)(unsigned long)mem->userspace_addr,
			mem->memory_size)))
797
		goto out;
798
	if (mem->slot >= KVM_MEM_SLOTS_NUM)
A
Avi Kivity 已提交
799 800 801 802
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

803
	slot = id_to_memslot(kvm->memslots, mem->slot);
A
Avi Kivity 已提交
804 805 806
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

807 808 809
	if (npages > KVM_MEM_MAX_NR_PAGES)
		goto out;

A
Avi Kivity 已提交
810 811 812
	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

813
	new = old = *slot;
A
Avi Kivity 已提交
814

815
	new.id = mem->slot;
A
Avi Kivity 已提交
816 817 818 819
	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

820 821 822 823 824
	if (npages) {
		if (!old.npages)
			change = KVM_MR_CREATE;
		else { /* Modify an existing slot. */
			if ((mem->userspace_addr != old.userspace_addr) ||
825 826
			    (npages != old.npages) ||
			    ((new.flags ^ old.flags) & KVM_MEM_READONLY))
827 828 829 830 831 832 833 834 835 836 837 838 839 840
				goto out;

			if (base_gfn != old.base_gfn)
				change = KVM_MR_MOVE;
			else if (new.flags != old.flags)
				change = KVM_MR_FLAGS_ONLY;
			else { /* Nothing to change. */
				r = 0;
				goto out;
			}
		}
	} else if (old.npages) {
		change = KVM_MR_DELETE;
	} else /* Modify a non-existent slot: disallowed. */
841
		goto out;
A
Avi Kivity 已提交
842

843
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
844 845 846
		/* Check for overlaps */
		r = -EEXIST;
		kvm_for_each_memslot(slot, kvm->memslots) {
847 848
			if ((slot->id >= KVM_USER_MEM_SLOTS) ||
			    (slot->id == mem->slot))
849 850 851 852 853
				continue;
			if (!((base_gfn + npages <= slot->base_gfn) ||
			      (base_gfn >= slot->base_gfn + slot->npages)))
				goto out;
		}
A
Avi Kivity 已提交
854 855 856 857
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
858
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
859 860

	r = -ENOMEM;
861
	if (change == KVM_MR_CREATE) {
862
		new.userspace_addr = mem->userspace_addr;
863

864
		if (kvm_arch_create_memslot(kvm, &new, npages))
865
			goto out_free;
A
Avi Kivity 已提交
866
	}
867

A
Avi Kivity 已提交
868 869
	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
870
		if (kvm_create_dirty_bitmap(&new) < 0)
871
			goto out_free;
A
Avi Kivity 已提交
872 873
	}

874 875 876 877 878
	slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
			GFP_KERNEL);
	if (!slots)
		goto out_free;

879
	if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
880 881 882
		slot = id_to_memslot(slots, mem->slot);
		slot->flags |= KVM_MEMSLOT_INVALID;

883
		old_memslots = install_new_memslots(kvm, slots);
884

885 886
		/* slot was deleted or moved, clear iommu mapping */
		kvm_iommu_unmap_pages(kvm, &old);
887 888
		/* From this point no new shadow pages pointing to a deleted,
		 * or moved, memslot will be created.
889 890 891 892 893
		 *
		 * validation of sp->gfn happens in:
		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
		 * 	- kvm_is_visible_gfn (mmu_check_roots)
		 */
894
		kvm_arch_flush_shadow_memslot(kvm, slot);
895 896 897 898 899 900

		/*
		 * We can re-use the old_memslots from above, the only difference
		 * from the currently installed memslots is the invalid flag.  This
		 * will get overwritten by update_memslots anyway.
		 */
901
		slots = old_memslots;
902
	}
903

904
	r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
905
	if (r)
906
		goto out_slots;
907

908
	/* actual memory is freed via old in kvm_free_physmem_slot below */
909
	if (change == KVM_MR_DELETE) {
910
		new.dirty_bitmap = NULL;
911
		memset(&new.arch, 0, sizeof(new.arch));
912 913
	}

914 915
	update_memslots(slots, &new);
	old_memslots = install_new_memslots(kvm, slots);
916

917
	kvm_arch_commit_memory_region(kvm, mem, &old, change);
918

919
	kvm_free_physmem_slot(kvm, &old, &new);
920 921
	kfree(old_memslots);

922 923
	/*
	 * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
924 925 926 927 928 929
	 * un-mapped and re-mapped if their base changes.  Since base change
	 * unmapping is handled above with slot deletion, mapping alone is
	 * needed here.  Anything else the iommu might care about for existing
	 * slots (size changes, userspace addr changes and read-only flag
	 * changes) is disallowed above, so any other attribute changes getting
	 * here can be skipped.
930
	 */
931 932
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
		r = kvm_iommu_map_pages(kvm, &new);
933
		return r;
934 935
	}

A
Avi Kivity 已提交
936 937
	return 0;

938 939
out_slots:
	kfree(slots);
940
out_free:
941
	kvm_free_physmem_slot(kvm, &new, &old);
A
Avi Kivity 已提交
942 943
out:
	return r;
944
}
945 946 947
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
948
			  struct kvm_userspace_memory_region *mem)
949 950 951
{
	int r;

952
	mutex_lock(&kvm->slots_lock);
953
	r = __kvm_set_memory_region(kvm, mem);
954
	mutex_unlock(&kvm->slots_lock);
955 956
	return r;
}
957 958
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

959 960
static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
					  struct kvm_userspace_memory_region *mem)
961
{
962
	if (mem->slot >= KVM_USER_MEM_SLOTS)
963
		return -EINVAL;
964
	return kvm_set_memory_region(kvm, mem);
A
Avi Kivity 已提交
965 966
}

967 968
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
969 970 971
{
	struct kvm_memory_slot *memslot;
	int r, i;
972
	unsigned long n;
A
Avi Kivity 已提交
973 974 975
	unsigned long any = 0;

	r = -EINVAL;
976
	if (log->slot >= KVM_USER_MEM_SLOTS)
A
Avi Kivity 已提交
977 978
		goto out;

979
	memslot = id_to_memslot(kvm->memslots, log->slot);
A
Avi Kivity 已提交
980 981 982 983
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

984
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
985

986
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
987 988 989 990 991 992
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

993 994
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
995 996 997 998 999

	r = 0;
out:
	return r;
}
1000
EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
A
Avi Kivity 已提交
1001

1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
/**
 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
 *	are dirty write protect them for next write.
 * @kvm:	pointer to kvm instance
 * @log:	slot id and address to which we copy the log
 * @is_dirty:	flag set if any page is dirty
 *
 * We need to keep it in mind that VCPU threads can write to the bitmap
 * concurrently. So, to avoid losing track of dirty pages we keep the
 * following order:
 *
 *    1. Take a snapshot of the bit and clear it if needed.
 *    2. Write protect the corresponding page.
 *    3. Copy the snapshot to the userspace.
 *    4. Upon return caller flushes TLB's if needed.
 *
 * Between 2 and 4, the guest may write to the page using the remaining TLB
 * entry.  This is not a problem because the page is reported dirty using
 * the snapshot taken before and step 4 ensures that writes done after
 * exiting to userspace will be logged for the next call.
 *
 */
int kvm_get_dirty_log_protect(struct kvm *kvm,
			struct kvm_dirty_log *log, bool *is_dirty)
{
	struct kvm_memory_slot *memslot;
	int r, i;
	unsigned long n;
	unsigned long *dirty_bitmap;
	unsigned long *dirty_bitmap_buffer;

	r = -EINVAL;
	if (log->slot >= KVM_USER_MEM_SLOTS)
		goto out;

	memslot = id_to_memslot(kvm->memslots, log->slot);

	dirty_bitmap = memslot->dirty_bitmap;
	r = -ENOENT;
	if (!dirty_bitmap)
		goto out;

	n = kvm_dirty_bitmap_bytes(memslot);

	dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
	memset(dirty_bitmap_buffer, 0, n);

	spin_lock(&kvm->mmu_lock);
	*is_dirty = false;
	for (i = 0; i < n / sizeof(long); i++) {
		unsigned long mask;
		gfn_t offset;

		if (!dirty_bitmap[i])
			continue;

		*is_dirty = true;

		mask = xchg(&dirty_bitmap[i], 0);
		dirty_bitmap_buffer[i] = mask;

		offset = i * BITS_PER_LONG;
1065
		kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset,
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
								mask);
	}

	spin_unlock(&kvm->mmu_lock);

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
		goto out;

	r = 0;
out:
	return r;
}
EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
#endif

1082 1083 1084 1085 1086
bool kvm_largepages_enabled(void)
{
	return largepages_enabled;
}

1087 1088 1089 1090 1091 1092
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

1093 1094 1095 1096
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
A
Avi Kivity 已提交
1097
EXPORT_SYMBOL_GPL(gfn_to_memslot);
A
Avi Kivity 已提交
1098

1099 1100
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
1101
	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1102

1103
	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1104 1105
	      memslot->flags & KVM_MEMSLOT_INVALID)
		return 0;
1106

1107
	return 1;
1108 1109 1110
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

J
Joerg Roedel 已提交
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

X
Xiao Guangrong 已提交
1135 1136 1137 1138 1139 1140 1141
static bool memslot_is_readonly(struct kvm_memory_slot *slot)
{
	return slot->flags & KVM_MEM_READONLY;
}

static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				       gfn_t *nr_pages, bool write)
I
Izik Eidus 已提交
1142
{
1143
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
X
Xiao Guangrong 已提交
1144
		return KVM_HVA_ERR_BAD;
1145

X
Xiao Guangrong 已提交
1146 1147
	if (memslot_is_readonly(slot) && write)
		return KVM_HVA_ERR_RO_BAD;
1148 1149 1150 1151

	if (nr_pages)
		*nr_pages = slot->npages - (gfn - slot->base_gfn);

X
Xiao Guangrong 已提交
1152
	return __gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
1153
}
1154

X
Xiao Guangrong 已提交
1155 1156 1157 1158
static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				     gfn_t *nr_pages)
{
	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
I
Izik Eidus 已提交
1159
}
1160

X
Xiao Guangrong 已提交
1161
unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1162
					gfn_t gfn)
X
Xiao Guangrong 已提交
1163 1164 1165 1166 1167
{
	return gfn_to_hva_many(slot, gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);

1168 1169
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
1170
	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1171
}
1172
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1173

1174
/*
1175 1176
 * If writable is set to false, the hva returned by this function is only
 * allowed to be read.
1177
 */
1178 1179
unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
				      gfn_t gfn, bool *writable)
1180
{
1181 1182 1183
	unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);

	if (!kvm_is_error_hva(hva) && writable)
1184 1185
		*writable = !memslot_is_readonly(slot);

1186
	return hva;
1187 1188
}

1189 1190 1191 1192 1193 1194 1195
unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

	return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

1196
static int kvm_read_hva(void *data, void __user *hva, int len)
1197
{
1198 1199 1200 1201 1202 1203
	return __copy_from_user(data, hva, len);
}

static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
{
	return __copy_from_user_inatomic(data, hva, len);
1204 1205
}

G
Geoff Levand 已提交
1206
static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
	unsigned long start, int write, struct page **page)
{
	int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;

	if (write)
		flags |= FOLL_WRITE;

	return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
}

1217 1218 1219 1220 1221 1222 1223 1224 1225
static inline int check_user_page_hwpoison(unsigned long addr)
{
	int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;

	rc = __get_user_pages(current, current->mm, addr, 1,
			      flags, NULL, NULL, NULL);
	return rc == -EHWPOISON;
}

X
Xiao Guangrong 已提交
1226 1227 1228 1229 1230 1231
/*
 * The atomic path to get the writable pfn which will be stored in @pfn,
 * true indicates success, otherwise false is returned.
 */
static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
			    bool write_fault, bool *writable, pfn_t *pfn)
A
Avi Kivity 已提交
1232
{
1233
	struct page *page[1];
X
Xiao Guangrong 已提交
1234
	int npages;
A
Avi Kivity 已提交
1235

X
Xiao Guangrong 已提交
1236 1237
	if (!(async || atomic))
		return false;
1238

1239 1240 1241 1242 1243 1244 1245
	/*
	 * Fast pin a writable pfn only if it is a write fault request
	 * or the caller allows to map a writable pfn for a read fault
	 * request.
	 */
	if (!(write_fault || writable))
		return false;
1246

X
Xiao Guangrong 已提交
1247 1248 1249
	npages = __get_user_pages_fast(addr, 1, 1, page);
	if (npages == 1) {
		*pfn = page_to_pfn(page[0]);
1250

X
Xiao Guangrong 已提交
1251 1252 1253 1254
		if (writable)
			*writable = true;
		return true;
	}
1255

X
Xiao Guangrong 已提交
1256 1257
	return false;
}
1258

X
Xiao Guangrong 已提交
1259 1260 1261 1262 1263 1264 1265 1266 1267
/*
 * The slow path to get the pfn of the specified host virtual address,
 * 1 indicates success, -errno is returned if error is detected.
 */
static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
			   bool *writable, pfn_t *pfn)
{
	struct page *page[1];
	int npages = 0;
1268

X
Xiao Guangrong 已提交
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
	might_sleep();

	if (writable)
		*writable = write_fault;

	if (async) {
		down_read(&current->mm->mmap_sem);
		npages = get_user_page_nowait(current, current->mm,
					      addr, write_fault, page);
		up_read(&current->mm->mmap_sem);
1279 1280 1281 1282
	} else
		npages = __get_user_pages_unlocked(current, current->mm, addr, 1,
						   write_fault, 0, page,
						   FOLL_TOUCH|FOLL_HWPOISON);
X
Xiao Guangrong 已提交
1283 1284 1285 1286
	if (npages != 1)
		return npages;

	/* map read fault as writable if possible */
1287
	if (unlikely(!write_fault) && writable) {
X
Xiao Guangrong 已提交
1288 1289 1290 1291 1292 1293 1294
		struct page *wpage[1];

		npages = __get_user_pages_fast(addr, 1, 1, wpage);
		if (npages == 1) {
			*writable = true;
			put_page(page[0]);
			page[0] = wpage[0];
1295
		}
X
Xiao Guangrong 已提交
1296 1297

		npages = 1;
1298
	}
X
Xiao Guangrong 已提交
1299 1300 1301
	*pfn = page_to_pfn(page[0]);
	return npages;
}
I
Izik Eidus 已提交
1302

X
Xiao Guangrong 已提交
1303 1304 1305 1306
static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
{
	if (unlikely(!(vma->vm_flags & VM_READ)))
		return false;
1307

X
Xiao Guangrong 已提交
1308 1309
	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
		return false;
1310

X
Xiao Guangrong 已提交
1311 1312
	return true;
}
1313

1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327
/*
 * Pin guest page in memory and return its pfn.
 * @addr: host virtual address which maps memory to the guest
 * @atomic: whether this function can sleep
 * @async: whether this function need to wait IO complete if the
 *         host page is not in the memory
 * @write_fault: whether we should get a writable host page
 * @writable: whether it allows to map a writable host page for !@write_fault
 *
 * The function will map a writable host page for these two cases:
 * 1): @write_fault = true
 * 2): @write_fault = false && @writable, @writable will tell the caller
 *     whether the mapping is writable.
 */
X
Xiao Guangrong 已提交
1328 1329 1330 1331 1332 1333
static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
			bool write_fault, bool *writable)
{
	struct vm_area_struct *vma;
	pfn_t pfn = 0;
	int npages;
1334

X
Xiao Guangrong 已提交
1335 1336
	/* we can do it either atomically or asynchronously, not both */
	BUG_ON(atomic && async);
1337

X
Xiao Guangrong 已提交
1338 1339 1340 1341 1342 1343 1344 1345 1346
	if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
		return pfn;

	if (atomic)
		return KVM_PFN_ERR_FAULT;

	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
	if (npages == 1)
		return pfn;
1347

X
Xiao Guangrong 已提交
1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361
	down_read(&current->mm->mmap_sem);
	if (npages == -EHWPOISON ||
	      (!async && check_user_page_hwpoison(addr))) {
		pfn = KVM_PFN_ERR_HWPOISON;
		goto exit;
	}

	vma = find_vma_intersection(current->mm, addr, addr + 1);

	if (vma == NULL)
		pfn = KVM_PFN_ERR_FAULT;
	else if ((vma->vm_flags & VM_PFNMAP)) {
		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
			vma->vm_pgoff;
1362
		BUG_ON(!kvm_is_reserved_pfn(pfn));
X
Xiao Guangrong 已提交
1363
	} else {
X
Xiao Guangrong 已提交
1364
		if (async && vma_is_valid(vma, write_fault))
X
Xiao Guangrong 已提交
1365 1366 1367 1368 1369
			*async = true;
		pfn = KVM_PFN_ERR_FAULT;
	}
exit:
	up_read(&current->mm->mmap_sem);
1370
	return pfn;
1371 1372
}

X
Xiao Guangrong 已提交
1373 1374 1375
static pfn_t
__gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
		     bool *async, bool write_fault, bool *writable)
1376
{
X
Xiao Guangrong 已提交
1377 1378 1379 1380 1381 1382
	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);

	if (addr == KVM_HVA_ERR_RO_BAD)
		return KVM_PFN_ERR_RO_FAULT;

	if (kvm_is_error_hva(addr))
1383
		return KVM_PFN_NOSLOT;
X
Xiao Guangrong 已提交
1384 1385 1386 1387 1388 1389 1390 1391 1392

	/* Do not map writable pfn in the readonly memslot. */
	if (writable && memslot_is_readonly(slot)) {
		*writable = false;
		writable = NULL;
	}

	return hva_to_pfn(addr, atomic, async, write_fault,
			  writable);
1393 1394
}

1395 1396
static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
			  bool write_fault, bool *writable)
1397
{
X
Xiao Guangrong 已提交
1398
	struct kvm_memory_slot *slot;
1399

1400 1401 1402
	if (async)
		*async = false;

X
Xiao Guangrong 已提交
1403
	slot = gfn_to_memslot(kvm, gfn);
1404

X
Xiao Guangrong 已提交
1405 1406
	return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
				    writable);
1407 1408 1409 1410
}

pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
{
1411
	return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1412 1413 1414
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);

1415 1416
pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
		       bool write_fault, bool *writable)
1417
{
1418
	return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1419 1420 1421
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_async);

1422 1423
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
{
1424
	return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1425
}
1426 1427
EXPORT_SYMBOL_GPL(gfn_to_pfn);

1428 1429 1430 1431 1432 1433 1434
pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
		      bool *writable)
{
	return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);

1435
pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1436
{
X
Xiao Guangrong 已提交
1437
	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1438 1439
}

1440
pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1441
{
X
Xiao Guangrong 已提交
1442
	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1443
}
1444
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1445

1446 1447 1448 1449 1450 1451
int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
								  int nr_pages)
{
	unsigned long addr;
	gfn_t entry;

1452
	addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462
	if (kvm_is_error_hva(addr))
		return -1;

	if (entry < nr_pages)
		return 0;

	return __get_user_pages_fast(addr, nr_pages, 1, pages);
}
EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);

X
Xiao Guangrong 已提交
1463 1464
static struct page *kvm_pfn_to_page(pfn_t pfn)
{
1465
	if (is_error_noslot_pfn(pfn))
1466
		return KVM_ERR_PTR_BAD_PAGE;
X
Xiao Guangrong 已提交
1467

1468
	if (kvm_is_reserved_pfn(pfn)) {
1469
		WARN_ON(1);
1470
		return KVM_ERR_PTR_BAD_PAGE;
1471
	}
X
Xiao Guangrong 已提交
1472 1473 1474 1475

	return pfn_to_page(pfn);
}

1476 1477
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
1478 1479 1480 1481
	pfn_t pfn;

	pfn = gfn_to_pfn(kvm, gfn);

X
Xiao Guangrong 已提交
1482
	return kvm_pfn_to_page(pfn);
A
Avi Kivity 已提交
1483
}
1484

A
Avi Kivity 已提交
1485 1486
EXPORT_SYMBOL_GPL(gfn_to_page);

1487 1488
void kvm_release_page_clean(struct page *page)
{
1489 1490
	WARN_ON(is_error_page(page));

1491
	kvm_release_pfn_clean(page_to_pfn(page));
1492 1493 1494
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

1495 1496
void kvm_release_pfn_clean(pfn_t pfn)
{
1497
	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1498
		put_page(pfn_to_page(pfn));
1499 1500 1501
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1502
void kvm_release_page_dirty(struct page *page)
1503
{
X
Xiao Guangrong 已提交
1504 1505
	WARN_ON(is_error_page(page));

1506 1507 1508 1509
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

1510
static void kvm_release_pfn_dirty(pfn_t pfn)
1511 1512 1513 1514 1515 1516 1517
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}

void kvm_set_pfn_dirty(pfn_t pfn)
{
1518
	if (!kvm_is_reserved_pfn(pfn)) {
1519 1520 1521 1522
		struct page *page = pfn_to_page(pfn);
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1523
}
1524 1525 1526 1527
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
1528
	if (!kvm_is_reserved_pfn(pfn))
1529
		mark_page_accessed(pfn_to_page(pfn));
1530 1531 1532 1533 1534
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
1535
	if (!kvm_is_reserved_pfn(pfn))
1536
		get_page(pfn_to_page(pfn));
1537 1538
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1539

1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
1551 1552
	int r;
	unsigned long addr;
1553

1554
	addr = gfn_to_hva_prot(kvm, gfn, NULL);
1555 1556
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1557
	r = kvm_read_hva(data, (void __user *)addr + offset, len);
1558
	if (r)
1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1584 1585 1586 1587 1588 1589 1590 1591
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

1592
	addr = gfn_to_hva_prot(kvm, gfn, NULL);
1593 1594
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1595
	pagefault_disable();
1596
	r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
1597
	pagefault_enable();
1598 1599 1600 1601 1602 1603
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

1604 1605 1606
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
1607 1608
	int r;
	unsigned long addr;
1609

1610 1611 1612
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1613
	r = __copy_to_user((void __user *)addr + offset, data, len);
1614
	if (r)
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
1640
EXPORT_SYMBOL_GPL(kvm_write_guest);
1641

1642
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1643
			      gpa_t gpa, unsigned long len)
1644 1645 1646
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int offset = offset_in_page(gpa);
1647 1648 1649 1650
	gfn_t start_gfn = gpa >> PAGE_SHIFT;
	gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
	gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
	gfn_t nr_pages_avail;
1651 1652 1653

	ghc->gpa = gpa;
	ghc->generation = slots->generation;
1654 1655 1656 1657
	ghc->len = len;
	ghc->memslot = gfn_to_memslot(kvm, start_gfn);
	ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, &nr_pages_avail);
	if (!kvm_is_error_hva(ghc->hva) && nr_pages_avail >= nr_pages_needed) {
1658
		ghc->hva += offset;
1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674
	} else {
		/*
		 * If the requested region crosses two memslots, we still
		 * verify that the entire region is valid here.
		 */
		while (start_gfn <= end_gfn) {
			ghc->memslot = gfn_to_memslot(kvm, start_gfn);
			ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
						   &nr_pages_avail);
			if (kvm_is_error_hva(ghc->hva))
				return -EFAULT;
			start_gfn += nr_pages_avail;
		}
		/* Use the slow path for cross page reads and writes. */
		ghc->memslot = NULL;
	}
1675 1676 1677 1678 1679 1680 1681 1682 1683 1684
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);

int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

1685 1686
	BUG_ON(len > ghc->len);

1687
	if (slots->generation != ghc->generation)
1688 1689 1690 1691
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);

	if (unlikely(!ghc->memslot))
		return kvm_write_guest(kvm, ghc->gpa, data, len);
1692 1693 1694 1695

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

1696
	r = __copy_to_user((void __user *)ghc->hva, data, len);
1697 1698 1699 1700 1701 1702 1703 1704
	if (r)
		return -EFAULT;
	mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);

1705 1706 1707 1708 1709 1710
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

1711 1712
	BUG_ON(len > ghc->len);

1713
	if (slots->generation != ghc->generation)
1714 1715 1716 1717
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);

	if (unlikely(!ghc->memslot))
		return kvm_read_guest(kvm, ghc->gpa, data, len);
1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

	r = __copy_from_user(data, (void __user *)ghc->hva, len);
	if (r)
		return -EFAULT;

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_cached);

1730 1731
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
1732 1733 1734
	const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));

	return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

1745
	while ((seg = next_segment(len, offset)) != 0) {
1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

1757 1758 1759
static void mark_page_dirty_in_slot(struct kvm *kvm,
				    struct kvm_memory_slot *memslot,
				    gfn_t gfn)
A
Avi Kivity 已提交
1760
{
R
Rusty Russell 已提交
1761 1762
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1763

1764
		set_bit_le(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1765 1766 1767
	}
}

1768 1769 1770 1771 1772 1773 1774
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = gfn_to_memslot(kvm, gfn);
	mark_page_dirty_in_slot(kvm, memslot, gfn);
}
1775
EXPORT_SYMBOL_GPL(mark_page_dirty);
1776

1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790
static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
{
	if (kvm_arch_vcpu_runnable(vcpu)) {
		kvm_make_request(KVM_REQ_UNHALT, vcpu);
		return -EINTR;
	}
	if (kvm_cpu_has_pending_timer(vcpu))
		return -EINTR;
	if (signal_pending(current))
		return -EINTR;

	return 0;
}

E
Eddie Dong 已提交
1791 1792 1793
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1794
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1795
{
1796
	ktime_t start, cur;
1797
	DEFINE_WAIT(wait);
1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814
	bool waited = false;

	start = cur = ktime_get();
	if (halt_poll_ns) {
		ktime_t stop = ktime_add_ns(ktime_get(), halt_poll_ns);
		do {
			/*
			 * This sets KVM_REQ_UNHALT if an interrupt
			 * arrives.
			 */
			if (kvm_vcpu_check_block(vcpu) < 0) {
				++vcpu->stat.halt_successful_poll;
				goto out;
			}
			cur = ktime_get();
		} while (single_task_running() && ktime_before(cur, stop));
	}
1815 1816 1817 1818

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

1819
		if (kvm_vcpu_check_block(vcpu) < 0)
1820 1821
			break;

1822
		waited = true;
E
Eddie Dong 已提交
1823 1824
		schedule();
	}
1825

1826
	finish_wait(&vcpu->wq, &wait);
1827 1828 1829 1830
	cur = ktime_get();

out:
	trace_kvm_vcpu_wakeup(ktime_to_ns(cur) - ktime_to_ns(start), waited);
E
Eddie Dong 已提交
1831
}
1832
EXPORT_SYMBOL_GPL(kvm_vcpu_block);
E
Eddie Dong 已提交
1833

1834
#ifndef CONFIG_S390
1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855
/*
 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
 */
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
	int me;
	int cpu = vcpu->cpu;
	wait_queue_head_t *wqp;

	wqp = kvm_arch_vcpu_wq(vcpu);
	if (waitqueue_active(wqp)) {
		wake_up_interruptible(wqp);
		++vcpu->stat.halt_wakeup;
	}

	me = get_cpu();
	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
		if (kvm_arch_vcpu_should_kick(vcpu))
			smp_send_reschedule(cpu);
	put_cpu();
}
1856
EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
1857
#endif /* !CONFIG_S390 */
1858

1859
int kvm_vcpu_yield_to(struct kvm_vcpu *target)
1860 1861 1862
{
	struct pid *pid;
	struct task_struct *task = NULL;
1863
	int ret = 0;
1864 1865 1866 1867

	rcu_read_lock();
	pid = rcu_dereference(target->pid);
	if (pid)
1868
		task = get_pid_task(pid, PIDTYPE_PID);
1869 1870
	rcu_read_unlock();
	if (!task)
1871 1872
		return ret;
	ret = yield_to(task, 1);
1873
	put_task_struct(task);
1874 1875

	return ret;
1876 1877 1878
}
EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);

1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900
/*
 * Helper that checks whether a VCPU is eligible for directed yield.
 * Most eligible candidate to yield is decided by following heuristics:
 *
 *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
 *  (preempted lock holder), indicated by @in_spin_loop.
 *  Set at the beiginning and cleared at the end of interception/PLE handler.
 *
 *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
 *  chance last time (mostly it has become eligible now since we have probably
 *  yielded to lockholder in last iteration. This is done by toggling
 *  @dy_eligible each time a VCPU checked for eligibility.)
 *
 *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
 *  to preempted lock-holder could result in wrong VCPU selection and CPU
 *  burning. Giving priority for a potential lock-holder increases lock
 *  progress.
 *
 *  Since algorithm is based on heuristics, accessing another VCPU data without
 *  locking does not harm. It may result in trying to yield to  same VCPU, fail
 *  and continue with next VCPU and so on.
 */
1901
static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
1902
{
1903
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
1904 1905 1906
	bool eligible;

	eligible = !vcpu->spin_loop.in_spin_loop ||
1907
		    vcpu->spin_loop.dy_eligible;
1908 1909 1910 1911 1912

	if (vcpu->spin_loop.in_spin_loop)
		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);

	return eligible;
1913 1914
#else
	return true;
1915
#endif
1916
}
1917

1918
void kvm_vcpu_on_spin(struct kvm_vcpu *me)
Z
Zhai, Edwin 已提交
1919
{
1920 1921 1922 1923
	struct kvm *kvm = me->kvm;
	struct kvm_vcpu *vcpu;
	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
	int yielded = 0;
1924
	int try = 3;
1925 1926
	int pass;
	int i;
Z
Zhai, Edwin 已提交
1927

1928
	kvm_vcpu_set_in_spin_loop(me, true);
1929 1930 1931 1932 1933 1934 1935
	/*
	 * We boost the priority of a VCPU that is runnable but not
	 * currently running, because it got preempted by something
	 * else and called schedule in __vcpu_run.  Hopefully that
	 * VCPU is holding the lock that we need and will release it.
	 * We approximate round-robin by starting at the last boosted VCPU.
	 */
1936
	for (pass = 0; pass < 2 && !yielded && try; pass++) {
1937
		kvm_for_each_vcpu(i, vcpu, kvm) {
1938
			if (!pass && i <= last_boosted_vcpu) {
1939 1940 1941 1942
				i = last_boosted_vcpu;
				continue;
			} else if (pass && i > last_boosted_vcpu)
				break;
1943 1944
			if (!ACCESS_ONCE(vcpu->preempted))
				continue;
1945 1946
			if (vcpu == me)
				continue;
1947
			if (waitqueue_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
1948
				continue;
1949 1950
			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
				continue;
1951 1952 1953

			yielded = kvm_vcpu_yield_to(vcpu);
			if (yielded > 0) {
1954 1955
				kvm->last_boosted_vcpu = i;
				break;
1956 1957 1958 1959
			} else if (yielded < 0) {
				try--;
				if (!try)
					break;
1960 1961 1962
			}
		}
	}
1963
	kvm_vcpu_set_in_spin_loop(me, false);
1964 1965 1966

	/* Ensure vcpu is not eligible during next spinloop */
	kvm_vcpu_set_dy_eligible(me, false);
Z
Zhai, Edwin 已提交
1967 1968 1969
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

1970
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1971 1972 1973 1974
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

1975
	if (vmf->pgoff == 0)
1976
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
1977
#ifdef CONFIG_X86
1978
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1979
		page = virt_to_page(vcpu->arch.pio_data);
1980 1981 1982 1983
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
1984
#endif
1985
	else
1986
		return kvm_arch_vcpu_fault(vcpu, vmf);
1987
	get_page(page);
1988 1989
	vmf->page = page;
	return 0;
1990 1991
}

1992
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1993
	.fault = kvm_vcpu_fault,
1994 1995 1996 1997 1998 1999 2000 2001
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
2002 2003 2004 2005
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
2006
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
2007 2008 2009
	return 0;
}

2010
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
2011 2012
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
2013
#ifdef CONFIG_KVM_COMPAT
2014 2015
	.compat_ioctl   = kvm_vcpu_compat_ioctl,
#endif
2016
	.mmap           = kvm_vcpu_mmap,
2017
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
2018 2019 2020 2021 2022 2023 2024
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
2025
	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
A
Avi Kivity 已提交
2026 2027
}

2028 2029 2030
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
2031
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2032 2033
{
	int r;
2034
	struct kvm_vcpu *vcpu, *v;
2035

2036 2037 2038
	if (id >= KVM_MAX_VCPUS)
		return -EINVAL;

2039
	vcpu = kvm_arch_vcpu_create(kvm, id);
R
Rusty Russell 已提交
2040 2041
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
2042

2043 2044
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

2045 2046
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
2047
		goto vcpu_destroy;
2048

S
Shaohua Li 已提交
2049
	mutex_lock(&kvm->lock);
2050 2051 2052 2053
	if (!kvm_vcpu_compatible(vcpu)) {
		r = -EINVAL;
		goto unlock_vcpu_destroy;
	}
2054 2055
	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
		r = -EINVAL;
2056
		goto unlock_vcpu_destroy;
R
Rusty Russell 已提交
2057
	}
2058

2059 2060
	kvm_for_each_vcpu(r, v, kvm)
		if (v->vcpu_id == id) {
2061
			r = -EEXIST;
2062
			goto unlock_vcpu_destroy;
2063 2064 2065
		}

	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2066

R
Rusty Russell 已提交
2067
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
2068
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
2069
	r = create_vcpu_fd(vcpu);
2070 2071
	if (r < 0) {
		kvm_put_kvm(kvm);
2072
		goto unlock_vcpu_destroy;
2073 2074 2075 2076 2077 2078 2079
	}

	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

	mutex_unlock(&kvm->lock);
2080
	kvm_arch_vcpu_postcreate(vcpu);
R
Rusty Russell 已提交
2081
	return r;
2082

2083
unlock_vcpu_destroy:
2084
	mutex_unlock(&kvm->lock);
2085
vcpu_destroy:
2086
	kvm_arch_vcpu_destroy(vcpu);
2087 2088 2089
	return r;
}

A
Avi Kivity 已提交
2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
2101 2102
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
2103
{
A
Avi Kivity 已提交
2104
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
2105
	void __user *argp = (void __user *)arg;
2106
	int r;
2107 2108
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
2109

2110 2111
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
2112

2113 2114 2115
	if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
		return -EINVAL;

2116
#if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
2117 2118 2119 2120 2121 2122 2123 2124 2125
	/*
	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
	 * so vcpu_load() would break it.
	 */
	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
#endif


2126 2127 2128
	r = vcpu_load(vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
2129
	switch (ioctl) {
2130
	case KVM_RUN:
2131 2132 2133
		r = -EINVAL;
		if (arg)
			goto out;
2134 2135 2136 2137 2138 2139 2140 2141 2142
		if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
			/* The thread running this VCPU changed. */
			struct pid *oldpid = vcpu->pid;
			struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
			rcu_assign_pointer(vcpu->pid, newpid);
			if (oldpid)
				synchronize_rcu();
			put_pid(oldpid);
		}
2143
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2144
		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
A
Avi Kivity 已提交
2145 2146
		break;
	case KVM_GET_REGS: {
2147
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2148

2149 2150 2151
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
2152
			goto out;
2153 2154 2155
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
2156
		r = -EFAULT;
2157 2158
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
2159
		r = 0;
2160 2161
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
2162 2163 2164
		break;
	}
	case KVM_SET_REGS: {
2165
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2166

2167
		r = -ENOMEM;
2168 2169 2170
		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
		if (IS_ERR(kvm_regs)) {
			r = PTR_ERR(kvm_regs);
A
Avi Kivity 已提交
2171
			goto out;
2172
		}
2173 2174
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
		kfree(kvm_regs);
A
Avi Kivity 已提交
2175 2176 2177
		break;
	}
	case KVM_GET_SREGS: {
2178 2179 2180 2181 2182
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2183 2184 2185
		if (r)
			goto out;
		r = -EFAULT;
2186
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
2187 2188 2189 2190 2191
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
2192 2193 2194
		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
		if (IS_ERR(kvm_sregs)) {
			r = PTR_ERR(kvm_sregs);
G
Guo Chao 已提交
2195
			kvm_sregs = NULL;
A
Avi Kivity 已提交
2196
			goto out;
2197
		}
2198
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2199 2200
		break;
	}
2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		break;
	}
A
Avi Kivity 已提交
2222 2223 2224 2225
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
2226
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
2227
			goto out;
2228
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2229 2230 2231
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2232
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
2233 2234 2235 2236
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
2237 2238
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
2239 2240

		r = -EFAULT;
A
Al Viro 已提交
2241
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
2242
			goto out;
J
Jan Kiszka 已提交
2243
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
2244 2245
		break;
	}
A
Avi Kivity 已提交
2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
2266
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
A
Avi Kivity 已提交
2267 2268
		break;
	}
A
Avi Kivity 已提交
2269
	case KVM_GET_FPU: {
2270 2271 2272 2273 2274
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2275 2276 2277
		if (r)
			goto out;
		r = -EFAULT;
2278
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
2279 2280 2281 2282 2283
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
2284 2285 2286
		fpu = memdup_user(argp, sizeof(*fpu));
		if (IS_ERR(fpu)) {
			r = PTR_ERR(fpu);
G
Guo Chao 已提交
2287
			fpu = NULL;
A
Avi Kivity 已提交
2288
			goto out;
2289
		}
2290
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2291 2292
		break;
	}
A
Avi Kivity 已提交
2293
	default:
2294
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2295 2296
	}
out:
2297
	vcpu_put(vcpu);
2298 2299
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
2300 2301 2302
	return r;
}

2303
#ifdef CONFIG_KVM_COMPAT
2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332
static long kvm_vcpu_compat_ioctl(struct file *filp,
				  unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = compat_ptr(arg);
	int r;

	if (vcpu->kvm->mm != current->mm)
		return -EIO;

	switch (ioctl) {
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		compat_sigset_t csigset;
		sigset_t sigset;

		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof csigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&csigset, sigmask_arg->sigset,
					   sizeof csigset))
				goto out;
2333 2334 2335 2336
			sigset_from_compat(&sigset, &csigset);
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		} else
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347
		break;
	}
	default:
		r = kvm_vcpu_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

S
Scott Wood 已提交
2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394
static int kvm_device_ioctl_attr(struct kvm_device *dev,
				 int (*accessor)(struct kvm_device *dev,
						 struct kvm_device_attr *attr),
				 unsigned long arg)
{
	struct kvm_device_attr attr;

	if (!accessor)
		return -EPERM;

	if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
		return -EFAULT;

	return accessor(dev, &attr);
}

static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
			     unsigned long arg)
{
	struct kvm_device *dev = filp->private_data;

	switch (ioctl) {
	case KVM_SET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
	case KVM_GET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
	case KVM_HAS_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
	default:
		if (dev->ops->ioctl)
			return dev->ops->ioctl(dev, ioctl, arg);

		return -ENOTTY;
	}
}

static int kvm_device_release(struct inode *inode, struct file *filp)
{
	struct kvm_device *dev = filp->private_data;
	struct kvm *kvm = dev->kvm;

	kvm_put_kvm(kvm);
	return 0;
}

static const struct file_operations kvm_device_fops = {
	.unlocked_ioctl = kvm_device_ioctl,
2395
#ifdef CONFIG_KVM_COMPAT
2396 2397
	.compat_ioctl = kvm_device_ioctl,
#endif
S
Scott Wood 已提交
2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408
	.release = kvm_device_release,
};

struct kvm_device *kvm_device_from_filp(struct file *filp)
{
	if (filp->f_op != &kvm_device_fops)
		return NULL;

	return filp->private_data;
}

2409
static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2410
#ifdef CONFIG_KVM_MPIC
2411 2412
	[KVM_DEV_TYPE_FSL_MPIC_20]	= &kvm_mpic_ops,
	[KVM_DEV_TYPE_FSL_MPIC_42]	= &kvm_mpic_ops,
2413
#endif
2414

2415
#ifdef CONFIG_KVM_XICS
2416
	[KVM_DEV_TYPE_XICS]		= &kvm_xics_ops,
A
Alex Williamson 已提交
2417
#endif
2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431
};

int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
{
	if (type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENOSPC;

	if (kvm_device_ops_table[type] != NULL)
		return -EEXIST;

	kvm_device_ops_table[type] = ops;
	return 0;
}

2432 2433 2434 2435 2436 2437
void kvm_unregister_device_ops(u32 type)
{
	if (kvm_device_ops_table[type] != NULL)
		kvm_device_ops_table[type] = NULL;
}

S
Scott Wood 已提交
2438 2439 2440 2441 2442 2443 2444 2445
static int kvm_ioctl_create_device(struct kvm *kvm,
				   struct kvm_create_device *cd)
{
	struct kvm_device_ops *ops = NULL;
	struct kvm_device *dev;
	bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
	int ret;

2446 2447 2448 2449 2450
	if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
		return -ENODEV;

	ops = kvm_device_ops_table[cd->type];
	if (ops == NULL)
S
Scott Wood 已提交
2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468
		return -ENODEV;

	if (test)
		return 0;

	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
	if (!dev)
		return -ENOMEM;

	dev->ops = ops;
	dev->kvm = kvm;

	ret = ops->create(dev, cd->type);
	if (ret < 0) {
		kfree(dev);
		return ret;
	}

2469
	ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
S
Scott Wood 已提交
2470 2471 2472 2473 2474
	if (ret < 0) {
		ops->destroy(dev);
		return ret;
	}

2475
	list_add(&dev->vm_node, &kvm->devices);
S
Scott Wood 已提交
2476 2477 2478 2479 2480
	kvm_get_kvm(kvm);
	cd->fd = ret;
	return 0;
}

2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493
static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
{
	switch (arg) {
	case KVM_CAP_USER_MEMORY:
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_CAP_SET_BOOT_CPU_ID:
#endif
	case KVM_CAP_INTERNAL_ERROR_DATA:
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_CAP_SIGNAL_MSI:
#endif
2494
#ifdef CONFIG_HAVE_KVM_IRQFD
2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508
	case KVM_CAP_IRQFD_RESAMPLE:
#endif
	case KVM_CAP_CHECK_EXTENSION_VM:
		return 1;
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_CAP_IRQ_ROUTING:
		return KVM_MAX_IRQ_ROUTES;
#endif
	default:
		break;
	}
	return kvm_vm_ioctl_check_extension(kvm, arg);
}

A
Avi Kivity 已提交
2509 2510 2511 2512 2513
static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
2514
	int r;
A
Avi Kivity 已提交
2515

2516 2517
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
2518 2519 2520 2521
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		break;
2522 2523 2524 2525 2526 2527 2528 2529
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

2530
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
A
Avi Kivity 已提交
2531 2532 2533 2534 2535 2536
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
2537
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
2538
			goto out;
2539
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
2540 2541
		break;
	}
2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		break;
	}
#endif
G
Gregory Haskins 已提交
2560 2561 2562 2563 2564 2565
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
2566
		r = kvm_irqfd(kvm, &data);
G
Gregory Haskins 已提交
2567 2568
		break;
	}
G
Gregory Haskins 已提交
2569 2570 2571 2572 2573 2574 2575 2576 2577
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
2578 2579 2580
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
2581
		mutex_lock(&kvm->lock);
2582 2583 2584 2585
		if (atomic_read(&kvm->online_vcpus) != 0)
			r = -EBUSY;
		else
			kvm->bsp_vcpu_id = arg;
2586
		mutex_unlock(&kvm->lock);
2587
		break;
2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
#endif
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_SIGNAL_MSI: {
		struct kvm_msi msi;

		r = -EFAULT;
		if (copy_from_user(&msi, argp, sizeof msi))
			goto out;
		r = kvm_send_userspace_msi(kvm, &msi);
		break;
	}
2599 2600 2601 2602 2603 2604 2605 2606 2607 2608
#endif
#ifdef __KVM_HAVE_IRQ_LINE
	case KVM_IRQ_LINE_STATUS:
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
		if (copy_from_user(&irq_event, argp, sizeof irq_event))
			goto out;

2609 2610
		r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
					ioctl == KVM_IRQ_LINE_STATUS);
2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622
		if (r)
			goto out;

		r = -EFAULT;
		if (ioctl == KVM_IRQ_LINE_STATUS) {
			if (copy_to_user(argp, &irq_event, sizeof irq_event))
				goto out;
		}

		r = 0;
		break;
	}
2623
#endif
2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_SET_GSI_ROUTING: {
		struct kvm_irq_routing routing;
		struct kvm_irq_routing __user *urouting;
		struct kvm_irq_routing_entry *entries;

		r = -EFAULT;
		if (copy_from_user(&routing, argp, sizeof(routing)))
			goto out;
		r = -EINVAL;
		if (routing.nr >= KVM_MAX_IRQ_ROUTES)
			goto out;
		if (routing.flags)
			goto out;
		r = -ENOMEM;
		entries = vmalloc(routing.nr * sizeof(*entries));
		if (!entries)
			goto out;
		r = -EFAULT;
		urouting = argp;
		if (copy_from_user(entries, urouting->entries,
				   routing.nr * sizeof(*entries)))
			goto out_free_irq_routing;
		r = kvm_set_irq_routing(kvm, entries, routing.nr,
					routing.flags);
	out_free_irq_routing:
		vfree(entries);
		break;
	}
#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
S
Scott Wood 已提交
2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671
	case KVM_CREATE_DEVICE: {
		struct kvm_create_device cd;

		r = -EFAULT;
		if (copy_from_user(&cd, argp, sizeof(cd)))
			goto out;

		r = kvm_ioctl_create_device(kvm, &cd);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(argp, &cd, sizeof(cd)))
			goto out;

		r = 0;
		break;
	}
2672 2673 2674
	case KVM_CHECK_EXTENSION:
		r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
		break;
2675
	default:
2676
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2677 2678 2679 2680 2681
	}
out:
	return r;
}

2682
#ifdef CONFIG_KVM_COMPAT
2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		r = -EFAULT;
		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
			goto out;
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

2726
static struct file_operations kvm_vm_fops = {
2727 2728
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
2729
#ifdef CONFIG_KVM_COMPAT
2730 2731
	.compat_ioctl   = kvm_vm_compat_ioctl,
#endif
2732
	.llseek		= noop_llseek,
2733 2734
};

2735
static int kvm_dev_ioctl_create_vm(unsigned long type)
2736
{
2737
	int r;
2738 2739
	struct kvm *kvm;

2740
	kvm = kvm_create_vm(type);
2741 2742
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
2743 2744 2745 2746 2747 2748 2749
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	r = kvm_coalesced_mmio_init(kvm);
	if (r < 0) {
		kvm_put_kvm(kvm);
		return r;
	}
#endif
2750
	r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC);
2751
	if (r < 0)
A
Al Viro 已提交
2752
		kvm_put_kvm(kvm);
2753

2754
	return r;
2755 2756 2757 2758 2759
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
2760
	long r = -EINVAL;
2761 2762 2763

	switch (ioctl) {
	case KVM_GET_API_VERSION:
2764 2765
		if (arg)
			goto out;
2766 2767 2768
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
2769
		r = kvm_dev_ioctl_create_vm(arg);
2770
		break;
2771
	case KVM_CHECK_EXTENSION:
2772
		r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
2773
		break;
2774 2775 2776
	case KVM_GET_VCPU_MMAP_SIZE:
		if (arg)
			goto out;
2777 2778 2779
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
2780 2781 2782
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
		r += PAGE_SIZE;    /* coalesced mmio ring page */
2783
#endif
2784
		break;
2785 2786 2787
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
2788
		r = -EOPNOTSUPP;
2789
		break;
A
Avi Kivity 已提交
2790
	default:
2791
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2792 2793 2794 2795 2796 2797 2798 2799
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
2800
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
2801 2802 2803
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
2804
	KVM_MINOR,
A
Avi Kivity 已提交
2805 2806 2807 2808
	"kvm",
	&kvm_chardev_ops,
};

2809
static void hardware_enable_nolock(void *junk)
2810 2811
{
	int cpu = raw_smp_processor_id();
2812
	int r;
2813

2814
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2815
		return;
2816

2817
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
2818

2819
	r = kvm_arch_hardware_enable();
2820 2821 2822 2823 2824 2825 2826

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
		printk(KERN_INFO "kvm: enabling virtualization on "
				 "CPU%d failed\n", cpu);
	}
2827 2828
}

2829
static void hardware_enable(void)
2830
{
2831
	raw_spin_lock(&kvm_count_lock);
2832 2833
	if (kvm_usage_count)
		hardware_enable_nolock(NULL);
2834
	raw_spin_unlock(&kvm_count_lock);
2835 2836 2837
}

static void hardware_disable_nolock(void *junk)
2838 2839 2840
{
	int cpu = raw_smp_processor_id();

2841
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2842
		return;
2843
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2844
	kvm_arch_hardware_disable();
2845 2846
}

2847
static void hardware_disable(void)
2848
{
2849
	raw_spin_lock(&kvm_count_lock);
2850 2851
	if (kvm_usage_count)
		hardware_disable_nolock(NULL);
2852
	raw_spin_unlock(&kvm_count_lock);
2853 2854
}

2855 2856 2857 2858 2859 2860
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
2861
		on_each_cpu(hardware_disable_nolock, NULL, 1);
2862 2863 2864 2865
}

static void hardware_disable_all(void)
{
2866
	raw_spin_lock(&kvm_count_lock);
2867
	hardware_disable_all_nolock();
2868
	raw_spin_unlock(&kvm_count_lock);
2869 2870 2871 2872 2873 2874
}

static int hardware_enable_all(void)
{
	int r = 0;

2875
	raw_spin_lock(&kvm_count_lock);
2876 2877 2878 2879

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
2880
		on_each_cpu(hardware_enable_nolock, NULL, 1);
2881 2882 2883 2884 2885 2886 2887

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

2888
	raw_spin_unlock(&kvm_count_lock);
2889 2890 2891 2892

	return r;
}

A
Avi Kivity 已提交
2893 2894 2895 2896 2897
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

2898
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
2899
	switch (val) {
2900
	case CPU_DYING:
2901 2902
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
2903
		hardware_disable();
2904
		break;
2905
	case CPU_STARTING:
2906 2907
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
2908
		hardware_enable();
A
Avi Kivity 已提交
2909 2910 2911 2912 2913
		break;
	}
	return NOTIFY_OK;
}

2914
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
2915
		      void *v)
2916
{
2917 2918 2919 2920 2921 2922 2923 2924
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
	kvm_rebooting = true;
2925
	on_each_cpu(hardware_disable_nolock, NULL, 1);
2926 2927 2928 2929 2930 2931 2932 2933
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
2934
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2935 2936 2937 2938
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
2939
		struct kvm_io_device *pos = bus->range[i].dev;
2940 2941 2942

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
2943
	kfree(bus);
2944 2945
}

2946 2947
static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
                                 const struct kvm_io_range *r2)
2948 2949 2950 2951 2952 2953 2954 2955
{
	if (r1->addr < r2->addr)
		return -1;
	if (r1->addr + r1->len > r2->addr + r2->len)
		return 1;
	return 0;
}

2956 2957
static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
{
2958
	return kvm_io_bus_cmp(p1, p2);
2959 2960
}

G
Geoff Levand 已提交
2961
static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975
			  gpa_t addr, int len)
{
	bus->range[bus->dev_count++] = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
		.dev = dev,
	};

	sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
		kvm_io_bus_sort_cmp, NULL);

	return 0;
}

G
Geoff Levand 已提交
2976
static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993
			     gpa_t addr, int len)
{
	struct kvm_io_range *range, key;
	int off;

	key = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	range = bsearch(&key, bus->range, bus->dev_count,
			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
	if (range == NULL)
		return -ENOENT;

	off = range - bus->range;

2994
	while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
2995 2996 2997 2998 2999
		off--;

	return off;
}

C
Cornelia Huck 已提交
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009
static int __kvm_io_bus_write(struct kvm_io_bus *bus,
			      struct kvm_io_range *range, const void *val)
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
3010
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
C
Cornelia Huck 已提交
3011 3012 3013 3014 3015 3016 3017 3018 3019
		if (!kvm_iodevice_write(bus->range[idx].dev, range->addr,
					range->len, val))
			return idx;
		idx++;
	}

	return -EOPNOTSUPP;
}

3020
/* kvm_io_bus_write - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
3021
int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3022
		     int len, const void *val)
3023
{
3024
	struct kvm_io_bus *bus;
3025
	struct kvm_io_range range;
C
Cornelia Huck 已提交
3026
	int r;
3027 3028 3029 3030 3031

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
3032 3033

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
C
Cornelia Huck 已提交
3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053
	r = __kvm_io_bus_write(bus, &range, val);
	return r < 0 ? r : 0;
}

/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, const void *val, long cookie)
{
	struct kvm_io_bus *bus;
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);

	/* First try the device referenced by cookie. */
	if ((cookie >= 0) && (cookie < bus->dev_count) &&
3054
	    (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
C
Cornelia Huck 已提交
3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071
		if (!kvm_iodevice_write(bus->range[cookie].dev, addr, len,
					val))
			return cookie;

	/*
	 * cookie contained garbage; fall back to search and return the
	 * correct cookie value.
	 */
	return __kvm_io_bus_write(bus, &range, val);
}

static int __kvm_io_bus_read(struct kvm_io_bus *bus, struct kvm_io_range *range,
			     void *val)
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3072 3073 3074 3075
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
3076
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
C
Cornelia Huck 已提交
3077 3078 3079
		if (!kvm_iodevice_read(bus->range[idx].dev, range->addr,
				       range->len, val))
			return idx;
3080 3081 3082
		idx++;
	}

3083 3084
	return -EOPNOTSUPP;
}
3085
EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3086

3087
/* kvm_io_bus_read - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
3088 3089
int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
		    int len, void *val)
3090
{
3091
	struct kvm_io_bus *bus;
3092
	struct kvm_io_range range;
C
Cornelia Huck 已提交
3093
	int r;
3094 3095 3096 3097 3098

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
M
Marcelo Tosatti 已提交
3099

3100
	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
C
Cornelia Huck 已提交
3101 3102 3103
	r = __kvm_io_bus_read(bus, &range, val);
	return r < 0 ? r : 0;
}
3104

3105

3106
/* Caller must hold slots_lock. */
3107 3108
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, struct kvm_io_device *dev)
3109
{
M
Marcelo Tosatti 已提交
3110
	struct kvm_io_bus *new_bus, *bus;
3111

M
Marcelo Tosatti 已提交
3112
	bus = kvm->buses[bus_idx];
3113 3114
	/* exclude ioeventfd which is limited by maximum fd */
	if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3115
		return -ENOSPC;
3116

3117 3118
	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
M
Marcelo Tosatti 已提交
3119 3120
	if (!new_bus)
		return -ENOMEM;
3121 3122
	memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
	       sizeof(struct kvm_io_range)));
3123
	kvm_io_bus_insert_dev(new_bus, dev, addr, len);
M
Marcelo Tosatti 已提交
3124 3125 3126
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
3127 3128 3129 3130

	return 0;
}

3131
/* Caller must hold slots_lock. */
M
Marcelo Tosatti 已提交
3132 3133
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			      struct kvm_io_device *dev)
3134
{
M
Marcelo Tosatti 已提交
3135 3136
	int i, r;
	struct kvm_io_bus *new_bus, *bus;
3137

3138
	bus = kvm->buses[bus_idx];
M
Marcelo Tosatti 已提交
3139
	r = -ENOENT;
3140 3141
	for (i = 0; i < bus->dev_count; i++)
		if (bus->range[i].dev == dev) {
M
Marcelo Tosatti 已提交
3142
			r = 0;
3143 3144
			break;
		}
M
Marcelo Tosatti 已提交
3145

3146
	if (r)
M
Marcelo Tosatti 已提交
3147
		return r;
3148 3149 3150 3151 3152 3153 3154 3155 3156 3157

	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
	if (!new_bus)
		return -ENOMEM;

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count--;
	memcpy(new_bus->range + i, bus->range + i + 1,
	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
3158 3159 3160 3161 3162

	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
	return r;
3163 3164
}

A
Avi Kivity 已提交
3165 3166 3167 3168
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
};

3169
static int vm_stat_get(void *_offset, u64 *val)
3170 3171 3172 3173
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

3174
	*val = 0;
3175
	spin_lock(&kvm_lock);
3176
	list_for_each_entry(kvm, &vm_list, vm_list)
3177
		*val += *(u32 *)((void *)kvm + offset);
3178
	spin_unlock(&kvm_lock);
3179
	return 0;
3180 3181 3182 3183
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

3184
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
3185 3186 3187 3188 3189 3190
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

3191
	*val = 0;
3192
	spin_lock(&kvm_lock);
A
Avi Kivity 已提交
3193
	list_for_each_entry(kvm, &vm_list, vm_list)
3194 3195 3196
		kvm_for_each_vcpu(i, vcpu, kvm)
			*val += *(u32 *)((void *)vcpu + offset);

3197
	spin_unlock(&kvm_lock);
3198
	return 0;
A
Avi Kivity 已提交
3199 3200
}

3201 3202
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

3203
static const struct file_operations *stat_fops[] = {
3204 3205 3206
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
3207

3208
static int kvm_init_debug(void)
A
Avi Kivity 已提交
3209
{
3210
	int r = -EEXIST;
A
Avi Kivity 已提交
3211 3212
	struct kvm_stats_debugfs_item *p;

3213
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3214 3215 3216 3217
	if (kvm_debugfs_dir == NULL)
		goto out;

	for (p = debugfs_entries; p->name; ++p) {
3218
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
3219
						(void *)(long)p->offset,
3220
						stat_fops[p->kind]);
3221 3222 3223 3224 3225 3226 3227 3228 3229 3230
		if (p->dentry == NULL)
			goto out_dir;
	}

	return 0;

out_dir:
	debugfs_remove_recursive(kvm_debugfs_dir);
out:
	return r;
A
Avi Kivity 已提交
3231 3232 3233 3234 3235 3236 3237 3238
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
3239
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
3240 3241
}

3242
static int kvm_suspend(void)
3243
{
3244
	if (kvm_usage_count)
3245
		hardware_disable_nolock(NULL);
3246 3247 3248
	return 0;
}

3249
static void kvm_resume(void)
3250
{
3251
	if (kvm_usage_count) {
3252
		WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3253
		hardware_enable_nolock(NULL);
3254
	}
3255 3256
}

3257
static struct syscore_ops kvm_syscore_ops = {
3258 3259 3260 3261
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

3262 3263 3264 3265 3266 3267 3268 3269 3270
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3271 3272
	if (vcpu->preempted)
		vcpu->preempted = false;
3273

R
Radim Krčmář 已提交
3274 3275
	kvm_arch_sched_in(vcpu, cpu);

3276
	kvm_arch_vcpu_load(vcpu, cpu);
3277 3278 3279 3280 3281 3282 3283
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

3284 3285
	if (current->state == TASK_RUNNING)
		vcpu->preempted = true;
3286
	kvm_arch_vcpu_put(vcpu);
3287 3288
}

3289
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3290
		  struct module *module)
A
Avi Kivity 已提交
3291 3292
{
	int r;
Y
Yang, Sheng 已提交
3293
	int cpu;
A
Avi Kivity 已提交
3294

3295 3296
	r = kvm_arch_init(opaque);
	if (r)
3297
		goto out_fail;
3298

3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309
	/*
	 * kvm_arch_init makes sure there's at most one caller
	 * for architectures that support multiple implementations,
	 * like intel and amd on x86.
	 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
	 * conflicts in case kvm is already setup for another implementation.
	 */
	r = kvm_irqfd_init();
	if (r)
		goto out_irqfd;

3310
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
3311 3312 3313 3314
		r = -ENOMEM;
		goto out_free_0;
	}

3315
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
3316
	if (r < 0)
3317
		goto out_free_0a;
A
Avi Kivity 已提交
3318

Y
Yang, Sheng 已提交
3319 3320
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
3321
				kvm_arch_check_processor_compat,
3322
				&r, 1);
Y
Yang, Sheng 已提交
3323
		if (r < 0)
3324
			goto out_free_1;
Y
Yang, Sheng 已提交
3325 3326
	}

A
Avi Kivity 已提交
3327 3328
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
3329
		goto out_free_2;
A
Avi Kivity 已提交
3330 3331
	register_reboot_notifier(&kvm_reboot_notifier);

3332
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
3333 3334 3335
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
J
Joe Perches 已提交
3336
					   0, NULL);
3337 3338
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
3339
		goto out_free_3;
3340 3341
	}

3342 3343 3344 3345
	r = kvm_async_pf_init();
	if (r)
		goto out_free;

A
Avi Kivity 已提交
3346
	kvm_chardev_ops.owner = module;
3347 3348
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
3349 3350 3351

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
3352
		printk(KERN_ERR "kvm: misc device register failed\n");
3353
		goto out_unreg;
A
Avi Kivity 已提交
3354 3355
	}

3356 3357
	register_syscore_ops(&kvm_syscore_ops);

3358 3359 3360
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

3361 3362 3363 3364 3365
	r = kvm_init_debug();
	if (r) {
		printk(KERN_ERR "kvm: create debugfs files failed\n");
		goto out_undebugfs;
	}
3366

P
Paolo Bonzini 已提交
3367 3368 3369
	r = kvm_vfio_ops_init();
	WARN_ON(r);

3370
	return 0;
A
Avi Kivity 已提交
3371

3372 3373
out_undebugfs:
	unregister_syscore_ops(&kvm_syscore_ops);
3374
	misc_deregister(&kvm_dev);
3375 3376
out_unreg:
	kvm_async_pf_deinit();
A
Avi Kivity 已提交
3377
out_free:
3378
	kmem_cache_destroy(kvm_vcpu_cache);
3379
out_free_3:
A
Avi Kivity 已提交
3380
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
3381
	unregister_cpu_notifier(&kvm_cpu_notifier);
3382 3383
out_free_2:
out_free_1:
3384
	kvm_arch_hardware_unsetup();
3385 3386
out_free_0a:
	free_cpumask_var(cpus_hardware_enabled);
3387
out_free_0:
3388 3389
	kvm_irqfd_exit();
out_irqfd:
3390 3391
	kvm_arch_exit();
out_fail:
A
Avi Kivity 已提交
3392 3393
	return r;
}
3394
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
3395

3396
void kvm_exit(void)
A
Avi Kivity 已提交
3397
{
3398
	kvm_exit_debug();
A
Avi Kivity 已提交
3399
	misc_deregister(&kvm_dev);
3400
	kmem_cache_destroy(kvm_vcpu_cache);
3401
	kvm_async_pf_deinit();
3402
	unregister_syscore_ops(&kvm_syscore_ops);
A
Avi Kivity 已提交
3403
	unregister_reboot_notifier(&kvm_reboot_notifier);
3404
	unregister_cpu_notifier(&kvm_cpu_notifier);
3405
	on_each_cpu(hardware_disable_nolock, NULL, 1);
3406
	kvm_arch_hardware_unsetup();
3407
	kvm_arch_exit();
3408
	kvm_irqfd_exit();
3409
	free_cpumask_var(cpus_hardware_enabled);
3410
	kvm_vfio_ops_exit();
A
Avi Kivity 已提交
3411
}
3412
EXPORT_SYMBOL_GPL(kvm_exit);