kvm_main.c 73.5 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "iodev.h"
A
Avi Kivity 已提交
20

21
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/syscore_ops.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
35
#include <linux/sched.h>
36 37
#include <linux/cpumask.h>
#include <linux/smp.h>
38
#include <linux/anon_inodes.h>
39
#include <linux/profile.h>
40
#include <linux/kvm_para.h>
41
#include <linux/pagemap.h>
42
#include <linux/mman.h>
43
#include <linux/swap.h>
44
#include <linux/bitops.h>
45
#include <linux/spinlock.h>
46
#include <linux/compat.h>
47
#include <linux/srcu.h>
J
Joerg Roedel 已提交
48
#include <linux/hugetlb.h>
49
#include <linux/slab.h>
50 51
#include <linux/sort.h>
#include <linux/bsearch.h>
A
Avi Kivity 已提交
52

A
Avi Kivity 已提交
53 54 55
#include <asm/processor.h>
#include <asm/io.h>
#include <asm/uaccess.h>
56
#include <asm/pgtable.h>
A
Avi Kivity 已提交
57

58
#include "coalesced_mmio.h"
59
#include "async_pf.h"
60

61 62 63
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

A
Avi Kivity 已提交
64 65 66
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

67 68 69
/*
 * Ordering of locks:
 *
70
 * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
71 72
 */

73
DEFINE_SPINLOCK(kvm_lock);
74
static DEFINE_RAW_SPINLOCK(kvm_count_lock);
75
LIST_HEAD(vm_list);
76

77
static cpumask_var_t cpus_hardware_enabled;
78 79
static int kvm_usage_count = 0;
static atomic_t hardware_enable_failed;
80

81 82
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
83

84 85
static __read_mostly struct preempt_ops kvm_preempt_ops;

86
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
87

A
Avi Kivity 已提交
88 89
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
90 91 92 93
#ifdef CONFIG_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
				  unsigned long arg);
#endif
94 95
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
96

M
Marcelo Tosatti 已提交
97 98
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);

99 100
bool kvm_rebooting;
EXPORT_SYMBOL_GPL(kvm_rebooting);
101

102 103
static bool largepages_enabled = true;

X
Xiao Guangrong 已提交
104
bool kvm_is_mmio_pfn(pfn_t pfn)
B
Ben-Ami Yassour 已提交
105
{
106 107
	if (pfn_valid(pfn))
		return PageReserved(pfn_to_page(pfn));
B
Ben-Ami Yassour 已提交
108 109 110 111

	return true;
}

A
Avi Kivity 已提交
112 113 114
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
115
int vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
116
{
117 118
	int cpu;

119 120
	if (mutex_lock_killable(&vcpu->mutex))
		return -EINTR;
121 122 123 124 125 126 127 128
	if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
		/* The thread running this VCPU changed. */
		struct pid *oldpid = vcpu->pid;
		struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
		rcu_assign_pointer(vcpu->pid, newpid);
		synchronize_rcu();
		put_pid(oldpid);
	}
129 130
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
131
	kvm_arch_vcpu_load(vcpu, cpu);
132
	put_cpu();
133
	return 0;
A
Avi Kivity 已提交
134 135
}

136
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
137
{
138
	preempt_disable();
139
	kvm_arch_vcpu_put(vcpu);
140 141
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
142 143 144
	mutex_unlock(&vcpu->mutex);
}

145 146 147 148
static void ack_flush(void *_completed)
{
}

149
static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
150
{
151
	int i, cpu, me;
152 153
	cpumask_var_t cpus;
	bool called = true;
154 155
	struct kvm_vcpu *vcpu;

156
	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
157

158
	me = get_cpu();
159
	kvm_for_each_vcpu(i, vcpu, kvm) {
160
		kvm_make_request(req, vcpu);
161
		cpu = vcpu->cpu;
162 163 164 165 166 167

		/* Set ->requests bit before we read ->mode */
		smp_mb();

		if (cpus != NULL && cpu != -1 && cpu != me &&
		      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
168
			cpumask_set_cpu(cpu, cpus);
169
	}
170 171 172 173 174 175
	if (unlikely(cpus == NULL))
		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
	else if (!cpumask_empty(cpus))
		smp_call_function_many(cpus, ack_flush, NULL, 1);
	else
		called = false;
176
	put_cpu();
177
	free_cpumask_var(cpus);
178
	return called;
179 180
}

181
void kvm_flush_remote_tlbs(struct kvm *kvm)
182
{
183
	long dirty_count = kvm->tlbs_dirty;
184 185

	smp_mb();
186 187
	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
		++kvm->stat.remote_tlb_flush;
188
	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
189
}
190
EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
191

192 193 194 195
void kvm_reload_remote_mmus(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
}
196

197 198 199 200 201
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}

202
void kvm_make_scan_ioapic_request(struct kvm *kvm)
203
{
204
	make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
205 206
}

R
Rusty Russell 已提交
207 208 209 210 211 212 213 214 215
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
216
	vcpu->pid = NULL;
E
Eddie Dong 已提交
217
	init_waitqueue_head(&vcpu->wq);
218
	kvm_async_pf_vcpu_init(vcpu);
R
Rusty Russell 已提交
219 220 221 222 223 224 225 226

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

227 228
	kvm_vcpu_set_in_spin_loop(vcpu, false);
	kvm_vcpu_set_dy_eligible(vcpu, false);
229
	vcpu->preempted = false;
230

231
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
232
	if (r < 0)
233
		goto fail_free_run;
R
Rusty Russell 已提交
234 235 236 237 238
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
239
	return r;
R
Rusty Russell 已提交
240 241 242 243 244
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
245
	put_pid(vcpu->pid);
246
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
247 248 249 250
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

251 252 253 254 255 256 257 258 259 260 261
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
262
	int need_tlb_flush, idx;
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
282
	idx = srcu_read_lock(&kvm->srcu);
283
	spin_lock(&kvm->mmu_lock);
284

285
	kvm->mmu_notifier_seq++;
286
	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
287 288 289 290
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

291 292
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
293 294
}

295 296 297 298 299 300
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
301
	int idx;
302

303
	idx = srcu_read_lock(&kvm->srcu);
304 305 306 307
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	kvm_set_spte_hva(kvm, address, pte);
	spin_unlock(&kvm->mmu_lock);
308
	srcu_read_unlock(&kvm->srcu, idx);
309 310
}

311 312 313 314 315 316
static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
317
	int need_tlb_flush = 0, idx;
318

319
	idx = srcu_read_lock(&kvm->srcu);
320 321 322 323 324 325 326
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
327
	need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
328
	need_tlb_flush |= kvm->tlbs_dirty;
329 330 331
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
332 333 334

	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
351
	smp_wmb();
352 353
	/*
	 * The above sequence increase must be visible before the
354 355
	 * below count decrease, which is ensured by the smp_wmb above
	 * in conjunction with the smp_rmb in mmu_notifier_retry().
356 357 358 359 360 361 362 363 364 365 366 367
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
					      unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
368
	int young, idx;
369

370
	idx = srcu_read_lock(&kvm->srcu);
371 372
	spin_lock(&kvm->mmu_lock);

373
	young = kvm_age_hva(kvm, address);
374 375 376
	if (young)
		kvm_flush_remote_tlbs(kvm);

377 378 379
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

380 381 382
	return young;
}

A
Andrea Arcangeli 已提交
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
				       struct mm_struct *mm,
				       unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	young = kvm_test_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

399 400 401 402
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
403 404 405
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
406
	kvm_arch_flush_shadow_all(kvm);
407
	srcu_read_unlock(&kvm->srcu, idx);
408 409
}

410 411 412 413 414
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
A
Andrea Arcangeli 已提交
415
	.test_young		= kvm_mmu_notifier_test_young,
416
	.change_pte		= kvm_mmu_notifier_change_pte,
417
	.release		= kvm_mmu_notifier_release,
418
};
419 420 421 422 423 424 425 426 427 428 429 430 431 432

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

433 434
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

435 436 437 438 439 440
static void kvm_init_memslots_id(struct kvm *kvm)
{
	int i;
	struct kvm_memslots *slots = kvm->memslots;

	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
441
		slots->id_to_index[i] = slots->memslots[i].id = i;
442 443
}

444
static struct kvm *kvm_create_vm(unsigned long type)
A
Avi Kivity 已提交
445
{
446 447
	int r, i;
	struct kvm *kvm = kvm_arch_alloc_vm();
A
Avi Kivity 已提交
448

449 450 451
	if (!kvm)
		return ERR_PTR(-ENOMEM);

452
	r = kvm_arch_init_vm(kvm, type);
453 454
	if (r)
		goto out_err_nodisable;
455 456 457 458 459

	r = hardware_enable_all();
	if (r)
		goto out_err_nodisable;

460 461
#ifdef CONFIG_HAVE_KVM_IRQCHIP
	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
462
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
463
#endif
A
Avi Kivity 已提交
464

465 466
	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);

467 468 469
	r = -ENOMEM;
	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
	if (!kvm->memslots)
470
		goto out_err_nosrcu;
471
	kvm_init_memslots_id(kvm);
472
	if (init_srcu_struct(&kvm->srcu))
473
		goto out_err_nosrcu;
M
Marcelo Tosatti 已提交
474 475 476
	for (i = 0; i < KVM_NR_BUSES; i++) {
		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
					GFP_KERNEL);
477
		if (!kvm->buses[i])
M
Marcelo Tosatti 已提交
478 479
			goto out_err;
	}
480

481
	spin_lock_init(&kvm->mmu_lock);
482 483
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
G
Gregory Haskins 已提交
484
	kvm_eventfd_init(kvm);
S
Shaohua Li 已提交
485
	mutex_init(&kvm->lock);
486
	mutex_init(&kvm->irq_lock);
487
	mutex_init(&kvm->slots_lock);
I
Izik Eidus 已提交
488
	atomic_set(&kvm->users_count, 1);
489
	INIT_LIST_HEAD(&kvm->devices);
490 491 492 493 494

	r = kvm_init_mmu_notifier(kvm);
	if (r)
		goto out_err;

495
	spin_lock(&kvm_lock);
496
	list_add(&kvm->vm_list, &vm_list);
497
	spin_unlock(&kvm_lock);
498

499
	return kvm;
500 501

out_err:
502 503
	cleanup_srcu_struct(&kvm->srcu);
out_err_nosrcu:
504 505
	hardware_disable_all();
out_err_nodisable:
M
Marcelo Tosatti 已提交
506 507
	for (i = 0; i < KVM_NR_BUSES; i++)
		kfree(kvm->buses[i]);
508
	kfree(kvm->memslots);
509
	kvm_arch_free_vm(kvm);
510
	return ERR_PTR(r);
511 512
}

513 514 515 516
/*
 * Avoid using vmalloc for a small buffer.
 * Should not be used when the size is statically known.
 */
517
void *kvm_kvzalloc(unsigned long size)
518 519 520 521 522 523 524
{
	if (size > PAGE_SIZE)
		return vzalloc(size);
	else
		return kzalloc(size, GFP_KERNEL);
}

525
void kvm_kvfree(const void *addr)
526 527 528 529 530 531 532
{
	if (is_vmalloc_addr(addr))
		vfree(addr);
	else
		kfree(addr);
}

533 534 535 536 537
static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
	if (!memslot->dirty_bitmap)
		return;

538
	kvm_kvfree(memslot->dirty_bitmap);
539 540 541
	memslot->dirty_bitmap = NULL;
}

A
Avi Kivity 已提交
542 543 544
/*
 * Free any memory in @free but not in @dont.
 */
545
static void kvm_free_physmem_slot(struct kvm *kvm, struct kvm_memory_slot *free,
A
Avi Kivity 已提交
546 547 548
				  struct kvm_memory_slot *dont)
{
	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
549
		kvm_destroy_dirty_bitmap(free);
A
Avi Kivity 已提交
550

551
	kvm_arch_free_memslot(kvm, free, dont);
M
Marcelo Tosatti 已提交
552

A
Avi Kivity 已提交
553 554 555
	free->npages = 0;
}

556
void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
557
{
558
	struct kvm_memslots *slots = kvm->memslots;
559
	struct kvm_memory_slot *memslot;
560

561
	kvm_for_each_memslot(memslot, slots)
562
		kvm_free_physmem_slot(kvm, memslot, NULL);
A
Avi Kivity 已提交
563

564
	kfree(kvm->memslots);
A
Avi Kivity 已提交
565 566
}

567 568 569 570 571 572 573 574 575 576 577 578 579
static void kvm_destroy_devices(struct kvm *kvm)
{
	struct list_head *node, *tmp;

	list_for_each_safe(node, tmp, &kvm->devices) {
		struct kvm_device *dev =
			list_entry(node, struct kvm_device, vm_node);

		list_del(node);
		dev->ops->destroy(dev);
	}
}

580 581
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
582
	int i;
583 584
	struct mm_struct *mm = kvm->mm;

585
	kvm_arch_sync_events(kvm);
586
	spin_lock(&kvm_lock);
587
	list_del(&kvm->vm_list);
588
	spin_unlock(&kvm_lock);
589
	kvm_free_irq_routing(kvm);
M
Marcelo Tosatti 已提交
590 591
	for (i = 0; i < KVM_NR_BUSES; i++)
		kvm_io_bus_destroy(kvm->buses[i]);
592
	kvm_coalesced_mmio_free(kvm);
593 594
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
595
#else
596
	kvm_arch_flush_shadow_all(kvm);
597
#endif
598
	kvm_arch_destroy_vm(kvm);
599
	kvm_destroy_devices(kvm);
600 601 602
	kvm_free_physmem(kvm);
	cleanup_srcu_struct(&kvm->srcu);
	kvm_arch_free_vm(kvm);
603
	hardware_disable_all();
604
	mmdrop(mm);
605 606
}

I
Izik Eidus 已提交
607 608 609 610 611 612 613 614 615 616 617 618 619 620
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


621 622 623 624
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
625 626
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
627
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
628 629 630
	return 0;
}

631 632
/*
 * Allocation size is twice as large as the actual dirty bitmap size.
633
 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
634
 */
635 636
static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
{
637
#ifndef CONFIG_S390
638
	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
639

640
	memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
641 642 643
	if (!memslot->dirty_bitmap)
		return -ENOMEM;

644
#endif /* !CONFIG_S390 */
645 646 647
	return 0;
}

648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
static int cmp_memslot(const void *slot1, const void *slot2)
{
	struct kvm_memory_slot *s1, *s2;

	s1 = (struct kvm_memory_slot *)slot1;
	s2 = (struct kvm_memory_slot *)slot2;

	if (s1->npages < s2->npages)
		return 1;
	if (s1->npages > s2->npages)
		return -1;

	return 0;
}

/*
 * Sort the memslots base on its size, so the larger slots
 * will get better fit.
 */
static void sort_memslots(struct kvm_memslots *slots)
{
669 670
	int i;

671 672
	sort(slots->memslots, KVM_MEM_SLOTS_NUM,
	      sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
673 674 675

	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
		slots->id_to_index[slots->memslots[i].id] = i;
676 677
}

678 679
void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new,
		     u64 last_generation)
680 681 682
{
	if (new) {
		int id = new->id;
683
		struct kvm_memory_slot *old = id_to_memslot(slots, id);
684
		unsigned long npages = old->npages;
685

686
		*old = *new;
687 688
		if (new->npages != npages)
			sort_memslots(slots);
689 690
	}

691
	slots->generation = last_generation + 1;
692 693
}

694 695
static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
{
X
Xiao Guangrong 已提交
696 697 698 699 700 701 702
	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;

#ifdef KVM_CAP_READONLY_MEM
	valid_flags |= KVM_MEM_READONLY;
#endif

	if (mem->flags & ~valid_flags)
703 704 705 706 707
		return -EINVAL;

	return 0;
}

708 709 710 711 712 713 714 715
static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
		struct kvm_memslots *slots, struct kvm_memory_slot *new)
{
	struct kvm_memslots *old_memslots = kvm->memslots;

	update_memslots(slots, new, kvm->memslots->generation);
	rcu_assign_pointer(kvm->memslots, slots);
	synchronize_srcu_expedited(&kvm->srcu);
716 717 718 719

	kvm_arch_memslots_updated(kvm);

	return old_memslots;
720 721
}

A
Avi Kivity 已提交
722 723 724 725 726
/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
727
 *
728
 * Must be called holding mmap_sem for write.
A
Avi Kivity 已提交
729
 */
730
int __kvm_set_memory_region(struct kvm *kvm,
731
			    struct kvm_userspace_memory_region *mem)
A
Avi Kivity 已提交
732
{
733
	int r;
A
Avi Kivity 已提交
734
	gfn_t base_gfn;
735
	unsigned long npages;
736
	struct kvm_memory_slot *slot;
A
Avi Kivity 已提交
737
	struct kvm_memory_slot old, new;
738
	struct kvm_memslots *slots = NULL, *old_memslots;
739
	enum kvm_mr_change change;
A
Avi Kivity 已提交
740

741 742 743 744
	r = check_memory_region_flags(mem);
	if (r)
		goto out;

A
Avi Kivity 已提交
745 746 747 748 749 750
	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
751
	/* We can read the guest memory with __xxx_user() later on. */
752
	if ((mem->slot < KVM_USER_MEM_SLOTS) &&
753
	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
754 755 756
	     !access_ok(VERIFY_WRITE,
			(void __user *)(unsigned long)mem->userspace_addr,
			mem->memory_size)))
757
		goto out;
758
	if (mem->slot >= KVM_MEM_SLOTS_NUM)
A
Avi Kivity 已提交
759 760 761 762
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

763
	slot = id_to_memslot(kvm->memslots, mem->slot);
A
Avi Kivity 已提交
764 765 766
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

767 768 769 770
	r = -EINVAL;
	if (npages > KVM_MEM_MAX_NR_PAGES)
		goto out;

A
Avi Kivity 已提交
771 772 773
	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

774
	new = old = *slot;
A
Avi Kivity 已提交
775

776
	new.id = mem->slot;
A
Avi Kivity 已提交
777 778 779 780 781
	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	r = -EINVAL;
782 783 784 785 786
	if (npages) {
		if (!old.npages)
			change = KVM_MR_CREATE;
		else { /* Modify an existing slot. */
			if ((mem->userspace_addr != old.userspace_addr) ||
787 788
			    (npages != old.npages) ||
			    ((new.flags ^ old.flags) & KVM_MEM_READONLY))
789 790 791 792 793 794 795 796 797 798 799 800 801 802
				goto out;

			if (base_gfn != old.base_gfn)
				change = KVM_MR_MOVE;
			else if (new.flags != old.flags)
				change = KVM_MR_FLAGS_ONLY;
			else { /* Nothing to change. */
				r = 0;
				goto out;
			}
		}
	} else if (old.npages) {
		change = KVM_MR_DELETE;
	} else /* Modify a non-existent slot: disallowed. */
803
		goto out;
A
Avi Kivity 已提交
804

805
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
806 807 808
		/* Check for overlaps */
		r = -EEXIST;
		kvm_for_each_memslot(slot, kvm->memslots) {
809 810
			if ((slot->id >= KVM_USER_MEM_SLOTS) ||
			    (slot->id == mem->slot))
811 812 813 814 815
				continue;
			if (!((base_gfn + npages <= slot->base_gfn) ||
			      (base_gfn >= slot->base_gfn + slot->npages)))
				goto out;
		}
A
Avi Kivity 已提交
816 817 818 819
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
820
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
821 822

	r = -ENOMEM;
823
	if (change == KVM_MR_CREATE) {
824
		new.userspace_addr = mem->userspace_addr;
825

826
		if (kvm_arch_create_memslot(kvm, &new, npages))
827
			goto out_free;
A
Avi Kivity 已提交
828
	}
829

A
Avi Kivity 已提交
830 831
	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
832
		if (kvm_create_dirty_bitmap(&new) < 0)
833
			goto out_free;
A
Avi Kivity 已提交
834 835
	}

836
	if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
837
		r = -ENOMEM;
838 839
		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
				GFP_KERNEL);
840 841
		if (!slots)
			goto out_free;
842 843 844
		slot = id_to_memslot(slots, mem->slot);
		slot->flags |= KVM_MEMSLOT_INVALID;

845
		old_memslots = install_new_memslots(kvm, slots, NULL);
846

847 848
		/* slot was deleted or moved, clear iommu mapping */
		kvm_iommu_unmap_pages(kvm, &old);
849 850
		/* From this point no new shadow pages pointing to a deleted,
		 * or moved, memslot will be created.
851 852 853 854 855
		 *
		 * validation of sp->gfn happens in:
		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
		 * 	- kvm_is_visible_gfn (mmu_check_roots)
		 */
856
		kvm_arch_flush_shadow_memslot(kvm, slot);
857
		slots = old_memslots;
858
	}
859

860
	r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
861
	if (r)
862
		goto out_slots;
863

864
	r = -ENOMEM;
865 866 867 868 869 870 871 872 873 874 875
	/*
	 * We can re-use the old_memslots from above, the only difference
	 * from the currently installed memslots is the invalid flag.  This
	 * will get overwritten by update_memslots anyway.
	 */
	if (!slots) {
		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
				GFP_KERNEL);
		if (!slots)
			goto out_free;
	}
876 877

	/* actual memory is freed via old in kvm_free_physmem_slot below */
878
	if (change == KVM_MR_DELETE) {
879
		new.dirty_bitmap = NULL;
880
		memset(&new.arch, 0, sizeof(new.arch));
881 882
	}

883
	old_memslots = install_new_memslots(kvm, slots, &new);
884

885
	kvm_arch_commit_memory_region(kvm, mem, &old, change);
886

887
	kvm_free_physmem_slot(kvm, &old, &new);
888 889
	kfree(old_memslots);

890 891
	/*
	 * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
892 893 894 895 896 897
	 * un-mapped and re-mapped if their base changes.  Since base change
	 * unmapping is handled above with slot deletion, mapping alone is
	 * needed here.  Anything else the iommu might care about for existing
	 * slots (size changes, userspace addr changes and read-only flag
	 * changes) is disallowed above, so any other attribute changes getting
	 * here can be skipped.
898
	 */
899 900
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
		r = kvm_iommu_map_pages(kvm, &new);
901
		return r;
902 903
	}

A
Avi Kivity 已提交
904 905
	return 0;

906 907
out_slots:
	kfree(slots);
908
out_free:
909
	kvm_free_physmem_slot(kvm, &new, &old);
A
Avi Kivity 已提交
910 911
out:
	return r;
912
}
913 914 915
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
916
			  struct kvm_userspace_memory_region *mem)
917 918 919
{
	int r;

920
	mutex_lock(&kvm->slots_lock);
921
	r = __kvm_set_memory_region(kvm, mem);
922
	mutex_unlock(&kvm->slots_lock);
923 924
	return r;
}
925 926
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

927
int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
928
				   struct kvm_userspace_memory_region *mem)
929
{
930
	if (mem->slot >= KVM_USER_MEM_SLOTS)
931
		return -EINVAL;
932
	return kvm_set_memory_region(kvm, mem);
A
Avi Kivity 已提交
933 934
}

935 936
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
937 938 939
{
	struct kvm_memory_slot *memslot;
	int r, i;
940
	unsigned long n;
A
Avi Kivity 已提交
941 942 943
	unsigned long any = 0;

	r = -EINVAL;
944
	if (log->slot >= KVM_USER_MEM_SLOTS)
A
Avi Kivity 已提交
945 946
		goto out;

947
	memslot = id_to_memslot(kvm->memslots, log->slot);
A
Avi Kivity 已提交
948 949 950 951
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

952
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
953

954
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
955 956 957 958 959 960
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

961 962
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
963 964 965 966 967

	r = 0;
out:
	return r;
}
968
EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
A
Avi Kivity 已提交
969

970 971 972 973 974
bool kvm_largepages_enabled(void)
{
	return largepages_enabled;
}

975 976 977 978 979 980
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

981 982 983 984
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
A
Avi Kivity 已提交
985
EXPORT_SYMBOL_GPL(gfn_to_memslot);
A
Avi Kivity 已提交
986

987 988
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
989
	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
990

991
	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
992 993
	      memslot->flags & KVM_MEMSLOT_INVALID)
		return 0;
994

995
	return 1;
996 997 998
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

J
Joerg Roedel 已提交
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

X
Xiao Guangrong 已提交
1023 1024 1025 1026 1027 1028 1029
static bool memslot_is_readonly(struct kvm_memory_slot *slot)
{
	return slot->flags & KVM_MEM_READONLY;
}

static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				       gfn_t *nr_pages, bool write)
I
Izik Eidus 已提交
1030
{
1031
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
X
Xiao Guangrong 已提交
1032
		return KVM_HVA_ERR_BAD;
1033

X
Xiao Guangrong 已提交
1034 1035
	if (memslot_is_readonly(slot) && write)
		return KVM_HVA_ERR_RO_BAD;
1036 1037 1038 1039

	if (nr_pages)
		*nr_pages = slot->npages - (gfn - slot->base_gfn);

X
Xiao Guangrong 已提交
1040
	return __gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
1041
}
1042

X
Xiao Guangrong 已提交
1043 1044 1045 1046
static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				     gfn_t *nr_pages)
{
	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
I
Izik Eidus 已提交
1047
}
1048

X
Xiao Guangrong 已提交
1049 1050 1051 1052 1053 1054 1055
unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
				 gfn_t gfn)
{
	return gfn_to_hva_many(slot, gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);

1056 1057
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
1058
	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1059
}
1060
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1061

1062
/*
1063 1064
 * If writable is set to false, the hva returned by this function is only
 * allowed to be read.
1065
 */
1066
unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1067
{
1068
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1069 1070 1071
	unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);

	if (!kvm_is_error_hva(hva) && writable)
1072 1073
		*writable = !memslot_is_readonly(slot);

1074
	return hva;
1075 1076 1077
}

static int kvm_read_hva(void *data, void __user *hva, int len)
1078
{
1079 1080 1081 1082 1083 1084
	return __copy_from_user(data, hva, len);
}

static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
{
	return __copy_from_user_inatomic(data, hva, len);
1085 1086
}

G
Geoff Levand 已提交
1087
static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
	unsigned long start, int write, struct page **page)
{
	int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;

	if (write)
		flags |= FOLL_WRITE;

	return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
}

1098 1099 1100 1101 1102 1103 1104 1105 1106
static inline int check_user_page_hwpoison(unsigned long addr)
{
	int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;

	rc = __get_user_pages(current, current->mm, addr, 1,
			      flags, NULL, NULL, NULL);
	return rc == -EHWPOISON;
}

X
Xiao Guangrong 已提交
1107 1108 1109 1110 1111 1112
/*
 * The atomic path to get the writable pfn which will be stored in @pfn,
 * true indicates success, otherwise false is returned.
 */
static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
			    bool write_fault, bool *writable, pfn_t *pfn)
A
Avi Kivity 已提交
1113
{
1114
	struct page *page[1];
X
Xiao Guangrong 已提交
1115
	int npages;
A
Avi Kivity 已提交
1116

X
Xiao Guangrong 已提交
1117 1118
	if (!(async || atomic))
		return false;
1119

1120 1121 1122 1123 1124 1125 1126
	/*
	 * Fast pin a writable pfn only if it is a write fault request
	 * or the caller allows to map a writable pfn for a read fault
	 * request.
	 */
	if (!(write_fault || writable))
		return false;
1127

X
Xiao Guangrong 已提交
1128 1129 1130
	npages = __get_user_pages_fast(addr, 1, 1, page);
	if (npages == 1) {
		*pfn = page_to_pfn(page[0]);
1131

X
Xiao Guangrong 已提交
1132 1133 1134 1135
		if (writable)
			*writable = true;
		return true;
	}
1136

X
Xiao Guangrong 已提交
1137 1138
	return false;
}
1139

X
Xiao Guangrong 已提交
1140 1141 1142 1143 1144 1145 1146 1147 1148
/*
 * The slow path to get the pfn of the specified host virtual address,
 * 1 indicates success, -errno is returned if error is detected.
 */
static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
			   bool *writable, pfn_t *pfn)
{
	struct page *page[1];
	int npages = 0;
1149

X
Xiao Guangrong 已提交
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
	might_sleep();

	if (writable)
		*writable = write_fault;

	if (async) {
		down_read(&current->mm->mmap_sem);
		npages = get_user_page_nowait(current, current->mm,
					      addr, write_fault, page);
		up_read(&current->mm->mmap_sem);
	} else
		npages = get_user_pages_fast(addr, 1, write_fault,
					     page);
	if (npages != 1)
		return npages;

	/* map read fault as writable if possible */
1167
	if (unlikely(!write_fault) && writable) {
X
Xiao Guangrong 已提交
1168 1169 1170 1171 1172 1173 1174
		struct page *wpage[1];

		npages = __get_user_pages_fast(addr, 1, 1, wpage);
		if (npages == 1) {
			*writable = true;
			put_page(page[0]);
			page[0] = wpage[0];
1175
		}
X
Xiao Guangrong 已提交
1176 1177

		npages = 1;
1178
	}
X
Xiao Guangrong 已提交
1179 1180 1181
	*pfn = page_to_pfn(page[0]);
	return npages;
}
I
Izik Eidus 已提交
1182

X
Xiao Guangrong 已提交
1183 1184 1185 1186
static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
{
	if (unlikely(!(vma->vm_flags & VM_READ)))
		return false;
1187

X
Xiao Guangrong 已提交
1188 1189
	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
		return false;
1190

X
Xiao Guangrong 已提交
1191 1192
	return true;
}
1193

1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207
/*
 * Pin guest page in memory and return its pfn.
 * @addr: host virtual address which maps memory to the guest
 * @atomic: whether this function can sleep
 * @async: whether this function need to wait IO complete if the
 *         host page is not in the memory
 * @write_fault: whether we should get a writable host page
 * @writable: whether it allows to map a writable host page for !@write_fault
 *
 * The function will map a writable host page for these two cases:
 * 1): @write_fault = true
 * 2): @write_fault = false && @writable, @writable will tell the caller
 *     whether the mapping is writable.
 */
X
Xiao Guangrong 已提交
1208 1209 1210 1211 1212 1213
static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
			bool write_fault, bool *writable)
{
	struct vm_area_struct *vma;
	pfn_t pfn = 0;
	int npages;
1214

X
Xiao Guangrong 已提交
1215 1216
	/* we can do it either atomically or asynchronously, not both */
	BUG_ON(atomic && async);
1217

X
Xiao Guangrong 已提交
1218 1219 1220 1221 1222 1223 1224 1225 1226
	if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
		return pfn;

	if (atomic)
		return KVM_PFN_ERR_FAULT;

	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
	if (npages == 1)
		return pfn;
1227

X
Xiao Guangrong 已提交
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
	down_read(&current->mm->mmap_sem);
	if (npages == -EHWPOISON ||
	      (!async && check_user_page_hwpoison(addr))) {
		pfn = KVM_PFN_ERR_HWPOISON;
		goto exit;
	}

	vma = find_vma_intersection(current->mm, addr, addr + 1);

	if (vma == NULL)
		pfn = KVM_PFN_ERR_FAULT;
	else if ((vma->vm_flags & VM_PFNMAP)) {
		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
			vma->vm_pgoff;
		BUG_ON(!kvm_is_mmio_pfn(pfn));
	} else {
X
Xiao Guangrong 已提交
1244
		if (async && vma_is_valid(vma, write_fault))
X
Xiao Guangrong 已提交
1245 1246 1247 1248 1249
			*async = true;
		pfn = KVM_PFN_ERR_FAULT;
	}
exit:
	up_read(&current->mm->mmap_sem);
1250
	return pfn;
1251 1252
}

X
Xiao Guangrong 已提交
1253 1254 1255
static pfn_t
__gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
		     bool *async, bool write_fault, bool *writable)
1256
{
X
Xiao Guangrong 已提交
1257 1258 1259 1260 1261 1262
	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);

	if (addr == KVM_HVA_ERR_RO_BAD)
		return KVM_PFN_ERR_RO_FAULT;

	if (kvm_is_error_hva(addr))
1263
		return KVM_PFN_NOSLOT;
X
Xiao Guangrong 已提交
1264 1265 1266 1267 1268 1269 1270 1271 1272

	/* Do not map writable pfn in the readonly memslot. */
	if (writable && memslot_is_readonly(slot)) {
		*writable = false;
		writable = NULL;
	}

	return hva_to_pfn(addr, atomic, async, write_fault,
			  writable);
1273 1274
}

1275 1276
static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
			  bool write_fault, bool *writable)
1277
{
X
Xiao Guangrong 已提交
1278
	struct kvm_memory_slot *slot;
1279

1280 1281 1282
	if (async)
		*async = false;

X
Xiao Guangrong 已提交
1283
	slot = gfn_to_memslot(kvm, gfn);
1284

X
Xiao Guangrong 已提交
1285 1286
	return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
				    writable);
1287 1288 1289 1290
}

pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
{
1291
	return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1292 1293 1294
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);

1295 1296
pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
		       bool write_fault, bool *writable)
1297
{
1298
	return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1299 1300 1301
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_async);

1302 1303
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
{
1304
	return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1305
}
1306 1307
EXPORT_SYMBOL_GPL(gfn_to_pfn);

1308 1309 1310 1311 1312 1313 1314
pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
		      bool *writable)
{
	return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);

1315
pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1316
{
X
Xiao Guangrong 已提交
1317
	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1318 1319
}

1320
pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1321
{
X
Xiao Guangrong 已提交
1322
	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1323
}
1324
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1325

1326 1327 1328 1329 1330 1331
int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
								  int nr_pages)
{
	unsigned long addr;
	gfn_t entry;

1332
	addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
	if (kvm_is_error_hva(addr))
		return -1;

	if (entry < nr_pages)
		return 0;

	return __get_user_pages_fast(addr, nr_pages, 1, pages);
}
EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);

X
Xiao Guangrong 已提交
1343 1344
static struct page *kvm_pfn_to_page(pfn_t pfn)
{
1345
	if (is_error_noslot_pfn(pfn))
1346
		return KVM_ERR_PTR_BAD_PAGE;
X
Xiao Guangrong 已提交
1347

1348 1349
	if (kvm_is_mmio_pfn(pfn)) {
		WARN_ON(1);
1350
		return KVM_ERR_PTR_BAD_PAGE;
1351
	}
X
Xiao Guangrong 已提交
1352 1353 1354 1355

	return pfn_to_page(pfn);
}

1356 1357
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
1358 1359 1360 1361
	pfn_t pfn;

	pfn = gfn_to_pfn(kvm, gfn);

X
Xiao Guangrong 已提交
1362
	return kvm_pfn_to_page(pfn);
A
Avi Kivity 已提交
1363
}
1364

A
Avi Kivity 已提交
1365 1366
EXPORT_SYMBOL_GPL(gfn_to_page);

1367 1368
void kvm_release_page_clean(struct page *page)
{
1369 1370
	WARN_ON(is_error_page(page));

1371
	kvm_release_pfn_clean(page_to_pfn(page));
1372 1373 1374
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

1375 1376
void kvm_release_pfn_clean(pfn_t pfn)
{
1377
	if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn))
1378
		put_page(pfn_to_page(pfn));
1379 1380 1381
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1382
void kvm_release_page_dirty(struct page *page)
1383
{
X
Xiao Guangrong 已提交
1384 1385
	WARN_ON(is_error_page(page));

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

void kvm_release_pfn_dirty(pfn_t pfn)
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);

void kvm_set_page_dirty(struct page *page)
{
	kvm_set_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_set_page_dirty);

void kvm_set_pfn_dirty(pfn_t pfn)
{
1405
	if (!kvm_is_mmio_pfn(pfn)) {
1406 1407 1408 1409
		struct page *page = pfn_to_page(pfn);
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1410
}
1411 1412 1413 1414
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
1415
	if (!kvm_is_mmio_pfn(pfn))
1416
		mark_page_accessed(pfn_to_page(pfn));
1417 1418 1419 1420 1421
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
1422
	if (!kvm_is_mmio_pfn(pfn))
1423
		get_page(pfn_to_page(pfn));
1424 1425
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1426

1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
1438 1439
	int r;
	unsigned long addr;
1440

1441
	addr = gfn_to_hva_prot(kvm, gfn, NULL);
1442 1443
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1444
	r = kvm_read_hva(data, (void __user *)addr + offset, len);
1445
	if (r)
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1471 1472 1473 1474 1475 1476 1477 1478
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

1479
	addr = gfn_to_hva_prot(kvm, gfn, NULL);
1480 1481
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1482
	pagefault_disable();
1483
	r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
1484
	pagefault_enable();
1485 1486 1487 1488 1489 1490
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

1491 1492 1493
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
1494 1495
	int r;
	unsigned long addr;
1496

1497 1498 1499
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1500
	r = __copy_to_user((void __user *)addr + offset, data, len);
1501
	if (r)
1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

1528
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1529
			      gpa_t gpa, unsigned long len)
1530 1531 1532
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int offset = offset_in_page(gpa);
1533 1534 1535 1536
	gfn_t start_gfn = gpa >> PAGE_SHIFT;
	gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
	gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
	gfn_t nr_pages_avail;
1537 1538 1539

	ghc->gpa = gpa;
	ghc->generation = slots->generation;
1540 1541 1542 1543
	ghc->len = len;
	ghc->memslot = gfn_to_memslot(kvm, start_gfn);
	ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, &nr_pages_avail);
	if (!kvm_is_error_hva(ghc->hva) && nr_pages_avail >= nr_pages_needed) {
1544
		ghc->hva += offset;
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
	} else {
		/*
		 * If the requested region crosses two memslots, we still
		 * verify that the entire region is valid here.
		 */
		while (start_gfn <= end_gfn) {
			ghc->memslot = gfn_to_memslot(kvm, start_gfn);
			ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
						   &nr_pages_avail);
			if (kvm_is_error_hva(ghc->hva))
				return -EFAULT;
			start_gfn += nr_pages_avail;
		}
		/* Use the slow path for cross page reads and writes. */
		ghc->memslot = NULL;
	}
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);

int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

1571 1572
	BUG_ON(len > ghc->len);

1573
	if (slots->generation != ghc->generation)
1574 1575 1576 1577
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);

	if (unlikely(!ghc->memslot))
		return kvm_write_guest(kvm, ghc->gpa, data, len);
1578 1579 1580 1581

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

1582
	r = __copy_to_user((void __user *)ghc->hva, data, len);
1583 1584 1585 1586 1587 1588 1589 1590
	if (r)
		return -EFAULT;
	mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);

1591 1592 1593 1594 1595 1596
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

1597 1598
	BUG_ON(len > ghc->len);

1599
	if (slots->generation != ghc->generation)
1600 1601 1602 1603
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);

	if (unlikely(!ghc->memslot))
		return kvm_read_guest(kvm, ghc->gpa, data, len);
1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

	r = __copy_from_user(data, (void __user *)ghc->hva, len);
	if (r)
		return -EFAULT;

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_cached);

1616 1617
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
1618 1619 1620
	const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));

	return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

1643 1644
void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
			     gfn_t gfn)
A
Avi Kivity 已提交
1645
{
R
Rusty Russell 已提交
1646 1647
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1648

1649
		set_bit_le(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1650 1651 1652
	}
}

1653 1654 1655 1656 1657 1658 1659
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = gfn_to_memslot(kvm, gfn);
	mark_page_dirty_in_slot(kvm, memslot, gfn);
}
1660
EXPORT_SYMBOL_GPL(mark_page_dirty);
1661

E
Eddie Dong 已提交
1662 1663 1664
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1665
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1666
{
1667 1668 1669 1670 1671
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

1672
		if (kvm_arch_vcpu_runnable(vcpu)) {
1673
			kvm_make_request(KVM_REQ_UNHALT, vcpu);
1674
			break;
1675
		}
1676 1677
		if (kvm_cpu_has_pending_timer(vcpu))
			break;
1678 1679 1680
		if (signal_pending(current))
			break;

E
Eddie Dong 已提交
1681 1682
		schedule();
	}
1683

1684
	finish_wait(&vcpu->wq, &wait);
E
Eddie Dong 已提交
1685
}
1686
EXPORT_SYMBOL_GPL(kvm_vcpu_block);
E
Eddie Dong 已提交
1687

1688
#ifndef CONFIG_S390
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709
/*
 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
 */
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
	int me;
	int cpu = vcpu->cpu;
	wait_queue_head_t *wqp;

	wqp = kvm_arch_vcpu_wq(vcpu);
	if (waitqueue_active(wqp)) {
		wake_up_interruptible(wqp);
		++vcpu->stat.halt_wakeup;
	}

	me = get_cpu();
	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
		if (kvm_arch_vcpu_should_kick(vcpu))
			smp_send_reschedule(cpu);
	put_cpu();
}
1710
EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
1711
#endif /* !CONFIG_S390 */
1712

1713 1714 1715 1716
bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
{
	struct pid *pid;
	struct task_struct *task = NULL;
1717
	bool ret = false;
1718 1719 1720 1721 1722 1723 1724

	rcu_read_lock();
	pid = rcu_dereference(target->pid);
	if (pid)
		task = get_pid_task(target->pid, PIDTYPE_PID);
	rcu_read_unlock();
	if (!task)
1725
		return ret;
1726 1727
	if (task->flags & PF_VCPU) {
		put_task_struct(task);
1728
		return ret;
1729
	}
1730
	ret = yield_to(task, 1);
1731
	put_task_struct(task);
1732 1733

	return ret;
1734 1735 1736
}
EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);

1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
/*
 * Helper that checks whether a VCPU is eligible for directed yield.
 * Most eligible candidate to yield is decided by following heuristics:
 *
 *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
 *  (preempted lock holder), indicated by @in_spin_loop.
 *  Set at the beiginning and cleared at the end of interception/PLE handler.
 *
 *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
 *  chance last time (mostly it has become eligible now since we have probably
 *  yielded to lockholder in last iteration. This is done by toggling
 *  @dy_eligible each time a VCPU checked for eligibility.)
 *
 *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
 *  to preempted lock-holder could result in wrong VCPU selection and CPU
 *  burning. Giving priority for a potential lock-holder increases lock
 *  progress.
 *
 *  Since algorithm is based on heuristics, accessing another VCPU data without
 *  locking does not harm. It may result in trying to yield to  same VCPU, fail
 *  and continue with next VCPU and so on.
 */
bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
{
	bool eligible;

	eligible = !vcpu->spin_loop.in_spin_loop ||
			(vcpu->spin_loop.in_spin_loop &&
			 vcpu->spin_loop.dy_eligible);

	if (vcpu->spin_loop.in_spin_loop)
		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);

	return eligible;
}
#endif
1774

1775
void kvm_vcpu_on_spin(struct kvm_vcpu *me)
Z
Zhai, Edwin 已提交
1776
{
1777 1778 1779 1780
	struct kvm *kvm = me->kvm;
	struct kvm_vcpu *vcpu;
	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
	int yielded = 0;
1781
	int try = 3;
1782 1783
	int pass;
	int i;
Z
Zhai, Edwin 已提交
1784

1785
	kvm_vcpu_set_in_spin_loop(me, true);
1786 1787 1788 1789 1790 1791 1792
	/*
	 * We boost the priority of a VCPU that is runnable but not
	 * currently running, because it got preempted by something
	 * else and called schedule in __vcpu_run.  Hopefully that
	 * VCPU is holding the lock that we need and will release it.
	 * We approximate round-robin by starting at the last boosted VCPU.
	 */
1793
	for (pass = 0; pass < 2 && !yielded && try; pass++) {
1794
		kvm_for_each_vcpu(i, vcpu, kvm) {
1795
			if (!pass && i <= last_boosted_vcpu) {
1796 1797 1798 1799
				i = last_boosted_vcpu;
				continue;
			} else if (pass && i > last_boosted_vcpu)
				break;
1800 1801
			if (!ACCESS_ONCE(vcpu->preempted))
				continue;
1802 1803 1804 1805
			if (vcpu == me)
				continue;
			if (waitqueue_active(&vcpu->wq))
				continue;
1806 1807
			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
				continue;
1808 1809 1810

			yielded = kvm_vcpu_yield_to(vcpu);
			if (yielded > 0) {
1811 1812
				kvm->last_boosted_vcpu = i;
				break;
1813 1814 1815 1816
			} else if (yielded < 0) {
				try--;
				if (!try)
					break;
1817 1818 1819
			}
		}
	}
1820
	kvm_vcpu_set_in_spin_loop(me, false);
1821 1822 1823

	/* Ensure vcpu is not eligible during next spinloop */
	kvm_vcpu_set_dy_eligible(me, false);
Z
Zhai, Edwin 已提交
1824 1825 1826
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

1827
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1828 1829 1830 1831
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

1832
	if (vmf->pgoff == 0)
1833
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
1834
#ifdef CONFIG_X86
1835
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1836
		page = virt_to_page(vcpu->arch.pio_data);
1837 1838 1839 1840
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
1841
#endif
1842
	else
1843
		return kvm_arch_vcpu_fault(vcpu, vmf);
1844
	get_page(page);
1845 1846
	vmf->page = page;
	return 0;
1847 1848
}

1849
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1850
	.fault = kvm_vcpu_fault,
1851 1852 1853 1854 1855 1856 1857 1858
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
1859 1860 1861 1862
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
1863
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1864 1865 1866
	return 0;
}

1867
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
1868 1869
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
1870 1871 1872
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vcpu_compat_ioctl,
#endif
1873
	.mmap           = kvm_vcpu_mmap,
1874
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
1875 1876 1877 1878 1879 1880 1881
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
1882
	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
A
Avi Kivity 已提交
1883 1884
}

1885 1886 1887
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
1888
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1889 1890
{
	int r;
1891
	struct kvm_vcpu *vcpu, *v;
1892

1893
	vcpu = kvm_arch_vcpu_create(kvm, id);
R
Rusty Russell 已提交
1894 1895
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
1896

1897 1898
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

1899 1900
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
1901
		goto vcpu_destroy;
1902

S
Shaohua Li 已提交
1903
	mutex_lock(&kvm->lock);
1904 1905 1906 1907
	if (!kvm_vcpu_compatible(vcpu)) {
		r = -EINVAL;
		goto unlock_vcpu_destroy;
	}
1908 1909
	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
		r = -EINVAL;
1910
		goto unlock_vcpu_destroy;
R
Rusty Russell 已提交
1911
	}
1912

1913 1914
	kvm_for_each_vcpu(r, v, kvm)
		if (v->vcpu_id == id) {
1915
			r = -EEXIST;
1916
			goto unlock_vcpu_destroy;
1917 1918 1919
		}

	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1920

R
Rusty Russell 已提交
1921
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
1922
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
1923
	r = create_vcpu_fd(vcpu);
1924 1925
	if (r < 0) {
		kvm_put_kvm(kvm);
1926
		goto unlock_vcpu_destroy;
1927 1928 1929 1930 1931 1932 1933
	}

	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

	mutex_unlock(&kvm->lock);
1934
	kvm_arch_vcpu_postcreate(vcpu);
R
Rusty Russell 已提交
1935
	return r;
1936

1937
unlock_vcpu_destroy:
1938
	mutex_unlock(&kvm->lock);
1939
vcpu_destroy:
1940
	kvm_arch_vcpu_destroy(vcpu);
1941 1942 1943
	return r;
}

A
Avi Kivity 已提交
1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
1955 1956
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
1957
{
A
Avi Kivity 已提交
1958
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
1959
	void __user *argp = (void __user *)arg;
1960
	int r;
1961 1962
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
1963

1964 1965
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
1966

1967
#if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
1968 1969 1970 1971 1972 1973 1974 1975 1976
	/*
	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
	 * so vcpu_load() would break it.
	 */
	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
#endif


1977 1978 1979
	r = vcpu_load(vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
1980
	switch (ioctl) {
1981
	case KVM_RUN:
1982 1983 1984
		r = -EINVAL;
		if (arg)
			goto out;
1985
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1986
		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
A
Avi Kivity 已提交
1987 1988
		break;
	case KVM_GET_REGS: {
1989
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1990

1991 1992 1993
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1994
			goto out;
1995 1996 1997
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
1998
		r = -EFAULT;
1999 2000
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
2001
		r = 0;
2002 2003
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
2004 2005 2006
		break;
	}
	case KVM_SET_REGS: {
2007
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
2008

2009
		r = -ENOMEM;
2010 2011 2012
		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
		if (IS_ERR(kvm_regs)) {
			r = PTR_ERR(kvm_regs);
A
Avi Kivity 已提交
2013
			goto out;
2014
		}
2015 2016
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
		kfree(kvm_regs);
A
Avi Kivity 已提交
2017 2018 2019
		break;
	}
	case KVM_GET_SREGS: {
2020 2021 2022 2023 2024
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2025 2026 2027
		if (r)
			goto out;
		r = -EFAULT;
2028
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
2029 2030 2031 2032 2033
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
2034 2035 2036
		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
		if (IS_ERR(kvm_sregs)) {
			r = PTR_ERR(kvm_sregs);
G
Guo Chao 已提交
2037
			kvm_sregs = NULL;
A
Avi Kivity 已提交
2038
			goto out;
2039
		}
2040
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2041 2042
		break;
	}
2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		break;
	}
A
Avi Kivity 已提交
2064 2065 2066 2067
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
2068
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
2069
			goto out;
2070
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2071 2072 2073
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2074
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
2075 2076 2077 2078
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
2079 2080
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
2081 2082

		r = -EFAULT;
A
Al Viro 已提交
2083
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
2084
			goto out;
J
Jan Kiszka 已提交
2085
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
2086 2087
		break;
	}
A
Avi Kivity 已提交
2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
2108
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
A
Avi Kivity 已提交
2109 2110
		break;
	}
A
Avi Kivity 已提交
2111
	case KVM_GET_FPU: {
2112 2113 2114 2115 2116
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2117 2118 2119
		if (r)
			goto out;
		r = -EFAULT;
2120
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
2121 2122 2123 2124 2125
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
2126 2127 2128
		fpu = memdup_user(argp, sizeof(*fpu));
		if (IS_ERR(fpu)) {
			r = PTR_ERR(fpu);
G
Guo Chao 已提交
2129
			fpu = NULL;
A
Avi Kivity 已提交
2130
			goto out;
2131
		}
2132
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2133 2134
		break;
	}
A
Avi Kivity 已提交
2135
	default:
2136
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2137 2138
	}
out:
2139
	vcpu_put(vcpu);
2140 2141
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
2142 2143 2144
	return r;
}

2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174
#ifdef CONFIG_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *filp,
				  unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = compat_ptr(arg);
	int r;

	if (vcpu->kvm->mm != current->mm)
		return -EIO;

	switch (ioctl) {
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		compat_sigset_t csigset;
		sigset_t sigset;

		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof csigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&csigset, sigmask_arg->sigset,
					   sizeof csigset))
				goto out;
2175 2176 2177 2178
			sigset_from_compat(&sigset, &csigset);
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		} else
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189
		break;
	}
	default:
		r = kvm_vcpu_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

S
Scott Wood 已提交
2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236
static int kvm_device_ioctl_attr(struct kvm_device *dev,
				 int (*accessor)(struct kvm_device *dev,
						 struct kvm_device_attr *attr),
				 unsigned long arg)
{
	struct kvm_device_attr attr;

	if (!accessor)
		return -EPERM;

	if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
		return -EFAULT;

	return accessor(dev, &attr);
}

static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
			     unsigned long arg)
{
	struct kvm_device *dev = filp->private_data;

	switch (ioctl) {
	case KVM_SET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
	case KVM_GET_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
	case KVM_HAS_DEVICE_ATTR:
		return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
	default:
		if (dev->ops->ioctl)
			return dev->ops->ioctl(dev, ioctl, arg);

		return -ENOTTY;
	}
}

static int kvm_device_release(struct inode *inode, struct file *filp)
{
	struct kvm_device *dev = filp->private_data;
	struct kvm *kvm = dev->kvm;

	kvm_put_kvm(kvm);
	return 0;
}

static const struct file_operations kvm_device_fops = {
	.unlocked_ioctl = kvm_device_ioctl,
2237 2238 2239
#ifdef CONFIG_COMPAT
	.compat_ioctl = kvm_device_ioctl,
#endif
S
Scott Wood 已提交
2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259
	.release = kvm_device_release,
};

struct kvm_device *kvm_device_from_filp(struct file *filp)
{
	if (filp->f_op != &kvm_device_fops)
		return NULL;

	return filp->private_data;
}

static int kvm_ioctl_create_device(struct kvm *kvm,
				   struct kvm_create_device *cd)
{
	struct kvm_device_ops *ops = NULL;
	struct kvm_device *dev;
	bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
	int ret;

	switch (cd->type) {
2260 2261 2262 2263 2264
#ifdef CONFIG_KVM_MPIC
	case KVM_DEV_TYPE_FSL_MPIC_20:
	case KVM_DEV_TYPE_FSL_MPIC_42:
		ops = &kvm_mpic_ops;
		break;
2265 2266 2267 2268 2269
#endif
#ifdef CONFIG_KVM_XICS
	case KVM_DEV_TYPE_XICS:
		ops = &kvm_xics_ops;
		break;
A
Alex Williamson 已提交
2270 2271 2272 2273 2274
#endif
#ifdef CONFIG_KVM_VFIO
	case KVM_DEV_TYPE_VFIO:
		ops = &kvm_vfio_ops;
		break;
2275
#endif
S
Scott Wood 已提交
2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295
	default:
		return -ENODEV;
	}

	if (test)
		return 0;

	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
	if (!dev)
		return -ENOMEM;

	dev->ops = ops;
	dev->kvm = kvm;

	ret = ops->create(dev, cd->type);
	if (ret < 0) {
		kfree(dev);
		return ret;
	}

2296
	ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
S
Scott Wood 已提交
2297 2298 2299 2300 2301
	if (ret < 0) {
		ops->destroy(dev);
		return ret;
	}

2302
	list_add(&dev->vm_node, &kvm->devices);
S
Scott Wood 已提交
2303 2304 2305 2306 2307
	kvm_get_kvm(kvm);
	cd->fd = ret;
	return 0;
}

A
Avi Kivity 已提交
2308 2309 2310 2311 2312
static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
2313
	int r;
A
Avi Kivity 已提交
2314

2315 2316
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
2317 2318 2319 2320
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		break;
2321 2322 2323 2324 2325 2326 2327 2328
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

2329
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
A
Avi Kivity 已提交
2330 2331 2332 2333 2334 2335
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
2336
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
2337
			goto out;
2338
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
2339 2340
		break;
	}
2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		break;
	}
#endif
G
Gregory Haskins 已提交
2359 2360 2361 2362 2363 2364
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
2365
		r = kvm_irqfd(kvm, &data);
G
Gregory Haskins 已提交
2366 2367
		break;
	}
G
Gregory Haskins 已提交
2368 2369 2370 2371 2372 2373 2374 2375 2376
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
2377 2378 2379
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
2380
		mutex_lock(&kvm->lock);
2381 2382 2383 2384
		if (atomic_read(&kvm->online_vcpus) != 0)
			r = -EBUSY;
		else
			kvm->bsp_vcpu_id = arg;
2385
		mutex_unlock(&kvm->lock);
2386
		break;
2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397
#endif
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_SIGNAL_MSI: {
		struct kvm_msi msi;

		r = -EFAULT;
		if (copy_from_user(&msi, argp, sizeof msi))
			goto out;
		r = kvm_send_userspace_msi(kvm, &msi);
		break;
	}
2398 2399 2400 2401 2402 2403 2404 2405 2406 2407
#endif
#ifdef __KVM_HAVE_IRQ_LINE
	case KVM_IRQ_LINE_STATUS:
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
		if (copy_from_user(&irq_event, argp, sizeof irq_event))
			goto out;

2408 2409
		r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
					ioctl == KVM_IRQ_LINE_STATUS);
2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421
		if (r)
			goto out;

		r = -EFAULT;
		if (ioctl == KVM_IRQ_LINE_STATUS) {
			if (copy_to_user(argp, &irq_event, sizeof irq_event))
				goto out;
		}

		r = 0;
		break;
	}
2422
#endif
2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_SET_GSI_ROUTING: {
		struct kvm_irq_routing routing;
		struct kvm_irq_routing __user *urouting;
		struct kvm_irq_routing_entry *entries;

		r = -EFAULT;
		if (copy_from_user(&routing, argp, sizeof(routing)))
			goto out;
		r = -EINVAL;
		if (routing.nr >= KVM_MAX_IRQ_ROUTES)
			goto out;
		if (routing.flags)
			goto out;
		r = -ENOMEM;
		entries = vmalloc(routing.nr * sizeof(*entries));
		if (!entries)
			goto out;
		r = -EFAULT;
		urouting = argp;
		if (copy_from_user(entries, urouting->entries,
				   routing.nr * sizeof(*entries)))
			goto out_free_irq_routing;
		r = kvm_set_irq_routing(kvm, entries, routing.nr,
					routing.flags);
	out_free_irq_routing:
		vfree(entries);
		break;
	}
#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
S
Scott Wood 已提交
2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470
	case KVM_CREATE_DEVICE: {
		struct kvm_create_device cd;

		r = -EFAULT;
		if (copy_from_user(&cd, argp, sizeof(cd)))
			goto out;

		r = kvm_ioctl_create_device(kvm, &cd);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(argp, &cd, sizeof(cd)))
			goto out;

		r = 0;
		break;
	}
2471
	default:
2472
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2473 2474
		if (r == -ENOTTY)
			r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2475 2476 2477 2478 2479
	}
out:
	return r;
}

2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523
#ifdef CONFIG_COMPAT
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		r = -EFAULT;
		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
			goto out;
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

2524
static struct file_operations kvm_vm_fops = {
2525 2526
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
2527 2528 2529
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vm_compat_ioctl,
#endif
2530
	.llseek		= noop_llseek,
2531 2532
};

2533
static int kvm_dev_ioctl_create_vm(unsigned long type)
2534
{
2535
	int r;
2536 2537
	struct kvm *kvm;

2538
	kvm = kvm_create_vm(type);
2539 2540
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
2541 2542 2543 2544 2545 2546 2547
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	r = kvm_coalesced_mmio_init(kvm);
	if (r < 0) {
		kvm_put_kvm(kvm);
		return r;
	}
#endif
2548
	r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC);
2549
	if (r < 0)
A
Al Viro 已提交
2550
		kvm_put_kvm(kvm);
2551

2552
	return r;
2553 2554
}

2555 2556 2557
static long kvm_dev_ioctl_check_extension_generic(long arg)
{
	switch (arg) {
2558
	case KVM_CAP_USER_MEMORY:
2559
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2560
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2561 2562 2563
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_CAP_SET_BOOT_CPU_ID:
#endif
2564
	case KVM_CAP_INTERNAL_ERROR_DATA:
2565 2566
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_CAP_SIGNAL_MSI:
2567 2568 2569
#endif
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
	case KVM_CAP_IRQFD_RESAMPLE:
2570
#endif
2571
		return 1;
2572
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2573
	case KVM_CAP_IRQ_ROUTING:
2574
		return KVM_MAX_IRQ_ROUTES;
2575
#endif
2576 2577 2578 2579 2580 2581
	default:
		break;
	}
	return kvm_dev_ioctl_check_extension(arg);
}

2582 2583 2584
static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
2585
	long r = -EINVAL;
2586 2587 2588

	switch (ioctl) {
	case KVM_GET_API_VERSION:
2589 2590 2591
		r = -EINVAL;
		if (arg)
			goto out;
2592 2593 2594
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
2595
		r = kvm_dev_ioctl_create_vm(arg);
2596
		break;
2597
	case KVM_CHECK_EXTENSION:
2598
		r = kvm_dev_ioctl_check_extension_generic(arg);
2599
		break;
2600 2601 2602 2603
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
2604 2605 2606
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
2607 2608 2609
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
		r += PAGE_SIZE;    /* coalesced mmio ring page */
2610
#endif
2611
		break;
2612 2613 2614
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
2615
		r = -EOPNOTSUPP;
2616
		break;
A
Avi Kivity 已提交
2617
	default:
2618
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2619 2620 2621 2622 2623 2624 2625 2626
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
2627
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
2628 2629 2630
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
2631
	KVM_MINOR,
A
Avi Kivity 已提交
2632 2633 2634 2635
	"kvm",
	&kvm_chardev_ops,
};

2636
static void hardware_enable_nolock(void *junk)
2637 2638
{
	int cpu = raw_smp_processor_id();
2639
	int r;
2640

2641
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2642
		return;
2643

2644
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
2645 2646 2647 2648 2649 2650 2651 2652 2653

	r = kvm_arch_hardware_enable(NULL);

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
		printk(KERN_INFO "kvm: enabling virtualization on "
				 "CPU%d failed\n", cpu);
	}
2654 2655
}

2656
static void hardware_enable(void)
2657
{
2658
	raw_spin_lock(&kvm_count_lock);
2659 2660
	if (kvm_usage_count)
		hardware_enable_nolock(NULL);
2661
	raw_spin_unlock(&kvm_count_lock);
2662 2663 2664
}

static void hardware_disable_nolock(void *junk)
2665 2666 2667
{
	int cpu = raw_smp_processor_id();

2668
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2669
		return;
2670
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2671
	kvm_arch_hardware_disable(NULL);
2672 2673
}

2674
static void hardware_disable(void)
2675
{
2676
	raw_spin_lock(&kvm_count_lock);
2677 2678
	if (kvm_usage_count)
		hardware_disable_nolock(NULL);
2679
	raw_spin_unlock(&kvm_count_lock);
2680 2681
}

2682 2683 2684 2685 2686 2687
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
2688
		on_each_cpu(hardware_disable_nolock, NULL, 1);
2689 2690 2691 2692
}

static void hardware_disable_all(void)
{
2693
	raw_spin_lock(&kvm_count_lock);
2694
	hardware_disable_all_nolock();
2695
	raw_spin_unlock(&kvm_count_lock);
2696 2697 2698 2699 2700 2701
}

static int hardware_enable_all(void)
{
	int r = 0;

2702
	raw_spin_lock(&kvm_count_lock);
2703 2704 2705 2706

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
2707
		on_each_cpu(hardware_enable_nolock, NULL, 1);
2708 2709 2710 2711 2712 2713 2714

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

2715
	raw_spin_unlock(&kvm_count_lock);
2716 2717 2718 2719

	return r;
}

A
Avi Kivity 已提交
2720 2721 2722 2723 2724
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

2725
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
2726
	switch (val) {
2727
	case CPU_DYING:
2728 2729
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
2730
		hardware_disable();
2731
		break;
2732
	case CPU_STARTING:
2733 2734
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
2735
		hardware_enable();
A
Avi Kivity 已提交
2736 2737 2738 2739 2740
		break;
	}
	return NOTIFY_OK;
}

2741
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
2742
		      void *v)
2743
{
2744 2745 2746 2747 2748 2749 2750 2751
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
	kvm_rebooting = true;
2752
	on_each_cpu(hardware_disable_nolock, NULL, 1);
2753 2754 2755 2756 2757 2758 2759 2760
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
2761
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2762 2763 2764 2765
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
2766
		struct kvm_io_device *pos = bus->range[i].dev;
2767 2768 2769

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
2770
	kfree(bus);
2771 2772
}

2773 2774
static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
                                 const struct kvm_io_range *r2)
2775 2776 2777 2778 2779 2780 2781 2782
{
	if (r1->addr < r2->addr)
		return -1;
	if (r1->addr + r1->len > r2->addr + r2->len)
		return 1;
	return 0;
}

2783 2784
static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
{
2785
	return kvm_io_bus_cmp(p1, p2);
2786 2787
}

G
Geoff Levand 已提交
2788
static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802
			  gpa_t addr, int len)
{
	bus->range[bus->dev_count++] = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
		.dev = dev,
	};

	sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
		kvm_io_bus_sort_cmp, NULL);

	return 0;
}

G
Geoff Levand 已提交
2803
static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820
			     gpa_t addr, int len)
{
	struct kvm_io_range *range, key;
	int off;

	key = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	range = bsearch(&key, bus->range, bus->dev_count,
			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
	if (range == NULL)
		return -ENOENT;

	off = range - bus->range;

2821
	while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
2822 2823 2824 2825 2826
		off--;

	return off;
}

C
Cornelia Huck 已提交
2827 2828 2829 2830 2831 2832 2833 2834 2835 2836
static int __kvm_io_bus_write(struct kvm_io_bus *bus,
			      struct kvm_io_range *range, const void *val)
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
2837
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
C
Cornelia Huck 已提交
2838 2839 2840 2841 2842 2843 2844 2845 2846
		if (!kvm_iodevice_write(bus->range[idx].dev, range->addr,
					range->len, val))
			return idx;
		idx++;
	}

	return -EOPNOTSUPP;
}

2847
/* kvm_io_bus_write - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2848
int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2849
		     int len, const void *val)
2850
{
2851
	struct kvm_io_bus *bus;
2852
	struct kvm_io_range range;
C
Cornelia Huck 已提交
2853
	int r;
2854 2855 2856 2857 2858

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
2859 2860

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
C
Cornelia Huck 已提交
2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880
	r = __kvm_io_bus_write(bus, &range, val);
	return r < 0 ? r : 0;
}

/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, const void *val, long cookie)
{
	struct kvm_io_bus *bus;
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);

	/* First try the device referenced by cookie. */
	if ((cookie >= 0) && (cookie < bus->dev_count) &&
2881
	    (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
C
Cornelia Huck 已提交
2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898
		if (!kvm_iodevice_write(bus->range[cookie].dev, addr, len,
					val))
			return cookie;

	/*
	 * cookie contained garbage; fall back to search and return the
	 * correct cookie value.
	 */
	return __kvm_io_bus_write(bus, &range, val);
}

static int __kvm_io_bus_read(struct kvm_io_bus *bus, struct kvm_io_range *range,
			     void *val)
{
	int idx;

	idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
2899 2900 2901 2902
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
2903
		kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
C
Cornelia Huck 已提交
2904 2905 2906
		if (!kvm_iodevice_read(bus->range[idx].dev, range->addr,
				       range->len, val))
			return idx;
2907 2908 2909
		idx++;
	}

2910 2911
	return -EOPNOTSUPP;
}
2912

2913
/* kvm_io_bus_read - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2914 2915
int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
		    int len, void *val)
2916
{
2917
	struct kvm_io_bus *bus;
2918
	struct kvm_io_range range;
C
Cornelia Huck 已提交
2919
	int r;
2920 2921 2922 2923 2924

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
M
Marcelo Tosatti 已提交
2925

2926
	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
C
Cornelia Huck 已提交
2927 2928 2929
	r = __kvm_io_bus_read(bus, &range, val);
	return r < 0 ? r : 0;
}
2930

C
Cornelia Huck 已提交
2931 2932 2933 2934 2935 2936
/* kvm_io_bus_read_cookie - called under kvm->slots_lock */
int kvm_io_bus_read_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			   int len, void *val, long cookie)
{
	struct kvm_io_bus *bus;
	struct kvm_io_range range;
2937

C
Cornelia Huck 已提交
2938 2939 2940 2941 2942 2943 2944 2945 2946
	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);

	/* First try the device referenced by cookie. */
	if ((cookie >= 0) && (cookie < bus->dev_count) &&
2947
	    (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
C
Cornelia Huck 已提交
2948 2949 2950 2951 2952 2953 2954 2955 2956
		if (!kvm_iodevice_read(bus->range[cookie].dev, addr, len,
				       val))
			return cookie;

	/*
	 * cookie contained garbage; fall back to search and return the
	 * correct cookie value.
	 */
	return __kvm_io_bus_read(bus, &range, val);
2957 2958
}

2959
/* Caller must hold slots_lock. */
2960 2961
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, struct kvm_io_device *dev)
2962
{
M
Marcelo Tosatti 已提交
2963
	struct kvm_io_bus *new_bus, *bus;
2964

M
Marcelo Tosatti 已提交
2965
	bus = kvm->buses[bus_idx];
2966 2967
	/* exclude ioeventfd which is limited by maximum fd */
	if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
2968
		return -ENOSPC;
2969

2970 2971
	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
M
Marcelo Tosatti 已提交
2972 2973
	if (!new_bus)
		return -ENOMEM;
2974 2975
	memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
	       sizeof(struct kvm_io_range)));
2976
	kvm_io_bus_insert_dev(new_bus, dev, addr, len);
M
Marcelo Tosatti 已提交
2977 2978 2979
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
2980 2981 2982 2983

	return 0;
}

2984
/* Caller must hold slots_lock. */
M
Marcelo Tosatti 已提交
2985 2986
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			      struct kvm_io_device *dev)
2987
{
M
Marcelo Tosatti 已提交
2988 2989
	int i, r;
	struct kvm_io_bus *new_bus, *bus;
2990

2991
	bus = kvm->buses[bus_idx];
M
Marcelo Tosatti 已提交
2992
	r = -ENOENT;
2993 2994
	for (i = 0; i < bus->dev_count; i++)
		if (bus->range[i].dev == dev) {
M
Marcelo Tosatti 已提交
2995
			r = 0;
2996 2997
			break;
		}
M
Marcelo Tosatti 已提交
2998

2999
	if (r)
M
Marcelo Tosatti 已提交
3000
		return r;
3001 3002 3003 3004 3005 3006 3007 3008 3009 3010

	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
	if (!new_bus)
		return -ENOMEM;

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count--;
	memcpy(new_bus->range + i, bus->range + i + 1,
	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
3011 3012 3013 3014 3015

	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
	return r;
3016 3017
}

A
Avi Kivity 已提交
3018 3019 3020 3021
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
};

3022
static int vm_stat_get(void *_offset, u64 *val)
3023 3024 3025 3026
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

3027
	*val = 0;
3028
	spin_lock(&kvm_lock);
3029
	list_for_each_entry(kvm, &vm_list, vm_list)
3030
		*val += *(u32 *)((void *)kvm + offset);
3031
	spin_unlock(&kvm_lock);
3032
	return 0;
3033 3034 3035 3036
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

3037
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
3038 3039 3040 3041 3042 3043
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

3044
	*val = 0;
3045
	spin_lock(&kvm_lock);
A
Avi Kivity 已提交
3046
	list_for_each_entry(kvm, &vm_list, vm_list)
3047 3048 3049
		kvm_for_each_vcpu(i, vcpu, kvm)
			*val += *(u32 *)((void *)vcpu + offset);

3050
	spin_unlock(&kvm_lock);
3051
	return 0;
A
Avi Kivity 已提交
3052 3053
}

3054 3055
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

3056
static const struct file_operations *stat_fops[] = {
3057 3058 3059
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
3060

3061
static int kvm_init_debug(void)
A
Avi Kivity 已提交
3062
{
3063
	int r = -EEXIST;
A
Avi Kivity 已提交
3064 3065
	struct kvm_stats_debugfs_item *p;

3066
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3067 3068 3069 3070
	if (kvm_debugfs_dir == NULL)
		goto out;

	for (p = debugfs_entries; p->name; ++p) {
3071
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
3072
						(void *)(long)p->offset,
3073
						stat_fops[p->kind]);
3074 3075 3076 3077 3078 3079 3080 3081 3082 3083
		if (p->dentry == NULL)
			goto out_dir;
	}

	return 0;

out_dir:
	debugfs_remove_recursive(kvm_debugfs_dir);
out:
	return r;
A
Avi Kivity 已提交
3084 3085 3086 3087 3088 3089 3090 3091
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
3092
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
3093 3094
}

3095
static int kvm_suspend(void)
3096
{
3097
	if (kvm_usage_count)
3098
		hardware_disable_nolock(NULL);
3099 3100 3101
	return 0;
}

3102
static void kvm_resume(void)
3103
{
3104
	if (kvm_usage_count) {
3105
		WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3106
		hardware_enable_nolock(NULL);
3107
	}
3108 3109
}

3110
static struct syscore_ops kvm_syscore_ops = {
3111 3112 3113 3114
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

3115 3116 3117 3118 3119 3120 3121 3122 3123
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3124 3125
	if (vcpu->preempted)
		vcpu->preempted = false;
3126

3127
	kvm_arch_vcpu_load(vcpu, cpu);
3128 3129 3130 3131 3132 3133 3134
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

3135 3136
	if (current->state == TASK_RUNNING)
		vcpu->preempted = true;
3137
	kvm_arch_vcpu_put(vcpu);
3138 3139
}

3140
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3141
		  struct module *module)
A
Avi Kivity 已提交
3142 3143
{
	int r;
Y
Yang, Sheng 已提交
3144
	int cpu;
A
Avi Kivity 已提交
3145

3146 3147
	r = kvm_arch_init(opaque);
	if (r)
3148
		goto out_fail;
3149

3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160
	/*
	 * kvm_arch_init makes sure there's at most one caller
	 * for architectures that support multiple implementations,
	 * like intel and amd on x86.
	 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
	 * conflicts in case kvm is already setup for another implementation.
	 */
	r = kvm_irqfd_init();
	if (r)
		goto out_irqfd;

3161
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
3162 3163 3164 3165
		r = -ENOMEM;
		goto out_free_0;
	}

3166
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
3167
	if (r < 0)
3168
		goto out_free_0a;
A
Avi Kivity 已提交
3169

Y
Yang, Sheng 已提交
3170 3171
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
3172
				kvm_arch_check_processor_compat,
3173
				&r, 1);
Y
Yang, Sheng 已提交
3174
		if (r < 0)
3175
			goto out_free_1;
Y
Yang, Sheng 已提交
3176 3177
	}

A
Avi Kivity 已提交
3178 3179
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
3180
		goto out_free_2;
A
Avi Kivity 已提交
3181 3182
	register_reboot_notifier(&kvm_reboot_notifier);

3183
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
3184 3185 3186
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
J
Joe Perches 已提交
3187
					   0, NULL);
3188 3189
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
3190
		goto out_free_3;
3191 3192
	}

3193 3194 3195 3196
	r = kvm_async_pf_init();
	if (r)
		goto out_free;

A
Avi Kivity 已提交
3197
	kvm_chardev_ops.owner = module;
3198 3199
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
3200 3201 3202

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
3203
		printk(KERN_ERR "kvm: misc device register failed\n");
3204
		goto out_unreg;
A
Avi Kivity 已提交
3205 3206
	}

3207 3208
	register_syscore_ops(&kvm_syscore_ops);

3209 3210 3211
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

3212 3213 3214 3215 3216
	r = kvm_init_debug();
	if (r) {
		printk(KERN_ERR "kvm: create debugfs files failed\n");
		goto out_undebugfs;
	}
3217

3218
	return 0;
A
Avi Kivity 已提交
3219

3220 3221
out_undebugfs:
	unregister_syscore_ops(&kvm_syscore_ops);
3222
	misc_deregister(&kvm_dev);
3223 3224
out_unreg:
	kvm_async_pf_deinit();
A
Avi Kivity 已提交
3225
out_free:
3226
	kmem_cache_destroy(kvm_vcpu_cache);
3227
out_free_3:
A
Avi Kivity 已提交
3228
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
3229
	unregister_cpu_notifier(&kvm_cpu_notifier);
3230 3231
out_free_2:
out_free_1:
3232
	kvm_arch_hardware_unsetup();
3233 3234
out_free_0a:
	free_cpumask_var(cpus_hardware_enabled);
3235
out_free_0:
3236 3237
	kvm_irqfd_exit();
out_irqfd:
3238 3239
	kvm_arch_exit();
out_fail:
A
Avi Kivity 已提交
3240 3241
	return r;
}
3242
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
3243

3244
void kvm_exit(void)
A
Avi Kivity 已提交
3245
{
3246
	kvm_exit_debug();
A
Avi Kivity 已提交
3247
	misc_deregister(&kvm_dev);
3248
	kmem_cache_destroy(kvm_vcpu_cache);
3249
	kvm_async_pf_deinit();
3250
	unregister_syscore_ops(&kvm_syscore_ops);
A
Avi Kivity 已提交
3251
	unregister_reboot_notifier(&kvm_reboot_notifier);
3252
	unregister_cpu_notifier(&kvm_cpu_notifier);
3253
	on_each_cpu(hardware_disable_nolock, NULL, 1);
3254
	kvm_arch_hardware_unsetup();
3255
	kvm_arch_exit();
3256
	kvm_irqfd_exit();
3257
	free_cpumask_var(cpus_hardware_enabled);
A
Avi Kivity 已提交
3258
}
3259
EXPORT_SYMBOL_GPL(kvm_exit);