kvm_main.c 67.1 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "iodev.h"
A
Avi Kivity 已提交
20

21
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/syscore_ops.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
35
#include <linux/sched.h>
36 37
#include <linux/cpumask.h>
#include <linux/smp.h>
38
#include <linux/anon_inodes.h>
39
#include <linux/profile.h>
40
#include <linux/kvm_para.h>
41
#include <linux/pagemap.h>
42
#include <linux/mman.h>
43
#include <linux/swap.h>
44
#include <linux/bitops.h>
45
#include <linux/spinlock.h>
46
#include <linux/compat.h>
47
#include <linux/srcu.h>
J
Joerg Roedel 已提交
48
#include <linux/hugetlb.h>
49
#include <linux/slab.h>
50 51
#include <linux/sort.h>
#include <linux/bsearch.h>
A
Avi Kivity 已提交
52

A
Avi Kivity 已提交
53 54 55
#include <asm/processor.h>
#include <asm/io.h>
#include <asm/uaccess.h>
56
#include <asm/pgtable.h>
A
Avi Kivity 已提交
57

58
#include "coalesced_mmio.h"
59
#include "async_pf.h"
60

61 62 63
#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

A
Avi Kivity 已提交
64 65 66
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

67 68 69
/*
 * Ordering of locks:
 *
70
 * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
71 72
 */

73
DEFINE_RAW_SPINLOCK(kvm_lock);
74
LIST_HEAD(vm_list);
75

76
static cpumask_var_t cpus_hardware_enabled;
77 78
static int kvm_usage_count = 0;
static atomic_t hardware_enable_failed;
79

80 81
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
82

83 84
static __read_mostly struct preempt_ops kvm_preempt_ops;

85
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
86

A
Avi Kivity 已提交
87 88
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);
89 90 91 92
#ifdef CONFIG_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
				  unsigned long arg);
#endif
93 94
static int hardware_enable_all(void);
static void hardware_disable_all(void);
A
Avi Kivity 已提交
95

M
Marcelo Tosatti 已提交
96 97
static void kvm_io_bus_destroy(struct kvm_io_bus *bus);

98 99
bool kvm_rebooting;
EXPORT_SYMBOL_GPL(kvm_rebooting);
100

101 102
static bool largepages_enabled = true;

X
Xiao Guangrong 已提交
103
bool kvm_is_mmio_pfn(pfn_t pfn)
B
Ben-Ami Yassour 已提交
104
{
105
	if (pfn_valid(pfn)) {
106
		int reserved;
107
		struct page *tail = pfn_to_page(pfn);
108 109
		struct page *head = compound_trans_head(tail);
		reserved = PageReserved(head);
110 111
		if (head != tail) {
			/*
112 113 114 115 116 117 118 119
			 * "head" is not a dangling pointer
			 * (compound_trans_head takes care of that)
			 * but the hugepage may have been splitted
			 * from under us (and we may not hold a
			 * reference count on the head page so it can
			 * be reused before we run PageReferenced), so
			 * we've to check PageTail before returning
			 * what we just read.
120
			 */
121 122 123
			smp_rmb();
			if (PageTail(tail))
				return reserved;
124 125
		}
		return PageReserved(tail);
126
	}
B
Ben-Ami Yassour 已提交
127 128 129 130

	return true;
}

A
Avi Kivity 已提交
131 132 133
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
134
int vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
135
{
136 137
	int cpu;

138 139
	if (mutex_lock_killable(&vcpu->mutex))
		return -EINTR;
140 141 142 143 144 145 146 147
	if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
		/* The thread running this VCPU changed. */
		struct pid *oldpid = vcpu->pid;
		struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
		rcu_assign_pointer(vcpu->pid, newpid);
		synchronize_rcu();
		put_pid(oldpid);
	}
148 149
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
150
	kvm_arch_vcpu_load(vcpu, cpu);
151
	put_cpu();
152
	return 0;
A
Avi Kivity 已提交
153 154
}

155
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
156
{
157
	preempt_disable();
158
	kvm_arch_vcpu_put(vcpu);
159 160
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
161 162 163
	mutex_unlock(&vcpu->mutex);
}

164 165 166 167
static void ack_flush(void *_completed)
{
}

168
static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
169
{
170
	int i, cpu, me;
171 172
	cpumask_var_t cpus;
	bool called = true;
173 174
	struct kvm_vcpu *vcpu;

175
	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
176

177
	me = get_cpu();
178
	kvm_for_each_vcpu(i, vcpu, kvm) {
179
		kvm_make_request(req, vcpu);
180
		cpu = vcpu->cpu;
181 182 183 184 185 186

		/* Set ->requests bit before we read ->mode */
		smp_mb();

		if (cpus != NULL && cpu != -1 && cpu != me &&
		      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
187
			cpumask_set_cpu(cpu, cpus);
188
	}
189 190 191 192 193 194
	if (unlikely(cpus == NULL))
		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
	else if (!cpumask_empty(cpus))
		smp_call_function_many(cpus, ack_flush, NULL, 1);
	else
		called = false;
195
	put_cpu();
196
	free_cpumask_var(cpus);
197
	return called;
198 199
}

200
void kvm_flush_remote_tlbs(struct kvm *kvm)
201
{
202
	long dirty_count = kvm->tlbs_dirty;
203 204

	smp_mb();
205 206
	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
		++kvm->stat.remote_tlb_flush;
207
	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
208 209
}

210 211 212 213
void kvm_reload_remote_mmus(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
}
214

215 216 217 218 219
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}

220 221 222 223 224
void kvm_make_update_eoibitmap_request(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_EOIBITMAP);
}

R
Rusty Russell 已提交
225 226 227 228 229 230 231 232 233
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
234
	vcpu->pid = NULL;
E
Eddie Dong 已提交
235
	init_waitqueue_head(&vcpu->wq);
236
	kvm_async_pf_vcpu_init(vcpu);
R
Rusty Russell 已提交
237 238 239 240 241 242 243 244

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

245 246 247
	kvm_vcpu_set_in_spin_loop(vcpu, false);
	kvm_vcpu_set_dy_eligible(vcpu, false);

248
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
249
	if (r < 0)
250
		goto fail_free_run;
R
Rusty Russell 已提交
251 252 253 254 255
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
256
	return r;
R
Rusty Russell 已提交
257 258 259 260 261
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
262
	put_pid(vcpu->pid);
263
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
264 265 266 267
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

268 269 270 271 272 273 274 275 276 277 278
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
279
	int need_tlb_flush, idx;
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
299
	idx = srcu_read_lock(&kvm->srcu);
300
	spin_lock(&kvm->mmu_lock);
301

302
	kvm->mmu_notifier_seq++;
303
	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
304 305 306 307
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

308 309
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
310 311
}

312 313 314 315 316 317
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
					struct mm_struct *mm,
					unsigned long address,
					pte_t pte)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
318
	int idx;
319

320
	idx = srcu_read_lock(&kvm->srcu);
321 322 323 324
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	kvm_set_spte_hva(kvm, address, pte);
	spin_unlock(&kvm->mmu_lock);
325
	srcu_read_unlock(&kvm->srcu, idx);
326 327
}

328 329 330 331 332 333
static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
334
	int need_tlb_flush = 0, idx;
335

336
	idx = srcu_read_lock(&kvm->srcu);
337 338 339 340 341 342 343
	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
344
	need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
345
	need_tlb_flush |= kvm->tlbs_dirty;
346 347 348
	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
349 350 351

	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
368
	smp_wmb();
369 370
	/*
	 * The above sequence increase must be visible before the
371 372
	 * below count decrease, which is ensured by the smp_wmb above
	 * in conjunction with the smp_rmb in mmu_notifier_retry().
373 374 375 376 377 378 379 380 381 382 383 384
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
					      unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
385
	int young, idx;
386

387
	idx = srcu_read_lock(&kvm->srcu);
388 389
	spin_lock(&kvm->mmu_lock);

390
	young = kvm_age_hva(kvm, address);
391 392 393
	if (young)
		kvm_flush_remote_tlbs(kvm);

394 395 396
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

397 398 399
	return young;
}

A
Andrea Arcangeli 已提交
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
				       struct mm_struct *mm,
				       unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young, idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);
	young = kvm_test_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);

	return young;
}

416 417 418 419
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
420 421 422
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
423
	kvm_arch_flush_shadow_all(kvm);
424
	srcu_read_unlock(&kvm->srcu, idx);
425 426
}

427 428 429 430 431
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
A
Andrea Arcangeli 已提交
432
	.test_young		= kvm_mmu_notifier_test_young,
433
	.change_pte		= kvm_mmu_notifier_change_pte,
434
	.release		= kvm_mmu_notifier_release,
435
};
436 437 438 439 440 441 442 443 444 445 446 447 448 449

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
	return 0;
}

450 451
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

452 453 454 455 456 457
static void kvm_init_memslots_id(struct kvm *kvm)
{
	int i;
	struct kvm_memslots *slots = kvm->memslots;

	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
458
		slots->id_to_index[i] = slots->memslots[i].id = i;
459 460
}

461
static struct kvm *kvm_create_vm(unsigned long type)
A
Avi Kivity 已提交
462
{
463 464
	int r, i;
	struct kvm *kvm = kvm_arch_alloc_vm();
A
Avi Kivity 已提交
465

466 467 468
	if (!kvm)
		return ERR_PTR(-ENOMEM);

469
	r = kvm_arch_init_vm(kvm, type);
470 471
	if (r)
		goto out_err_nodisable;
472 473 474 475 476

	r = hardware_enable_all();
	if (r)
		goto out_err_nodisable;

477 478
#ifdef CONFIG_HAVE_KVM_IRQCHIP
	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
479
	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
480
#endif
A
Avi Kivity 已提交
481

482 483
	BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);

484 485 486
	r = -ENOMEM;
	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
	if (!kvm->memslots)
487
		goto out_err_nosrcu;
488
	kvm_init_memslots_id(kvm);
489
	if (init_srcu_struct(&kvm->srcu))
490
		goto out_err_nosrcu;
M
Marcelo Tosatti 已提交
491 492 493
	for (i = 0; i < KVM_NR_BUSES; i++) {
		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
					GFP_KERNEL);
494
		if (!kvm->buses[i])
M
Marcelo Tosatti 已提交
495 496
			goto out_err;
	}
497

498
	spin_lock_init(&kvm->mmu_lock);
499 500
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
G
Gregory Haskins 已提交
501
	kvm_eventfd_init(kvm);
S
Shaohua Li 已提交
502
	mutex_init(&kvm->lock);
503
	mutex_init(&kvm->irq_lock);
504
	mutex_init(&kvm->slots_lock);
I
Izik Eidus 已提交
505
	atomic_set(&kvm->users_count, 1);
506 507 508 509 510

	r = kvm_init_mmu_notifier(kvm);
	if (r)
		goto out_err;

511
	raw_spin_lock(&kvm_lock);
512
	list_add(&kvm->vm_list, &vm_list);
513
	raw_spin_unlock(&kvm_lock);
514

515
	return kvm;
516 517

out_err:
518 519
	cleanup_srcu_struct(&kvm->srcu);
out_err_nosrcu:
520 521
	hardware_disable_all();
out_err_nodisable:
M
Marcelo Tosatti 已提交
522 523
	for (i = 0; i < KVM_NR_BUSES; i++)
		kfree(kvm->buses[i]);
524
	kfree(kvm->memslots);
525
	kvm_arch_free_vm(kvm);
526
	return ERR_PTR(r);
527 528
}

529 530 531 532
/*
 * Avoid using vmalloc for a small buffer.
 * Should not be used when the size is statically known.
 */
533
void *kvm_kvzalloc(unsigned long size)
534 535 536 537 538 539 540
{
	if (size > PAGE_SIZE)
		return vzalloc(size);
	else
		return kzalloc(size, GFP_KERNEL);
}

541
void kvm_kvfree(const void *addr)
542 543 544 545 546 547 548
{
	if (is_vmalloc_addr(addr))
		vfree(addr);
	else
		kfree(addr);
}

549 550 551 552 553
static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
	if (!memslot->dirty_bitmap)
		return;

554
	kvm_kvfree(memslot->dirty_bitmap);
555 556 557
	memslot->dirty_bitmap = NULL;
}

A
Avi Kivity 已提交
558 559 560 561 562 563 564
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
565
		kvm_destroy_dirty_bitmap(free);
A
Avi Kivity 已提交
566

567
	kvm_arch_free_memslot(free, dont);
M
Marcelo Tosatti 已提交
568

A
Avi Kivity 已提交
569 570 571
	free->npages = 0;
}

572
void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
573
{
574
	struct kvm_memslots *slots = kvm->memslots;
575
	struct kvm_memory_slot *memslot;
576

577 578
	kvm_for_each_memslot(memslot, slots)
		kvm_free_physmem_slot(memslot, NULL);
A
Avi Kivity 已提交
579

580
	kfree(kvm->memslots);
A
Avi Kivity 已提交
581 582
}

583 584
static void kvm_destroy_vm(struct kvm *kvm)
{
M
Marcelo Tosatti 已提交
585
	int i;
586 587
	struct mm_struct *mm = kvm->mm;

588
	kvm_arch_sync_events(kvm);
589
	raw_spin_lock(&kvm_lock);
590
	list_del(&kvm->vm_list);
591
	raw_spin_unlock(&kvm_lock);
592
	kvm_free_irq_routing(kvm);
M
Marcelo Tosatti 已提交
593 594
	for (i = 0; i < KVM_NR_BUSES; i++)
		kvm_io_bus_destroy(kvm->buses[i]);
595
	kvm_coalesced_mmio_free(kvm);
596 597
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
598
#else
599
	kvm_arch_flush_shadow_all(kvm);
600
#endif
601
	kvm_arch_destroy_vm(kvm);
602 603 604
	kvm_free_physmem(kvm);
	cleanup_srcu_struct(&kvm->srcu);
	kvm_arch_free_vm(kvm);
605
	hardware_disable_all();
606
	mmdrop(mm);
607 608
}

I
Izik Eidus 已提交
609 610 611 612 613 614 615 616 617 618 619 620 621 622
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


623 624 625 626
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

G
Gregory Haskins 已提交
627 628
	kvm_irqfd_release(kvm);

I
Izik Eidus 已提交
629
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
630 631 632
	return 0;
}

633 634
/*
 * Allocation size is twice as large as the actual dirty bitmap size.
635
 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
636
 */
637 638
static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
{
639
#ifndef CONFIG_S390
640
	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
641

642
	memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
643 644 645
	if (!memslot->dirty_bitmap)
		return -ENOMEM;

646
#endif /* !CONFIG_S390 */
647 648 649
	return 0;
}

650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
static int cmp_memslot(const void *slot1, const void *slot2)
{
	struct kvm_memory_slot *s1, *s2;

	s1 = (struct kvm_memory_slot *)slot1;
	s2 = (struct kvm_memory_slot *)slot2;

	if (s1->npages < s2->npages)
		return 1;
	if (s1->npages > s2->npages)
		return -1;

	return 0;
}

/*
 * Sort the memslots base on its size, so the larger slots
 * will get better fit.
 */
static void sort_memslots(struct kvm_memslots *slots)
{
671 672
	int i;

673 674
	sort(slots->memslots, KVM_MEM_SLOTS_NUM,
	      sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
675 676 677

	for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
		slots->id_to_index[slots->memslots[i].id] = i;
678 679
}

680 681
void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new,
		     u64 last_generation)
682 683 684
{
	if (new) {
		int id = new->id;
685
		struct kvm_memory_slot *old = id_to_memslot(slots, id);
686
		unsigned long npages = old->npages;
687

688
		*old = *new;
689 690
		if (new->npages != npages)
			sort_memslots(slots);
691 692
	}

693
	slots->generation = last_generation + 1;
694 695
}

696 697
static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
{
X
Xiao Guangrong 已提交
698 699 700 701 702 703 704
	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;

#ifdef KVM_CAP_READONLY_MEM
	valid_flags |= KVM_MEM_READONLY;
#endif

	if (mem->flags & ~valid_flags)
705 706 707 708 709
		return -EINVAL;

	return 0;
}

710 711 712 713 714 715 716 717 718 719 720
static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
		struct kvm_memslots *slots, struct kvm_memory_slot *new)
{
	struct kvm_memslots *old_memslots = kvm->memslots;

	update_memslots(slots, new, kvm->memslots->generation);
	rcu_assign_pointer(kvm->memslots, slots);
	synchronize_srcu_expedited(&kvm->srcu);
	return old_memslots; 
}

A
Avi Kivity 已提交
721 722 723 724 725
/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
726
 *
727
 * Must be called holding mmap_sem for write.
A
Avi Kivity 已提交
728
 */
729
int __kvm_set_memory_region(struct kvm *kvm,
730
			    struct kvm_userspace_memory_region *mem)
A
Avi Kivity 已提交
731
{
732
	int r;
A
Avi Kivity 已提交
733
	gfn_t base_gfn;
734
	unsigned long npages;
735
	struct kvm_memory_slot *slot;
A
Avi Kivity 已提交
736
	struct kvm_memory_slot old, new;
737
	struct kvm_memslots *slots = NULL, *old_memslots;
738
	enum kvm_mr_change change;
A
Avi Kivity 已提交
739

740 741 742 743
	r = check_memory_region_flags(mem);
	if (r)
		goto out;

A
Avi Kivity 已提交
744 745 746 747 748 749
	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
750
	/* We can read the guest memory with __xxx_user() later on. */
751
	if ((mem->slot < KVM_USER_MEM_SLOTS) &&
752
	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
753 754 755
	     !access_ok(VERIFY_WRITE,
			(void __user *)(unsigned long)mem->userspace_addr,
			mem->memory_size)))
756
		goto out;
757
	if (mem->slot >= KVM_MEM_SLOTS_NUM)
A
Avi Kivity 已提交
758 759 760 761
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

762
	slot = id_to_memslot(kvm->memslots, mem->slot);
A
Avi Kivity 已提交
763 764 765
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

766 767 768 769
	r = -EINVAL;
	if (npages > KVM_MEM_MAX_NR_PAGES)
		goto out;

A
Avi Kivity 已提交
770 771 772
	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

773
	new = old = *slot;
A
Avi Kivity 已提交
774

775
	new.id = mem->slot;
A
Avi Kivity 已提交
776 777 778 779 780
	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	r = -EINVAL;
781 782 783 784 785
	if (npages) {
		if (!old.npages)
			change = KVM_MR_CREATE;
		else { /* Modify an existing slot. */
			if ((mem->userspace_addr != old.userspace_addr) ||
786 787
			    (npages != old.npages) ||
			    ((new.flags ^ old.flags) & KVM_MEM_READONLY))
788 789 790 791 792 793 794 795 796 797 798 799 800 801
				goto out;

			if (base_gfn != old.base_gfn)
				change = KVM_MR_MOVE;
			else if (new.flags != old.flags)
				change = KVM_MR_FLAGS_ONLY;
			else { /* Nothing to change. */
				r = 0;
				goto out;
			}
		}
	} else if (old.npages) {
		change = KVM_MR_DELETE;
	} else /* Modify a non-existent slot: disallowed. */
802
		goto out;
A
Avi Kivity 已提交
803

804
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
805 806 807
		/* Check for overlaps */
		r = -EEXIST;
		kvm_for_each_memslot(slot, kvm->memslots) {
808 809
			if ((slot->id >= KVM_USER_MEM_SLOTS) ||
			    (slot->id == mem->slot))
810 811 812 813 814
				continue;
			if (!((base_gfn + npages <= slot->base_gfn) ||
			      (base_gfn >= slot->base_gfn + slot->npages)))
				goto out;
		}
A
Avi Kivity 已提交
815 816 817 818
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
819
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
820 821

	r = -ENOMEM;
822
	if (change == KVM_MR_CREATE) {
823
		new.userspace_addr = mem->userspace_addr;
824

825 826
		if (kvm_arch_create_memslot(&new, npages))
			goto out_free;
A
Avi Kivity 已提交
827
	}
828

A
Avi Kivity 已提交
829 830
	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
831
		if (kvm_create_dirty_bitmap(&new) < 0)
832
			goto out_free;
A
Avi Kivity 已提交
833 834
	}

835
	if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
836
		r = -ENOMEM;
837 838
		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
				GFP_KERNEL);
839 840
		if (!slots)
			goto out_free;
841 842 843
		slot = id_to_memslot(slots, mem->slot);
		slot->flags |= KVM_MEMSLOT_INVALID;

844
		old_memslots = install_new_memslots(kvm, slots, NULL);
845

846 847
		/* slot was deleted or moved, clear iommu mapping */
		kvm_iommu_unmap_pages(kvm, &old);
848 849
		/* From this point no new shadow pages pointing to a deleted,
		 * or moved, memslot will be created.
850 851 852 853 854
		 *
		 * validation of sp->gfn happens in:
		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
		 * 	- kvm_is_visible_gfn (mmu_check_roots)
		 */
855
		kvm_arch_flush_shadow_memslot(kvm, slot);
856
		slots = old_memslots;
857
	}
858

859
	r = kvm_arch_prepare_memory_region(kvm, &new, old, mem);
860
	if (r)
861
		goto out_slots;
862

863
	r = -ENOMEM;
864 865 866 867 868 869 870 871 872 873 874
	/*
	 * We can re-use the old_memslots from above, the only difference
	 * from the currently installed memslots is the invalid flag.  This
	 * will get overwritten by update_memslots anyway.
	 */
	if (!slots) {
		slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
				GFP_KERNEL);
		if (!slots)
			goto out_free;
	}
875

876 877
	/*
	 * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
878 879 880 881 882 883
	 * un-mapped and re-mapped if their base changes.  Since base change
	 * unmapping is handled above with slot deletion, mapping alone is
	 * needed here.  Anything else the iommu might care about for existing
	 * slots (size changes, userspace addr changes and read-only flag
	 * changes) is disallowed above, so any other attribute changes getting
	 * here can be skipped.
884
	 */
885 886 887 888
	if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
		r = kvm_iommu_map_pages(kvm, &new);
		if (r)
			goto out_slots;
889 890
	}

891
	/* actual memory is freed via old in kvm_free_physmem_slot below */
892
	if (change == KVM_MR_DELETE) {
893
		new.dirty_bitmap = NULL;
894
		memset(&new.arch, 0, sizeof(new.arch));
895 896
	}

897
	old_memslots = install_new_memslots(kvm, slots, &new);
898

899
	kvm_arch_commit_memory_region(kvm, mem, old);
900

901 902 903
	kvm_free_physmem_slot(&old, &new);
	kfree(old_memslots);

A
Avi Kivity 已提交
904 905
	return 0;

906 907
out_slots:
	kfree(slots);
908
out_free:
A
Avi Kivity 已提交
909 910 911
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
912
}
913 914 915
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
916
			  struct kvm_userspace_memory_region *mem)
917 918 919
{
	int r;

920
	mutex_lock(&kvm->slots_lock);
921
	r = __kvm_set_memory_region(kvm, mem);
922
	mutex_unlock(&kvm->slots_lock);
923 924
	return r;
}
925 926
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

927
int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
928
				   struct kvm_userspace_memory_region *mem)
929
{
930
	if (mem->slot >= KVM_USER_MEM_SLOTS)
931
		return -EINVAL;
932
	return kvm_set_memory_region(kvm, mem);
A
Avi Kivity 已提交
933 934
}

935 936
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
937 938 939
{
	struct kvm_memory_slot *memslot;
	int r, i;
940
	unsigned long n;
A
Avi Kivity 已提交
941 942 943
	unsigned long any = 0;

	r = -EINVAL;
944
	if (log->slot >= KVM_USER_MEM_SLOTS)
A
Avi Kivity 已提交
945 946
		goto out;

947
	memslot = id_to_memslot(kvm->memslots, log->slot);
A
Avi Kivity 已提交
948 949 950 951
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

952
	n = kvm_dirty_bitmap_bytes(memslot);
A
Avi Kivity 已提交
953

954
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
955 956 957 958 959 960
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

961 962
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
963 964 965 966 967 968

	r = 0;
out:
	return r;
}

969 970 971 972 973
bool kvm_largepages_enabled(void)
{
	return largepages_enabled;
}

974 975 976 977 978 979
void kvm_disable_largepages(void)
{
	largepages_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_largepages);

980 981 982 983
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
A
Avi Kivity 已提交
984
EXPORT_SYMBOL_GPL(gfn_to_memslot);
A
Avi Kivity 已提交
985

986 987
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
988
	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
989

990
	if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
991 992
	      memslot->flags & KVM_MEMSLOT_INVALID)
		return 0;
993

994
	return 1;
995 996 997
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

J
Joerg Roedel 已提交
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr, size;

	size = PAGE_SIZE;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return PAGE_SIZE;

	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, addr);
	if (!vma)
		goto out;

	size = vma_kernel_pagesize(vma);

out:
	up_read(&current->mm->mmap_sem);

	return size;
}

X
Xiao Guangrong 已提交
1022 1023 1024 1025 1026 1027 1028
static bool memslot_is_readonly(struct kvm_memory_slot *slot)
{
	return slot->flags & KVM_MEM_READONLY;
}

static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				       gfn_t *nr_pages, bool write)
I
Izik Eidus 已提交
1029
{
1030
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
X
Xiao Guangrong 已提交
1031
		return KVM_HVA_ERR_BAD;
1032

X
Xiao Guangrong 已提交
1033 1034
	if (memslot_is_readonly(slot) && write)
		return KVM_HVA_ERR_RO_BAD;
1035 1036 1037 1038

	if (nr_pages)
		*nr_pages = slot->npages - (gfn - slot->base_gfn);

X
Xiao Guangrong 已提交
1039
	return __gfn_to_hva_memslot(slot, gfn);
I
Izik Eidus 已提交
1040
}
1041

X
Xiao Guangrong 已提交
1042 1043 1044 1045
static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
				     gfn_t *nr_pages)
{
	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
I
Izik Eidus 已提交
1046
}
1047

X
Xiao Guangrong 已提交
1048 1049 1050 1051 1052 1053 1054
unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
				 gfn_t gfn)
{
	return gfn_to_hva_many(slot, gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);

1055 1056
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
1057
	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1058
}
1059
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1060

1061 1062 1063 1064 1065 1066
/*
 * The hva returned by this function is only allowed to be read.
 * It should pair with kvm_read_hva() or kvm_read_hva_atomic().
 */
static unsigned long gfn_to_hva_read(struct kvm *kvm, gfn_t gfn)
{
X
Xiao Guangrong 已提交
1067
	return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false);
1068 1069 1070
}

static int kvm_read_hva(void *data, void __user *hva, int len)
1071
{
1072 1073 1074 1075 1076 1077
	return __copy_from_user(data, hva, len);
}

static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
{
	return __copy_from_user_inatomic(data, hva, len);
1078 1079
}

1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
	unsigned long start, int write, struct page **page)
{
	int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;

	if (write)
		flags |= FOLL_WRITE;

	return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
}

1091 1092 1093 1094 1095 1096 1097 1098 1099
static inline int check_user_page_hwpoison(unsigned long addr)
{
	int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;

	rc = __get_user_pages(current, current->mm, addr, 1,
			      flags, NULL, NULL, NULL);
	return rc == -EHWPOISON;
}

X
Xiao Guangrong 已提交
1100 1101 1102 1103 1104 1105
/*
 * The atomic path to get the writable pfn which will be stored in @pfn,
 * true indicates success, otherwise false is returned.
 */
static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
			    bool write_fault, bool *writable, pfn_t *pfn)
A
Avi Kivity 已提交
1106
{
1107
	struct page *page[1];
X
Xiao Guangrong 已提交
1108
	int npages;
A
Avi Kivity 已提交
1109

X
Xiao Guangrong 已提交
1110 1111
	if (!(async || atomic))
		return false;
1112

1113 1114 1115 1116 1117 1118 1119
	/*
	 * Fast pin a writable pfn only if it is a write fault request
	 * or the caller allows to map a writable pfn for a read fault
	 * request.
	 */
	if (!(write_fault || writable))
		return false;
1120

X
Xiao Guangrong 已提交
1121 1122 1123
	npages = __get_user_pages_fast(addr, 1, 1, page);
	if (npages == 1) {
		*pfn = page_to_pfn(page[0]);
1124

X
Xiao Guangrong 已提交
1125 1126 1127 1128
		if (writable)
			*writable = true;
		return true;
	}
1129

X
Xiao Guangrong 已提交
1130 1131
	return false;
}
1132

X
Xiao Guangrong 已提交
1133 1134 1135 1136 1137 1138 1139 1140 1141
/*
 * The slow path to get the pfn of the specified host virtual address,
 * 1 indicates success, -errno is returned if error is detected.
 */
static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
			   bool *writable, pfn_t *pfn)
{
	struct page *page[1];
	int npages = 0;
1142

X
Xiao Guangrong 已提交
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
	might_sleep();

	if (writable)
		*writable = write_fault;

	if (async) {
		down_read(&current->mm->mmap_sem);
		npages = get_user_page_nowait(current, current->mm,
					      addr, write_fault, page);
		up_read(&current->mm->mmap_sem);
	} else
		npages = get_user_pages_fast(addr, 1, write_fault,
					     page);
	if (npages != 1)
		return npages;

	/* map read fault as writable if possible */
1160
	if (unlikely(!write_fault) && writable) {
X
Xiao Guangrong 已提交
1161 1162 1163 1164 1165 1166 1167
		struct page *wpage[1];

		npages = __get_user_pages_fast(addr, 1, 1, wpage);
		if (npages == 1) {
			*writable = true;
			put_page(page[0]);
			page[0] = wpage[0];
1168
		}
X
Xiao Guangrong 已提交
1169 1170

		npages = 1;
1171
	}
X
Xiao Guangrong 已提交
1172 1173 1174
	*pfn = page_to_pfn(page[0]);
	return npages;
}
I
Izik Eidus 已提交
1175

X
Xiao Guangrong 已提交
1176 1177 1178 1179
static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
{
	if (unlikely(!(vma->vm_flags & VM_READ)))
		return false;
1180

X
Xiao Guangrong 已提交
1181 1182
	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
		return false;
1183

X
Xiao Guangrong 已提交
1184 1185
	return true;
}
1186

1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200
/*
 * Pin guest page in memory and return its pfn.
 * @addr: host virtual address which maps memory to the guest
 * @atomic: whether this function can sleep
 * @async: whether this function need to wait IO complete if the
 *         host page is not in the memory
 * @write_fault: whether we should get a writable host page
 * @writable: whether it allows to map a writable host page for !@write_fault
 *
 * The function will map a writable host page for these two cases:
 * 1): @write_fault = true
 * 2): @write_fault = false && @writable, @writable will tell the caller
 *     whether the mapping is writable.
 */
X
Xiao Guangrong 已提交
1201 1202 1203 1204 1205 1206
static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
			bool write_fault, bool *writable)
{
	struct vm_area_struct *vma;
	pfn_t pfn = 0;
	int npages;
1207

X
Xiao Guangrong 已提交
1208 1209
	/* we can do it either atomically or asynchronously, not both */
	BUG_ON(atomic && async);
1210

X
Xiao Guangrong 已提交
1211 1212 1213 1214 1215 1216 1217 1218 1219
	if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
		return pfn;

	if (atomic)
		return KVM_PFN_ERR_FAULT;

	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
	if (npages == 1)
		return pfn;
1220

X
Xiao Guangrong 已提交
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
	down_read(&current->mm->mmap_sem);
	if (npages == -EHWPOISON ||
	      (!async && check_user_page_hwpoison(addr))) {
		pfn = KVM_PFN_ERR_HWPOISON;
		goto exit;
	}

	vma = find_vma_intersection(current->mm, addr, addr + 1);

	if (vma == NULL)
		pfn = KVM_PFN_ERR_FAULT;
	else if ((vma->vm_flags & VM_PFNMAP)) {
		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
			vma->vm_pgoff;
		BUG_ON(!kvm_is_mmio_pfn(pfn));
	} else {
X
Xiao Guangrong 已提交
1237
		if (async && vma_is_valid(vma, write_fault))
X
Xiao Guangrong 已提交
1238 1239 1240 1241 1242
			*async = true;
		pfn = KVM_PFN_ERR_FAULT;
	}
exit:
	up_read(&current->mm->mmap_sem);
1243
	return pfn;
1244 1245
}

X
Xiao Guangrong 已提交
1246 1247 1248
static pfn_t
__gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
		     bool *async, bool write_fault, bool *writable)
1249
{
X
Xiao Guangrong 已提交
1250 1251 1252 1253 1254 1255
	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);

	if (addr == KVM_HVA_ERR_RO_BAD)
		return KVM_PFN_ERR_RO_FAULT;

	if (kvm_is_error_hva(addr))
1256
		return KVM_PFN_NOSLOT;
X
Xiao Guangrong 已提交
1257 1258 1259 1260 1261 1262 1263 1264 1265

	/* Do not map writable pfn in the readonly memslot. */
	if (writable && memslot_is_readonly(slot)) {
		*writable = false;
		writable = NULL;
	}

	return hva_to_pfn(addr, atomic, async, write_fault,
			  writable);
1266 1267
}

1268 1269
static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
			  bool write_fault, bool *writable)
1270
{
X
Xiao Guangrong 已提交
1271
	struct kvm_memory_slot *slot;
1272

1273 1274 1275
	if (async)
		*async = false;

X
Xiao Guangrong 已提交
1276
	slot = gfn_to_memslot(kvm, gfn);
1277

X
Xiao Guangrong 已提交
1278 1279
	return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
				    writable);
1280 1281 1282 1283
}

pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
{
1284
	return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1285 1286 1287
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);

1288 1289
pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
		       bool write_fault, bool *writable)
1290
{
1291
	return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1292 1293 1294
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_async);

1295 1296
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
{
1297
	return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1298
}
1299 1300
EXPORT_SYMBOL_GPL(gfn_to_pfn);

1301 1302 1303 1304 1305 1306 1307
pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
		      bool *writable)
{
	return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
}
EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);

1308
pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1309
{
X
Xiao Guangrong 已提交
1310
	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1311 1312
}

1313
pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1314
{
X
Xiao Guangrong 已提交
1315
	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1316
}
1317
EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1318

1319 1320 1321 1322 1323 1324
int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
								  int nr_pages)
{
	unsigned long addr;
	gfn_t entry;

1325
	addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
	if (kvm_is_error_hva(addr))
		return -1;

	if (entry < nr_pages)
		return 0;

	return __get_user_pages_fast(addr, nr_pages, 1, pages);
}
EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);

X
Xiao Guangrong 已提交
1336 1337
static struct page *kvm_pfn_to_page(pfn_t pfn)
{
1338
	if (is_error_noslot_pfn(pfn))
1339
		return KVM_ERR_PTR_BAD_PAGE;
X
Xiao Guangrong 已提交
1340

1341 1342
	if (kvm_is_mmio_pfn(pfn)) {
		WARN_ON(1);
1343
		return KVM_ERR_PTR_BAD_PAGE;
1344
	}
X
Xiao Guangrong 已提交
1345 1346 1347 1348

	return pfn_to_page(pfn);
}

1349 1350
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
1351 1352 1353 1354
	pfn_t pfn;

	pfn = gfn_to_pfn(kvm, gfn);

X
Xiao Guangrong 已提交
1355
	return kvm_pfn_to_page(pfn);
A
Avi Kivity 已提交
1356
}
1357

A
Avi Kivity 已提交
1358 1359
EXPORT_SYMBOL_GPL(gfn_to_page);

1360 1361
void kvm_release_page_clean(struct page *page)
{
1362 1363
	WARN_ON(is_error_page(page));

1364
	kvm_release_pfn_clean(page_to_pfn(page));
1365 1366 1367
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

1368 1369
void kvm_release_pfn_clean(pfn_t pfn)
{
1370
	if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn))
1371
		put_page(pfn_to_page(pfn));
1372 1373 1374
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1375
void kvm_release_page_dirty(struct page *page)
1376
{
X
Xiao Guangrong 已提交
1377 1378
	WARN_ON(is_error_page(page));

1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

void kvm_release_pfn_dirty(pfn_t pfn)
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);

void kvm_set_page_dirty(struct page *page)
{
	kvm_set_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_set_page_dirty);

void kvm_set_pfn_dirty(pfn_t pfn)
{
1398
	if (!kvm_is_mmio_pfn(pfn)) {
1399 1400 1401 1402
		struct page *page = pfn_to_page(pfn);
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1403
}
1404 1405 1406 1407
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
1408
	if (!kvm_is_mmio_pfn(pfn))
1409
		mark_page_accessed(pfn_to_page(pfn));
1410 1411 1412 1413 1414
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
1415
	if (!kvm_is_mmio_pfn(pfn))
1416
		get_page(pfn_to_page(pfn));
1417 1418
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1419

1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
1431 1432
	int r;
	unsigned long addr;
1433

1434
	addr = gfn_to_hva_read(kvm, gfn);
1435 1436
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1437
	r = kvm_read_hva(data, (void __user *)addr + offset, len);
1438
	if (r)
1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1464 1465 1466 1467 1468 1469 1470 1471
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

1472
	addr = gfn_to_hva_read(kvm, gfn);
1473 1474
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1475
	pagefault_disable();
1476
	r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
1477
	pagefault_enable();
1478 1479 1480 1481 1482 1483
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

1484 1485 1486
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
1487 1488
	int r;
	unsigned long addr;
1489

1490 1491 1492
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1493
	r = __copy_to_user((void __user *)addr + offset, data, len);
1494
	if (r)
1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

1521 1522 1523 1524 1525 1526 1527 1528 1529
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			      gpa_t gpa)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int offset = offset_in_page(gpa);
	gfn_t gfn = gpa >> PAGE_SHIFT;

	ghc->gpa = gpa;
	ghc->generation = slots->generation;
1530
	ghc->memslot = gfn_to_memslot(kvm, gfn);
1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552
	ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
	if (!kvm_is_error_hva(ghc->hva))
		ghc->hva += offset;
	else
		return -EFAULT;

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);

int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

	if (slots->generation != ghc->generation)
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

1553
	r = __copy_to_user((void __user *)ghc->hva, data, len);
1554 1555 1556 1557 1558 1559 1560 1561
	if (r)
		return -EFAULT;
	mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);

1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len)
{
	struct kvm_memslots *slots = kvm_memslots(kvm);
	int r;

	if (slots->generation != ghc->generation)
		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);

	if (kvm_is_error_hva(ghc->hva))
		return -EFAULT;

	r = __copy_from_user(data, (void __user *)ghc->hva, len);
	if (r)
		return -EFAULT;

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_cached);

1582 1583
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
1584 1585
	return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
				    offset, len);
1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

1608 1609
void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
			     gfn_t gfn)
A
Avi Kivity 已提交
1610
{
R
Rusty Russell 已提交
1611 1612
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1613

1614
		set_bit_le(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1615 1616 1617
	}
}

1618 1619 1620 1621 1622 1623 1624 1625
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	memslot = gfn_to_memslot(kvm, gfn);
	mark_page_dirty_in_slot(kvm, memslot, gfn);
}

E
Eddie Dong 已提交
1626 1627 1628
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1629
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1630
{
1631 1632 1633 1634 1635
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

1636
		if (kvm_arch_vcpu_runnable(vcpu)) {
1637
			kvm_make_request(KVM_REQ_UNHALT, vcpu);
1638
			break;
1639
		}
1640 1641
		if (kvm_cpu_has_pending_timer(vcpu))
			break;
1642 1643 1644
		if (signal_pending(current))
			break;

E
Eddie Dong 已提交
1645 1646
		schedule();
	}
1647

1648
	finish_wait(&vcpu->wq, &wait);
E
Eddie Dong 已提交
1649 1650
}

1651
#ifndef CONFIG_S390
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672
/*
 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
 */
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
	int me;
	int cpu = vcpu->cpu;
	wait_queue_head_t *wqp;

	wqp = kvm_arch_vcpu_wq(vcpu);
	if (waitqueue_active(wqp)) {
		wake_up_interruptible(wqp);
		++vcpu->stat.halt_wakeup;
	}

	me = get_cpu();
	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
		if (kvm_arch_vcpu_should_kick(vcpu))
			smp_send_reschedule(cpu);
	put_cpu();
}
1673
#endif /* !CONFIG_S390 */
1674

A
Avi Kivity 已提交
1675 1676
void kvm_resched(struct kvm_vcpu *vcpu)
{
1677 1678
	if (!need_resched())
		return;
A
Avi Kivity 已提交
1679 1680 1681 1682
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

1683 1684 1685 1686
bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
{
	struct pid *pid;
	struct task_struct *task = NULL;
1687
	bool ret = false;
1688 1689 1690 1691 1692 1693 1694

	rcu_read_lock();
	pid = rcu_dereference(target->pid);
	if (pid)
		task = get_pid_task(target->pid, PIDTYPE_PID);
	rcu_read_unlock();
	if (!task)
1695
		return ret;
1696 1697
	if (task->flags & PF_VCPU) {
		put_task_struct(task);
1698
		return ret;
1699
	}
1700
	ret = yield_to(task, 1);
1701
	put_task_struct(task);
1702 1703

	return ret;
1704 1705 1706
}
EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);

1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
/*
 * Helper that checks whether a VCPU is eligible for directed yield.
 * Most eligible candidate to yield is decided by following heuristics:
 *
 *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
 *  (preempted lock holder), indicated by @in_spin_loop.
 *  Set at the beiginning and cleared at the end of interception/PLE handler.
 *
 *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
 *  chance last time (mostly it has become eligible now since we have probably
 *  yielded to lockholder in last iteration. This is done by toggling
 *  @dy_eligible each time a VCPU checked for eligibility.)
 *
 *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
 *  to preempted lock-holder could result in wrong VCPU selection and CPU
 *  burning. Giving priority for a potential lock-holder increases lock
 *  progress.
 *
 *  Since algorithm is based on heuristics, accessing another VCPU data without
 *  locking does not harm. It may result in trying to yield to  same VCPU, fail
 *  and continue with next VCPU and so on.
 */
bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
{
	bool eligible;

	eligible = !vcpu->spin_loop.in_spin_loop ||
			(vcpu->spin_loop.in_spin_loop &&
			 vcpu->spin_loop.dy_eligible);

	if (vcpu->spin_loop.in_spin_loop)
		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);

	return eligible;
}
#endif
1744

1745
void kvm_vcpu_on_spin(struct kvm_vcpu *me)
Z
Zhai, Edwin 已提交
1746
{
1747 1748 1749 1750
	struct kvm *kvm = me->kvm;
	struct kvm_vcpu *vcpu;
	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
	int yielded = 0;
1751
	int try = 3;
1752 1753
	int pass;
	int i;
Z
Zhai, Edwin 已提交
1754

1755
	kvm_vcpu_set_in_spin_loop(me, true);
1756 1757 1758 1759 1760 1761 1762
	/*
	 * We boost the priority of a VCPU that is runnable but not
	 * currently running, because it got preempted by something
	 * else and called schedule in __vcpu_run.  Hopefully that
	 * VCPU is holding the lock that we need and will release it.
	 * We approximate round-robin by starting at the last boosted VCPU.
	 */
1763
	for (pass = 0; pass < 2 && !yielded && try; pass++) {
1764
		kvm_for_each_vcpu(i, vcpu, kvm) {
1765
			if (!pass && i <= last_boosted_vcpu) {
1766 1767 1768 1769 1770 1771 1772 1773
				i = last_boosted_vcpu;
				continue;
			} else if (pass && i > last_boosted_vcpu)
				break;
			if (vcpu == me)
				continue;
			if (waitqueue_active(&vcpu->wq))
				continue;
1774 1775
			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
				continue;
1776 1777 1778

			yielded = kvm_vcpu_yield_to(vcpu);
			if (yielded > 0) {
1779 1780
				kvm->last_boosted_vcpu = i;
				break;
1781 1782 1783 1784
			} else if (yielded < 0) {
				try--;
				if (!try)
					break;
1785 1786 1787
			}
		}
	}
1788
	kvm_vcpu_set_in_spin_loop(me, false);
1789 1790 1791

	/* Ensure vcpu is not eligible during next spinloop */
	kvm_vcpu_set_dy_eligible(me, false);
Z
Zhai, Edwin 已提交
1792 1793 1794
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

1795
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1796 1797 1798 1799
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

1800
	if (vmf->pgoff == 0)
1801
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
1802
#ifdef CONFIG_X86
1803
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1804
		page = virt_to_page(vcpu->arch.pio_data);
1805 1806 1807 1808
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
1809
#endif
1810
	else
1811
		return kvm_arch_vcpu_fault(vcpu, vmf);
1812
	get_page(page);
1813 1814
	vmf->page = page;
	return 0;
1815 1816
}

1817
static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1818
	.fault = kvm_vcpu_fault,
1819 1820 1821 1822 1823 1824 1825 1826
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
1827 1828 1829 1830
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
1831
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1832 1833 1834
	return 0;
}

1835
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
1836 1837
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
1838 1839 1840
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vcpu_compat_ioctl,
#endif
1841
	.mmap           = kvm_vcpu_mmap,
1842
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
1843 1844 1845 1846 1847 1848 1849
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
1850
	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
A
Avi Kivity 已提交
1851 1852
}

1853 1854 1855
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
1856
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1857 1858
{
	int r;
1859
	struct kvm_vcpu *vcpu, *v;
1860

1861
	vcpu = kvm_arch_vcpu_create(kvm, id);
R
Rusty Russell 已提交
1862 1863
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
1864

1865 1866
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

1867 1868
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
1869
		goto vcpu_destroy;
1870

S
Shaohua Li 已提交
1871
	mutex_lock(&kvm->lock);
1872 1873 1874 1875
	if (!kvm_vcpu_compatible(vcpu)) {
		r = -EINVAL;
		goto unlock_vcpu_destroy;
	}
1876 1877
	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
		r = -EINVAL;
1878
		goto unlock_vcpu_destroy;
R
Rusty Russell 已提交
1879
	}
1880

1881 1882
	kvm_for_each_vcpu(r, v, kvm)
		if (v->vcpu_id == id) {
1883
			r = -EEXIST;
1884
			goto unlock_vcpu_destroy;
1885 1886 1887
		}

	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1888

R
Rusty Russell 已提交
1889
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
1890
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
1891
	r = create_vcpu_fd(vcpu);
1892 1893
	if (r < 0) {
		kvm_put_kvm(kvm);
1894
		goto unlock_vcpu_destroy;
1895 1896 1897 1898 1899 1900 1901
	}

	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
	smp_wmb();
	atomic_inc(&kvm->online_vcpus);

	mutex_unlock(&kvm->lock);
1902
	kvm_arch_vcpu_postcreate(vcpu);
R
Rusty Russell 已提交
1903
	return r;
1904

1905
unlock_vcpu_destroy:
1906
	mutex_unlock(&kvm->lock);
1907
vcpu_destroy:
1908
	kvm_arch_vcpu_destroy(vcpu);
1909 1910 1911
	return r;
}

A
Avi Kivity 已提交
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
1923 1924
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
1925
{
A
Avi Kivity 已提交
1926
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
1927
	void __user *argp = (void __user *)arg;
1928
	int r;
1929 1930
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
1931

1932 1933
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944

#if defined(CONFIG_S390) || defined(CONFIG_PPC)
	/*
	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
	 * so vcpu_load() would break it.
	 */
	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
#endif


1945 1946 1947
	r = vcpu_load(vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
1948
	switch (ioctl) {
1949
	case KVM_RUN:
1950 1951 1952
		r = -EINVAL;
		if (arg)
			goto out;
1953
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1954
		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
A
Avi Kivity 已提交
1955 1956
		break;
	case KVM_GET_REGS: {
1957
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1958

1959 1960 1961
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1962
			goto out;
1963 1964 1965
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
1966
		r = -EFAULT;
1967 1968
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
1969
		r = 0;
1970 1971
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
1972 1973 1974
		break;
	}
	case KVM_SET_REGS: {
1975
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1976

1977
		r = -ENOMEM;
1978 1979 1980
		kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
		if (IS_ERR(kvm_regs)) {
			r = PTR_ERR(kvm_regs);
A
Avi Kivity 已提交
1981
			goto out;
1982
		}
1983 1984
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
		kfree(kvm_regs);
A
Avi Kivity 已提交
1985 1986 1987
		break;
	}
	case KVM_GET_SREGS: {
1988 1989 1990 1991 1992
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
1993 1994 1995
		if (r)
			goto out;
		r = -EFAULT;
1996
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
1997 1998 1999 2000 2001
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
2002 2003 2004
		kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
		if (IS_ERR(kvm_sregs)) {
			r = PTR_ERR(kvm_sregs);
G
Guo Chao 已提交
2005
			kvm_sregs = NULL;
A
Avi Kivity 已提交
2006
			goto out;
2007
		}
2008
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
2009 2010
		break;
	}
2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		break;
	}
A
Avi Kivity 已提交
2032 2033 2034 2035
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
2036
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
2037
			goto out;
2038
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2039 2040 2041
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2042
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
2043 2044 2045 2046
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
2047 2048
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
2049 2050

		r = -EFAULT;
A
Al Viro 已提交
2051
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
2052
			goto out;
J
Jan Kiszka 已提交
2053
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
2054 2055
		break;
	}
A
Avi Kivity 已提交
2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
2076
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
A
Avi Kivity 已提交
2077 2078
		break;
	}
A
Avi Kivity 已提交
2079
	case KVM_GET_FPU: {
2080 2081 2082 2083 2084
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2085 2086 2087
		if (r)
			goto out;
		r = -EFAULT;
2088
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
2089 2090 2091 2092 2093
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
2094 2095 2096
		fpu = memdup_user(argp, sizeof(*fpu));
		if (IS_ERR(fpu)) {
			r = PTR_ERR(fpu);
G
Guo Chao 已提交
2097
			fpu = NULL;
A
Avi Kivity 已提交
2098
			goto out;
2099
		}
2100
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
2101 2102
		break;
	}
A
Avi Kivity 已提交
2103
	default:
2104
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2105 2106
	}
out:
2107
	vcpu_put(vcpu);
2108 2109
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
2110 2111 2112
	return r;
}

2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142
#ifdef CONFIG_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *filp,
				  unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = compat_ptr(arg);
	int r;

	if (vcpu->kvm->mm != current->mm)
		return -EIO;

	switch (ioctl) {
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		compat_sigset_t csigset;
		sigset_t sigset;

		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof csigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&csigset, sigmask_arg->sigset,
					   sizeof csigset))
				goto out;
2143 2144 2145 2146
			sigset_from_compat(&sigset, &csigset);
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		} else
			r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157
		break;
	}
	default:
		r = kvm_vcpu_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

A
Avi Kivity 已提交
2158 2159 2160 2161 2162
static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
2163
	int r;
A
Avi Kivity 已提交
2164

2165 2166
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
2167 2168 2169 2170
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		break;
2171 2172 2173 2174 2175 2176 2177 2178
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

2179
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
A
Avi Kivity 已提交
2180 2181 2182 2183 2184 2185
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
2186
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
2187
			goto out;
2188
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
2189 2190
		break;
	}
2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		break;
	}
#endif
G
Gregory Haskins 已提交
2209 2210 2211 2212 2213 2214
	case KVM_IRQFD: {
		struct kvm_irqfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
2215
		r = kvm_irqfd(kvm, &data);
G
Gregory Haskins 已提交
2216 2217
		break;
	}
G
Gregory Haskins 已提交
2218 2219 2220 2221 2222 2223 2224 2225 2226
	case KVM_IOEVENTFD: {
		struct kvm_ioeventfd data;

		r = -EFAULT;
		if (copy_from_user(&data, argp, sizeof data))
			goto out;
		r = kvm_ioeventfd(kvm, &data);
		break;
	}
2227 2228 2229
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
2230
		mutex_lock(&kvm->lock);
2231 2232 2233 2234
		if (atomic_read(&kvm->online_vcpus) != 0)
			r = -EBUSY;
		else
			kvm->bsp_vcpu_id = arg;
2235
		mutex_unlock(&kvm->lock);
2236
		break;
2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247
#endif
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_SIGNAL_MSI: {
		struct kvm_msi msi;

		r = -EFAULT;
		if (copy_from_user(&msi, argp, sizeof msi))
			goto out;
		r = kvm_send_userspace_msi(kvm, &msi);
		break;
	}
2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270
#endif
#ifdef __KVM_HAVE_IRQ_LINE
	case KVM_IRQ_LINE_STATUS:
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
		if (copy_from_user(&irq_event, argp, sizeof irq_event))
			goto out;

		r = kvm_vm_ioctl_irq_line(kvm, &irq_event);
		if (r)
			goto out;

		r = -EFAULT;
		if (ioctl == KVM_IRQ_LINE_STATUS) {
			if (copy_to_user(argp, &irq_event, sizeof irq_event))
				goto out;
		}

		r = 0;
		break;
	}
2271
#endif
2272
	default:
2273
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2274 2275
		if (r == -ENOTTY)
			r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2276 2277 2278 2279 2280
	}
out:
	return r;
}

2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324
#ifdef CONFIG_COMPAT
struct compat_kvm_dirty_log {
	__u32 slot;
	__u32 padding1;
	union {
		compat_uptr_t dirty_bitmap; /* one bit per page */
		__u64 padding2;
	};
};

static long kvm_vm_compat_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	int r;

	if (kvm->mm != current->mm)
		return -EIO;
	switch (ioctl) {
	case KVM_GET_DIRTY_LOG: {
		struct compat_kvm_dirty_log compat_log;
		struct kvm_dirty_log log;

		r = -EFAULT;
		if (copy_from_user(&compat_log, (void __user *)arg,
				   sizeof(compat_log)))
			goto out;
		log.slot	 = compat_log.slot;
		log.padding1	 = compat_log.padding1;
		log.padding2	 = compat_log.padding2;
		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
		break;
	}
	default:
		r = kvm_vm_ioctl(filp, ioctl, arg);
	}

out:
	return r;
}
#endif

2325
static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2326
{
2327 2328 2329 2330
	struct page *page[1];
	unsigned long addr;
	int npages;
	gfn_t gfn = vmf->pgoff;
2331 2332
	struct kvm *kvm = vma->vm_file->private_data;

2333 2334
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
2335
		return VM_FAULT_SIGBUS;
2336 2337 2338 2339

	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
				NULL);
	if (unlikely(npages != 1))
2340
		return VM_FAULT_SIGBUS;
2341 2342

	vmf->page = page[0];
2343
	return 0;
2344 2345
}

2346
static const struct vm_operations_struct kvm_vm_vm_ops = {
2347
	.fault = kvm_vm_fault,
2348 2349 2350 2351 2352 2353 2354 2355
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

2356
static struct file_operations kvm_vm_fops = {
2357 2358
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
2359 2360 2361
#ifdef CONFIG_COMPAT
	.compat_ioctl   = kvm_vm_compat_ioctl,
#endif
2362
	.mmap           = kvm_vm_mmap,
2363
	.llseek		= noop_llseek,
2364 2365
};

2366
static int kvm_dev_ioctl_create_vm(unsigned long type)
2367
{
2368
	int r;
2369 2370
	struct kvm *kvm;

2371
	kvm = kvm_create_vm(type);
2372 2373
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
2374 2375 2376 2377 2378 2379 2380
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	r = kvm_coalesced_mmio_init(kvm);
	if (r < 0) {
		kvm_put_kvm(kvm);
		return r;
	}
#endif
2381 2382
	r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
	if (r < 0)
A
Al Viro 已提交
2383
		kvm_put_kvm(kvm);
2384

2385
	return r;
2386 2387
}

2388 2389 2390
static long kvm_dev_ioctl_check_extension_generic(long arg)
{
	switch (arg) {
2391
	case KVM_CAP_USER_MEMORY:
2392
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2393
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2394 2395 2396
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	case KVM_CAP_SET_BOOT_CPU_ID:
#endif
2397
	case KVM_CAP_INTERNAL_ERROR_DATA:
2398 2399 2400
#ifdef CONFIG_HAVE_KVM_MSI
	case KVM_CAP_SIGNAL_MSI:
#endif
2401
		return 1;
2402
#ifdef KVM_CAP_IRQ_ROUTING
2403
	case KVM_CAP_IRQ_ROUTING:
2404
		return KVM_MAX_IRQ_ROUTES;
2405
#endif
2406 2407 2408 2409 2410 2411
	default:
		break;
	}
	return kvm_dev_ioctl_check_extension(arg);
}

2412 2413 2414
static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
2415
	long r = -EINVAL;
2416 2417 2418

	switch (ioctl) {
	case KVM_GET_API_VERSION:
2419 2420 2421
		r = -EINVAL;
		if (arg)
			goto out;
2422 2423 2424
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
2425
		r = kvm_dev_ioctl_create_vm(arg);
2426
		break;
2427
	case KVM_CHECK_EXTENSION:
2428
		r = kvm_dev_ioctl_check_extension_generic(arg);
2429
		break;
2430 2431 2432 2433
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
2434 2435 2436
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
2437 2438 2439
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
		r += PAGE_SIZE;    /* coalesced mmio ring page */
2440
#endif
2441
		break;
2442 2443 2444
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
2445
		r = -EOPNOTSUPP;
2446
		break;
A
Avi Kivity 已提交
2447
	default:
2448
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2449 2450 2451 2452 2453 2454 2455 2456
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
2457
	.llseek		= noop_llseek,
A
Avi Kivity 已提交
2458 2459 2460
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
2461
	KVM_MINOR,
A
Avi Kivity 已提交
2462 2463 2464 2465
	"kvm",
	&kvm_chardev_ops,
};

2466
static void hardware_enable_nolock(void *junk)
2467 2468
{
	int cpu = raw_smp_processor_id();
2469
	int r;
2470

2471
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2472
		return;
2473

2474
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
2475 2476 2477 2478 2479 2480 2481 2482 2483

	r = kvm_arch_hardware_enable(NULL);

	if (r) {
		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
		atomic_inc(&hardware_enable_failed);
		printk(KERN_INFO "kvm: enabling virtualization on "
				 "CPU%d failed\n", cpu);
	}
2484 2485
}

2486 2487
static void hardware_enable(void *junk)
{
2488
	raw_spin_lock(&kvm_lock);
2489
	hardware_enable_nolock(junk);
2490
	raw_spin_unlock(&kvm_lock);
2491 2492 2493
}

static void hardware_disable_nolock(void *junk)
2494 2495 2496
{
	int cpu = raw_smp_processor_id();

2497
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2498
		return;
2499
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2500
	kvm_arch_hardware_disable(NULL);
2501 2502
}

2503 2504
static void hardware_disable(void *junk)
{
2505
	raw_spin_lock(&kvm_lock);
2506
	hardware_disable_nolock(junk);
2507
	raw_spin_unlock(&kvm_lock);
2508 2509
}

2510 2511 2512 2513 2514 2515
static void hardware_disable_all_nolock(void)
{
	BUG_ON(!kvm_usage_count);

	kvm_usage_count--;
	if (!kvm_usage_count)
2516
		on_each_cpu(hardware_disable_nolock, NULL, 1);
2517 2518 2519 2520
}

static void hardware_disable_all(void)
{
2521
	raw_spin_lock(&kvm_lock);
2522
	hardware_disable_all_nolock();
2523
	raw_spin_unlock(&kvm_lock);
2524 2525 2526 2527 2528 2529
}

static int hardware_enable_all(void)
{
	int r = 0;

2530
	raw_spin_lock(&kvm_lock);
2531 2532 2533 2534

	kvm_usage_count++;
	if (kvm_usage_count == 1) {
		atomic_set(&hardware_enable_failed, 0);
2535
		on_each_cpu(hardware_enable_nolock, NULL, 1);
2536 2537 2538 2539 2540 2541 2542

		if (atomic_read(&hardware_enable_failed)) {
			hardware_disable_all_nolock();
			r = -EBUSY;
		}
	}

2543
	raw_spin_unlock(&kvm_lock);
2544 2545 2546 2547

	return r;
}

A
Avi Kivity 已提交
2548 2549 2550 2551 2552
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

2553 2554 2555
	if (!kvm_usage_count)
		return NOTIFY_OK;

2556
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
2557
	switch (val) {
2558
	case CPU_DYING:
2559 2560 2561 2562
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
2563
	case CPU_STARTING:
2564 2565
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
2566
		hardware_enable(NULL);
A
Avi Kivity 已提交
2567 2568 2569 2570 2571
		break;
	}
	return NOTIFY_OK;
}

2572

2573
asmlinkage void kvm_spurious_fault(void)
2574 2575 2576 2577
{
	/* Fault while not rebooting.  We want the trace. */
	BUG();
}
2578
EXPORT_SYMBOL_GPL(kvm_spurious_fault);
2579

2580
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
2581
		      void *v)
2582
{
2583 2584 2585 2586 2587 2588 2589 2590
	/*
	 * Some (well, at least mine) BIOSes hang on reboot if
	 * in vmx root mode.
	 *
	 * And Intel TXT required VMX off for all cpu when system shutdown.
	 */
	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
	kvm_rebooting = true;
2591
	on_each_cpu(hardware_disable_nolock, NULL, 1);
2592 2593 2594 2595 2596 2597 2598 2599
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

M
Marcelo Tosatti 已提交
2600
static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2601 2602 2603 2604
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
2605
		struct kvm_io_device *pos = bus->range[i].dev;
2606 2607 2608

		kvm_iodevice_destructor(pos);
	}
M
Marcelo Tosatti 已提交
2609
	kfree(bus);
2610 2611
}

2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662
int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
{
	const struct kvm_io_range *r1 = p1;
	const struct kvm_io_range *r2 = p2;

	if (r1->addr < r2->addr)
		return -1;
	if (r1->addr + r1->len > r2->addr + r2->len)
		return 1;
	return 0;
}

int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
			  gpa_t addr, int len)
{
	bus->range[bus->dev_count++] = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
		.dev = dev,
	};

	sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
		kvm_io_bus_sort_cmp, NULL);

	return 0;
}

int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
			     gpa_t addr, int len)
{
	struct kvm_io_range *range, key;
	int off;

	key = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};

	range = bsearch(&key, bus->range, bus->dev_count,
			sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
	if (range == NULL)
		return -ENOENT;

	off = range - bus->range;

	while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0)
		off--;

	return off;
}

2663
/* kvm_io_bus_write - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2664
int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2665
		     int len, const void *val)
2666
{
2667
	int idx;
2668
	struct kvm_io_bus *bus;
2669 2670 2671 2672 2673 2674
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
2675 2676

	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2677 2678 2679 2680 2681 2682 2683
	idx = kvm_io_bus_get_first_dev(bus, addr, len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
		kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
		if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val))
2684
			return 0;
2685 2686 2687
		idx++;
	}

2688 2689
	return -EOPNOTSUPP;
}
2690

2691
/* kvm_io_bus_read - called under kvm->slots_lock */
M
Marcelo Tosatti 已提交
2692 2693
int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
		    int len, void *val)
2694
{
2695
	int idx;
2696
	struct kvm_io_bus *bus;
2697 2698 2699 2700 2701 2702
	struct kvm_io_range range;

	range = (struct kvm_io_range) {
		.addr = addr,
		.len = len,
	};
M
Marcelo Tosatti 已提交
2703

2704
	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2705 2706 2707 2708 2709 2710 2711
	idx = kvm_io_bus_get_first_dev(bus, addr, len);
	if (idx < 0)
		return -EOPNOTSUPP;

	while (idx < bus->dev_count &&
		kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
		if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val))
2712
			return 0;
2713 2714 2715
		idx++;
	}

2716
	return -EOPNOTSUPP;
2717 2718
}

2719
/* Caller must hold slots_lock. */
2720 2721
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, struct kvm_io_device *dev)
2722
{
M
Marcelo Tosatti 已提交
2723
	struct kvm_io_bus *new_bus, *bus;
2724

M
Marcelo Tosatti 已提交
2725
	bus = kvm->buses[bus_idx];
2726
	if (bus->dev_count > NR_IOBUS_DEVS - 1)
2727
		return -ENOSPC;
2728

2729 2730
	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
M
Marcelo Tosatti 已提交
2731 2732
	if (!new_bus)
		return -ENOMEM;
2733 2734
	memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
	       sizeof(struct kvm_io_range)));
2735
	kvm_io_bus_insert_dev(new_bus, dev, addr, len);
M
Marcelo Tosatti 已提交
2736 2737 2738
	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
2739 2740 2741 2742

	return 0;
}

2743
/* Caller must hold slots_lock. */
M
Marcelo Tosatti 已提交
2744 2745
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			      struct kvm_io_device *dev)
2746
{
M
Marcelo Tosatti 已提交
2747 2748
	int i, r;
	struct kvm_io_bus *new_bus, *bus;
2749

2750
	bus = kvm->buses[bus_idx];
M
Marcelo Tosatti 已提交
2751
	r = -ENOENT;
2752 2753
	for (i = 0; i < bus->dev_count; i++)
		if (bus->range[i].dev == dev) {
M
Marcelo Tosatti 已提交
2754
			r = 0;
2755 2756
			break;
		}
M
Marcelo Tosatti 已提交
2757

2758
	if (r)
M
Marcelo Tosatti 已提交
2759
		return r;
2760 2761 2762 2763 2764 2765 2766 2767 2768 2769

	new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
			  sizeof(struct kvm_io_range)), GFP_KERNEL);
	if (!new_bus)
		return -ENOMEM;

	memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
	new_bus->dev_count--;
	memcpy(new_bus->range + i, bus->range + i + 1,
	       (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
M
Marcelo Tosatti 已提交
2770 2771 2772 2773 2774

	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	synchronize_srcu_expedited(&kvm->srcu);
	kfree(bus);
	return r;
2775 2776
}

A
Avi Kivity 已提交
2777 2778 2779 2780
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
};

2781
static int vm_stat_get(void *_offset, u64 *val)
2782 2783 2784 2785
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

2786
	*val = 0;
2787
	raw_spin_lock(&kvm_lock);
2788
	list_for_each_entry(kvm, &vm_list, vm_list)
2789
		*val += *(u32 *)((void *)kvm + offset);
2790
	raw_spin_unlock(&kvm_lock);
2791
	return 0;
2792 2793 2794 2795
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

2796
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
2797 2798 2799 2800 2801 2802
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

2803
	*val = 0;
2804
	raw_spin_lock(&kvm_lock);
A
Avi Kivity 已提交
2805
	list_for_each_entry(kvm, &vm_list, vm_list)
2806 2807 2808
		kvm_for_each_vcpu(i, vcpu, kvm)
			*val += *(u32 *)((void *)vcpu + offset);

2809
	raw_spin_unlock(&kvm_lock);
2810
	return 0;
A
Avi Kivity 已提交
2811 2812
}

2813 2814
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

2815
static const struct file_operations *stat_fops[] = {
2816 2817 2818
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
2819

2820
static int kvm_init_debug(void)
A
Avi Kivity 已提交
2821
{
2822
	int r = -EFAULT;
A
Avi Kivity 已提交
2823 2824
	struct kvm_stats_debugfs_item *p;

2825
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
2826 2827 2828 2829
	if (kvm_debugfs_dir == NULL)
		goto out;

	for (p = debugfs_entries; p->name; ++p) {
2830
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
2831
						(void *)(long)p->offset,
2832
						stat_fops[p->kind]);
2833 2834 2835 2836 2837 2838 2839 2840 2841 2842
		if (p->dentry == NULL)
			goto out_dir;
	}

	return 0;

out_dir:
	debugfs_remove_recursive(kvm_debugfs_dir);
out:
	return r;
A
Avi Kivity 已提交
2843 2844 2845 2846 2847 2848 2849 2850
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
2851
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
2852 2853
}

2854
static int kvm_suspend(void)
2855
{
2856
	if (kvm_usage_count)
2857
		hardware_disable_nolock(NULL);
2858 2859 2860
	return 0;
}

2861
static void kvm_resume(void)
2862
{
2863
	if (kvm_usage_count) {
2864
		WARN_ON(raw_spin_is_locked(&kvm_lock));
2865
		hardware_enable_nolock(NULL);
2866
	}
2867 2868
}

2869
static struct syscore_ops kvm_syscore_ops = {
2870 2871 2872 2873
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

2874 2875 2876 2877 2878 2879 2880 2881 2882 2883
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2884
	kvm_arch_vcpu_load(vcpu, cpu);
2885 2886 2887 2888 2889 2890 2891
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2892
	kvm_arch_vcpu_put(vcpu);
2893 2894
}

2895
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
2896
		  struct module *module)
A
Avi Kivity 已提交
2897 2898
{
	int r;
Y
Yang, Sheng 已提交
2899
	int cpu;
A
Avi Kivity 已提交
2900

2901 2902
	r = kvm_arch_init(opaque);
	if (r)
2903
		goto out_fail;
2904

2905
	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2906 2907 2908 2909
		r = -ENOMEM;
		goto out_free_0;
	}

2910
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
2911
	if (r < 0)
2912
		goto out_free_0a;
A
Avi Kivity 已提交
2913

Y
Yang, Sheng 已提交
2914 2915
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
2916
				kvm_arch_check_processor_compat,
2917
				&r, 1);
Y
Yang, Sheng 已提交
2918
		if (r < 0)
2919
			goto out_free_1;
Y
Yang, Sheng 已提交
2920 2921
	}

A
Avi Kivity 已提交
2922 2923
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
2924
		goto out_free_2;
A
Avi Kivity 已提交
2925 2926
	register_reboot_notifier(&kvm_reboot_notifier);

2927
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
2928 2929 2930
	if (!vcpu_align)
		vcpu_align = __alignof__(struct kvm_vcpu);
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
J
Joe Perches 已提交
2931
					   0, NULL);
2932 2933
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
2934
		goto out_free_3;
2935 2936
	}

2937 2938 2939 2940
	r = kvm_async_pf_init();
	if (r)
		goto out_free;

A
Avi Kivity 已提交
2941
	kvm_chardev_ops.owner = module;
2942 2943
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
2944 2945 2946

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
2947
		printk(KERN_ERR "kvm: misc device register failed\n");
2948
		goto out_unreg;
A
Avi Kivity 已提交
2949 2950
	}

2951 2952
	register_syscore_ops(&kvm_syscore_ops);

2953 2954 2955
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

2956 2957 2958 2959 2960
	r = kvm_init_debug();
	if (r) {
		printk(KERN_ERR "kvm: create debugfs files failed\n");
		goto out_undebugfs;
	}
2961

2962
	return 0;
A
Avi Kivity 已提交
2963

2964 2965
out_undebugfs:
	unregister_syscore_ops(&kvm_syscore_ops);
2966 2967
out_unreg:
	kvm_async_pf_deinit();
A
Avi Kivity 已提交
2968
out_free:
2969
	kmem_cache_destroy(kvm_vcpu_cache);
2970
out_free_3:
A
Avi Kivity 已提交
2971
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
2972
	unregister_cpu_notifier(&kvm_cpu_notifier);
2973 2974
out_free_2:
out_free_1:
2975
	kvm_arch_hardware_unsetup();
2976 2977
out_free_0a:
	free_cpumask_var(cpus_hardware_enabled);
2978
out_free_0:
2979
	kvm_arch_exit();
2980
out_fail:
A
Avi Kivity 已提交
2981 2982
	return r;
}
2983
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
2984

2985
void kvm_exit(void)
A
Avi Kivity 已提交
2986
{
2987
	kvm_exit_debug();
A
Avi Kivity 已提交
2988
	misc_deregister(&kvm_dev);
2989
	kmem_cache_destroy(kvm_vcpu_cache);
2990
	kvm_async_pf_deinit();
2991
	unregister_syscore_ops(&kvm_syscore_ops);
A
Avi Kivity 已提交
2992
	unregister_reboot_notifier(&kvm_reboot_notifier);
2993
	unregister_cpu_notifier(&kvm_cpu_notifier);
2994
	on_each_cpu(hardware_disable_nolock, NULL, 1);
2995
	kvm_arch_hardware_unsetup();
2996
	kvm_arch_exit();
2997
	free_cpumask_var(cpus_hardware_enabled);
A
Avi Kivity 已提交
2998
}
2999
EXPORT_SYMBOL_GPL(kvm_exit);