kvm_host.h 29.8 KB
Newer Older
1 2
#ifndef __KVM_HOST_H
#define __KVM_HOST_H
A
Avi Kivity 已提交
3 4 5 6 7 8 9

/*
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 */

#include <linux/types.h>
10
#include <linux/hardirq.h>
A
Avi Kivity 已提交
11 12 13
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
M
Markus Rechberger 已提交
14 15
#include <linux/signal.h>
#include <linux/sched.h>
16
#include <linux/bug.h>
A
Avi Kivity 已提交
17
#include <linux/mm.h>
18
#include <linux/mmu_notifier.h>
19
#include <linux/preempt.h>
20
#include <linux/msi.h>
21
#include <linux/slab.h>
22
#include <linux/rcupdate.h>
23
#include <linux/ratelimit.h>
X
Xiao Guangrong 已提交
24
#include <linux/err.h>
25
#include <linux/irqflags.h>
26
#include <linux/context_tracking.h>
A
Alexey Dobriyan 已提交
27
#include <asm/signal.h>
A
Avi Kivity 已提交
28 29

#include <linux/kvm.h>
I
Ingo Molnar 已提交
30
#include <linux/kvm_para.h>
A
Avi Kivity 已提交
31

32
#include <linux/kvm_types.h>
33

34
#include <asm/kvm_host.h>
35

A
Avi Kivity 已提交
36 37 38 39
#ifndef KVM_MMIO_SIZE
#define KVM_MMIO_SIZE 8
#endif

40 41 42 43 44 45 46
/*
 * The bit 16 ~ bit 31 of kvm_memory_region::flags are internally used
 * in kvm, other bits are visible for userspace which are defined in
 * include/linux/kvm_h.
 */
#define KVM_MEMSLOT_INVALID	(1UL << 16)

47 48
/* Two fragments for cross MMIO pages. */
#define KVM_MAX_MMIO_FRAGMENTS	2
A
Avi Kivity 已提交
49

50 51
/*
 * For the normal pfn, the highest 12 bits should be zero,
52 53
 * so we can mask bit 62 ~ bit 52  to indicate the error pfn,
 * mask bit 63 to indicate the noslot pfn.
54
 */
55 56 57
#define KVM_PFN_ERR_MASK	(0x7ffULL << 52)
#define KVM_PFN_ERR_NOSLOT_MASK	(0xfffULL << 52)
#define KVM_PFN_NOSLOT		(0x1ULL << 63)
58 59 60

#define KVM_PFN_ERR_FAULT	(KVM_PFN_ERR_MASK)
#define KVM_PFN_ERR_HWPOISON	(KVM_PFN_ERR_MASK + 1)
61
#define KVM_PFN_ERR_RO_FAULT	(KVM_PFN_ERR_MASK + 2)
62

63 64 65 66
/*
 * error pfns indicate that the gfn is in slot but faild to
 * translate it to pfn on host.
 */
67
static inline bool is_error_pfn(pfn_t pfn)
X
Xiao Guangrong 已提交
68
{
69
	return !!(pfn & KVM_PFN_ERR_MASK);
X
Xiao Guangrong 已提交
70 71
}

72 73 74 75 76 77
/*
 * error_noslot pfns indicate that the gfn can not be
 * translated to pfn - it is not in slot or failed to
 * translate it to pfn.
 */
static inline bool is_error_noslot_pfn(pfn_t pfn)
X
Xiao Guangrong 已提交
78
{
79
	return !!(pfn & KVM_PFN_ERR_NOSLOT_MASK);
X
Xiao Guangrong 已提交
80 81
}

82 83
/* noslot pfn indicates that the gfn is not in slot. */
static inline bool is_noslot_pfn(pfn_t pfn)
X
Xiao Guangrong 已提交
84
{
85
	return pfn == KVM_PFN_NOSLOT;
X
Xiao Guangrong 已提交
86 87
}

88 89 90 91 92 93
/*
 * architectures with KVM_HVA_ERR_BAD other than PAGE_OFFSET (e.g. s390)
 * provide own defines and kvm_is_error_hva
 */
#ifndef KVM_HVA_ERR_BAD

94 95
#define KVM_HVA_ERR_BAD		(PAGE_OFFSET)
#define KVM_HVA_ERR_RO_BAD	(PAGE_OFFSET + PAGE_SIZE)
X
Xiao Guangrong 已提交
96 97 98

static inline bool kvm_is_error_hva(unsigned long addr)
{
99
	return addr >= PAGE_OFFSET;
X
Xiao Guangrong 已提交
100 101
}

102 103
#endif

104 105
#define KVM_ERR_PTR_BAD_PAGE	(ERR_PTR(-ENOENT))

106
static inline bool is_error_page(struct page *page)
107 108 109 110
{
	return IS_ERR(page);
}

111 112 113
/*
 * vcpu->requests bit members
 */
114
#define KVM_REQ_TLB_FLUSH          0
115
#define KVM_REQ_MIGRATE_TIMER      1
116
#define KVM_REQ_REPORT_TPR_ACCESS  2
117
#define KVM_REQ_MMU_RELOAD         3
J
Joerg Roedel 已提交
118
#define KVM_REQ_TRIPLE_FAULT       4
119
#define KVM_REQ_PENDING_TIMER      5
120
#define KVM_REQ_UNHALT             6
121
#define KVM_REQ_MMU_SYNC           7
Z
Zachary Amsden 已提交
122
#define KVM_REQ_CLOCK_UPDATE       8
123
#define KVM_REQ_KICK               9
124
#define KVM_REQ_DEACTIVATE_FPU    10
125
#define KVM_REQ_EVENT             11
126
#define KVM_REQ_APF_HALT          12
G
Glauber Costa 已提交
127
#define KVM_REQ_STEAL_UPDATE      13
A
Avi Kivity 已提交
128
#define KVM_REQ_NMI               14
129 130 131 132 133 134 135
#define KVM_REQ_PMU               15
#define KVM_REQ_PMI               16
#define KVM_REQ_WATCHDOG          17
#define KVM_REQ_MASTERCLOCK_UPDATE 18
#define KVM_REQ_MCLOCK_INPROGRESS 19
#define KVM_REQ_EPR_EXIT          20
#define KVM_REQ_SCAN_IOAPIC       21
136
#define KVM_REQ_GLOBAL_CLOCK_UPDATE 22
A
Avi Kivity 已提交
137

138 139
#define KVM_USERSPACE_IRQ_SOURCE_ID		0
#define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID	1
140

141
struct kvm;
A
Avi Kivity 已提交
142
struct kvm_vcpu;
143
extern struct kmem_cache *kvm_vcpu_cache;
A
Avi Kivity 已提交
144

145
extern spinlock_t kvm_lock;
146 147
extern struct list_head vm_list;

148 149 150 151 152 153
struct kvm_io_range {
	gpa_t addr;
	int len;
	struct kvm_io_device *dev;
};

154
#define NR_IOBUS_DEVS 1000
155

156
struct kvm_io_bus {
157 158
	int dev_count;
	int ioeventfd_count;
159
	struct kvm_io_range range[];
160 161
};

M
Marcelo Tosatti 已提交
162 163 164
enum kvm_bus {
	KVM_MMIO_BUS,
	KVM_PIO_BUS,
165
	KVM_VIRTIO_CCW_NOTIFY_BUS,
M
Marcelo Tosatti 已提交
166 167 168 169 170
	KVM_NR_BUSES
};

int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
		     int len, const void *val);
C
Cornelia Huck 已提交
171 172
int kvm_io_bus_write_cookie(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, const void *val, long cookie);
M
Marcelo Tosatti 已提交
173
int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len,
174
		    void *val);
175 176
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
			    int len, struct kvm_io_device *dev);
M
Marcelo Tosatti 已提交
177 178
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
			      struct kvm_io_device *dev);
179

180 181 182 183 184 185 186 187 188 189
#ifdef CONFIG_KVM_ASYNC_PF
struct kvm_async_pf {
	struct work_struct work;
	struct list_head link;
	struct list_head queue;
	struct kvm_vcpu *vcpu;
	struct mm_struct *mm;
	gva_t gva;
	unsigned long addr;
	struct kvm_arch_async_pf arch;
190
	bool   wakeup_all;
191 192 193 194
};

void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu);
void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu);
195
int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, unsigned long hva,
196
		       struct kvm_arch_async_pf *arch);
197
int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
198 199
#endif

200 201 202
enum {
	OUTSIDE_GUEST_MODE,
	IN_GUEST_MODE,
203 204
	EXITING_GUEST_MODE,
	READING_SHADOW_PAGE_TABLES,
205 206
};

A
Avi Kivity 已提交
207 208 209 210 211 212 213 214 215 216
/*
 * Sometimes a large or cross-page mmio needs to be broken up into separate
 * exits for userspace servicing.
 */
struct kvm_mmio_fragment {
	gpa_t gpa;
	void *data;
	unsigned len;
};

217 218
struct kvm_vcpu {
	struct kvm *kvm;
219
#ifdef CONFIG_PREEMPT_NOTIFIERS
220
	struct preempt_notifier preempt_notifier;
221
#endif
222
	int cpu;
223
	int vcpu_id;
224 225
	int srcu_idx;
	int mode;
226
	unsigned long requests;
J
Jan Kiszka 已提交
227
	unsigned long guest_debug;
228 229 230

	struct mutex mutex;
	struct kvm_run *run;
231

232
	int fpu_active;
233
	int guest_fpu_loaded, guest_xcr0_loaded;
234
	wait_queue_head_t wq;
235
	struct pid *pid;
236 237 238 239
	int sigset_active;
	sigset_t sigset;
	struct kvm_vcpu_stat stat;

240
#ifdef CONFIG_HAS_IOMEM
241 242 243
	int mmio_needed;
	int mmio_read_completed;
	int mmio_is_write;
A
Avi Kivity 已提交
244 245 246
	int mmio_cur_fragment;
	int mmio_nr_fragments;
	struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS];
247
#endif
A
Avi Kivity 已提交
248

249 250 251 252 253 254 255 256 257
#ifdef CONFIG_KVM_ASYNC_PF
	struct {
		u32 queued;
		struct list_head queue;
		struct list_head done;
		spinlock_t lock;
	} async_pf;
#endif

258 259 260 261 262 263 264 265 266 267 268 269
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
	/*
	 * Cpu relax intercept or pause loop exit optimization
	 * in_spin_loop: set when a vcpu does a pause loop exit
	 *  or cpu relax intercepted.
	 * dy_eligible: indicates whether vcpu is eligible for directed yield.
	 */
	struct {
		bool in_spin_loop;
		bool dy_eligible;
	} spin_loop;
#endif
270
	bool preempted;
271 272 273
	struct kvm_vcpu_arch arch;
};

274 275 276 277 278
static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
{
	return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE);
}

279 280 281 282 283 284
/*
 * Some of the bitops functions do not support too long bitmaps.
 * This number must be determined not to exceed such limits.
 */
#define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1)

A
Avi Kivity 已提交
285 286 287 288
struct kvm_memory_slot {
	gfn_t base_gfn;
	unsigned long npages;
	unsigned long *dirty_bitmap;
289
	struct kvm_arch_memory_slot arch;
290
	unsigned long userspace_addr;
291
	u32 flags;
292
	short id;
A
Avi Kivity 已提交
293 294
};

295 296 297 298 299
static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot)
{
	return ALIGN(memslot->npages, BITS_PER_LONG) / 8;
}

300 301 302 303 304 305 306 307
struct kvm_s390_adapter_int {
	u64 ind_addr;
	u64 summary_addr;
	u64 ind_offset;
	u32 summary_offset;
	u32 adapter_id;
};

308 309
struct kvm_kernel_irq_routing_entry {
	u32 gsi;
310
	u32 type;
311
	int (*set)(struct kvm_kernel_irq_routing_entry *e,
312 313
		   struct kvm *kvm, int irq_source_id, int level,
		   bool line_status);
314 315 316 317 318
	union {
		struct {
			unsigned irqchip;
			unsigned pin;
		} irqchip;
S
Sheng Yang 已提交
319
		struct msi_msg msi;
320
		struct kvm_s390_adapter_int adapter;
321
	};
322 323 324
	struct hlist_node link;
};

325
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
326

327
struct kvm_irq_routing_table {
328
	int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS];
329 330 331 332 333 334 335
	struct kvm_kernel_irq_routing_entry *rt_entries;
	u32 nr_rt_entries;
	/*
	 * Array indexed by gsi. Each entry contains list of irq chips
	 * the gsi is connected to.
	 */
	struct hlist_head map[0];
336 337
};

338 339 340 341 342 343
#else

struct kvm_irq_routing_table {};

#endif

344 345 346 347
#ifndef KVM_PRIVATE_MEM_SLOTS
#define KVM_PRIVATE_MEM_SLOTS 0
#endif

348
#ifndef KVM_MEM_SLOTS_NUM
349
#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
350 351
#endif

352 353 354 355 356
/*
 * Note:
 * memslots are not sorted by id anymore, please use id_to_memslot()
 * to get the memslot by its id.
 */
357
struct kvm_memslots {
358
	u64 generation;
359
	struct kvm_memory_slot memslots[KVM_MEM_SLOTS_NUM];
360
	/* The mapping table from slot id to the index in memslots[]. */
361
	short id_to_index[KVM_MEM_SLOTS_NUM];
362 363
};

A
Avi Kivity 已提交
364
struct kvm {
365
	spinlock_t mmu_lock;
366
	struct mutex slots_lock;
367
	struct mm_struct *mm; /* userspace tied to this vm */
368
	struct kvm_memslots *memslots;
369
	struct srcu_struct srcu;
370 371 372
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
	u32 bsp_vcpu_id;
#endif
R
Rusty Russell 已提交
373
	struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
374
	atomic_t online_vcpus;
375
	int last_boosted_vcpu;
376
	struct list_head vm_list;
377
	struct mutex lock;
M
Marcelo Tosatti 已提交
378
	struct kvm_io_bus *buses[KVM_NR_BUSES];
G
Gregory Haskins 已提交
379 380 381 382
#ifdef CONFIG_HAVE_KVM_EVENTFD
	struct {
		spinlock_t        lock;
		struct list_head  items;
383 384
		struct list_head  resampler_list;
		struct mutex      resampler_lock;
G
Gregory Haskins 已提交
385
	} irqfds;
G
Gregory Haskins 已提交
386
	struct list_head ioeventfds;
G
Gregory Haskins 已提交
387
#endif
388
	struct kvm_vm_stat stat;
389
	struct kvm_arch arch;
I
Izik Eidus 已提交
390
	atomic_t users_count;
391 392
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
393 394
	spinlock_t ring_lock;
	struct list_head coalesced_zones;
395
#endif
396

397
	struct mutex irq_lock;
398
#ifdef CONFIG_HAVE_KVM_IRQCHIP
399 400 401 402
	/*
	 * Update side is protected by irq_lock and,
	 * if configured, irqfds.lock.
	 */
A
Arnd Bergmann 已提交
403
	struct kvm_irq_routing_table __rcu *irq_routing;
404
	struct hlist_head mask_notifier_list;
405
	struct hlist_head irq_ack_notifier_list;
406 407
#endif

408
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
409 410 411 412
	struct mmu_notifier mmu_notifier;
	unsigned long mmu_notifier_seq;
	long mmu_notifier_count;
#endif
413 414 415
	/* Protected by mmu_lock */
	bool tlbs_dirty;

416
	struct list_head devices;
A
Avi Kivity 已提交
417 418
};

419 420 421 422 423 424 425 426 427
#define kvm_err(fmt, ...) \
	pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
#define kvm_info(fmt, ...) \
	pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
#define kvm_debug(fmt, ...) \
	pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
#define kvm_pr_unimpl(fmt, ...) \
	pr_err_ratelimited("kvm [%i]: " fmt, \
			   task_tgid_nr(current), ## __VA_ARGS__)
428

429 430 431
/* The guest did something we don't support. */
#define vcpu_unimpl(vcpu, fmt, ...)					\
	kvm_pr_unimpl("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
A
Avi Kivity 已提交
432

433 434 435 436 437 438 439
static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
{
	smp_rmb();
	return kvm->vcpus[i];
}

#define kvm_for_each_vcpu(idx, vcpup, kvm) \
440 441 442 443
	for (idx = 0; \
	     idx < atomic_read(&kvm->online_vcpus) && \
	     (vcpup = kvm_get_vcpu(kvm, idx)) != NULL; \
	     idx++)
444

445 446
#define kvm_for_each_memslot(memslot, slots)	\
	for (memslot = &slots->memslots[0];	\
447 448
	      memslot < slots->memslots + KVM_MEM_SLOTS_NUM && memslot->npages;\
		memslot++)
449

R
Rusty Russell 已提交
450 451 452
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id);
void kvm_vcpu_uninit(struct kvm_vcpu *vcpu);

453
int __must_check vcpu_load(struct kvm_vcpu *vcpu);
454 455
void vcpu_put(struct kvm_vcpu *vcpu);

456
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
457 458 459 460 461 462 463 464 465 466 467 468
int kvm_irqfd_init(void);
void kvm_irqfd_exit(void);
#else
static inline int kvm_irqfd_init(void)
{
	return 0;
}

static inline void kvm_irqfd_exit(void)
{
}
#endif
469
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
470
		  struct module *module);
471
void kvm_exit(void);
A
Avi Kivity 已提交
472

I
Izik Eidus 已提交
473 474 475
void kvm_get_kvm(struct kvm *kvm);
void kvm_put_kvm(struct kvm *kvm);

476 477 478 479 480 481 482
static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm)
{
	return rcu_dereference_check(kvm->memslots,
			srcu_read_lock_held(&kvm->srcu)
			|| lockdep_is_held(&kvm->slots_lock));
}

483 484 485
static inline struct kvm_memory_slot *
id_to_memslot(struct kvm_memslots *slots, int id)
{
486 487
	int index = slots->id_to_index[id];
	struct kvm_memory_slot *slot;
488

489
	slot = &slots->memslots[index];
490

491 492
	WARN_ON(slot->id != id);
	return slot;
493 494
}

495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512
/*
 * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations:
 * - create a new memory slot
 * - delete an existing memory slot
 * - modify an existing memory slot
 *   -- move it in the guest physical memory space
 *   -- just change its flags
 *
 * Since flags can be changed by some of these operations, the following
 * differentiation is the best we can do for __kvm_set_memory_region():
 */
enum kvm_mr_change {
	KVM_MR_CREATE,
	KVM_MR_DELETE,
	KVM_MR_MOVE,
	KVM_MR_FLAGS_ONLY,
};

513
int kvm_set_memory_region(struct kvm *kvm,
514
			  struct kvm_userspace_memory_region *mem);
515
int __kvm_set_memory_region(struct kvm *kvm,
516
			    struct kvm_userspace_memory_region *mem);
517
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
518
			   struct kvm_memory_slot *dont);
519 520
int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
			    unsigned long npages);
521
void kvm_arch_memslots_updated(struct kvm *kvm);
522 523 524
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
				struct kvm_userspace_memory_region *mem,
525
				enum kvm_mr_change change);
526
void kvm_arch_commit_memory_region(struct kvm *kvm,
527
				struct kvm_userspace_memory_region *mem,
528 529
				const struct kvm_memory_slot *old,
				enum kvm_mr_change change);
530
bool kvm_largepages_enabled(void);
531
void kvm_disable_largepages(void);
532 533 534 535 536
/* flush all memory translations */
void kvm_arch_flush_shadow_all(struct kvm *kvm);
/* flush memory translations pointing to 'slot' */
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot);
537

538 539 540
int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
			    int nr_pages);

A
Avi Kivity 已提交
541
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
M
Marcelo Tosatti 已提交
542
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
543
unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable);
X
Xiao Guangrong 已提交
544
unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
545 546
void kvm_release_page_clean(struct page *page);
void kvm_release_page_dirty(struct page *page);
547 548
void kvm_set_page_accessed(struct page *page);

549
pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn);
550 551
pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
		       bool write_fault, bool *writable);
552
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
553 554
pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
		      bool *writable);
555
pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
556 557
pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn);

558 559 560 561 562
void kvm_release_pfn_clean(pfn_t pfn);
void kvm_set_pfn_dirty(pfn_t pfn);
void kvm_set_pfn_accessed(pfn_t pfn);
void kvm_get_pfn(pfn_t pfn);

563 564
int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len);
565 566
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len);
567
int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
568 569
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len);
570 571 572 573
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len);
int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len);
574 575 576
int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
			   void *data, unsigned long len);
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
577
			      gpa_t gpa, unsigned long len);
578 579
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len);
int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
A
Avi Kivity 已提交
580
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
581
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
J
Joerg Roedel 已提交
582
unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn);
A
Avi Kivity 已提交
583 584
void mark_page_dirty(struct kvm *kvm, gfn_t gfn);

585
void kvm_vcpu_block(struct kvm_vcpu *vcpu);
586
void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
587
bool kvm_vcpu_yield_to(struct kvm_vcpu *target);
Z
Zhai, Edwin 已提交
588
void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu);
589 590
void kvm_load_guest_fpu(struct kvm_vcpu *vcpu);
void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);
591

592
void kvm_flush_remote_tlbs(struct kvm *kvm);
593
void kvm_reload_remote_mmus(struct kvm *kvm);
594
void kvm_make_mclock_inprogress_request(struct kvm *kvm);
595
void kvm_make_scan_ioapic_request(struct kvm *kvm);
A
Avi Kivity 已提交
596

597 598
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg);
599 600
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg);
601
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf);
602 603 604

int kvm_dev_ioctl_check_extension(long ext);

605 606 607 608 609
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty);
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
				struct kvm_dirty_log *log);

610 611
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
			bool line_status);
612 613
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg);
614

615 616 617
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);

618 619 620
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr);

621 622 623 624 625 626
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs);
int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs);
627 628 629 630
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state);
int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state);
J
Jan Kiszka 已提交
631 632
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg);
633 634
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);

635 636
int kvm_arch_init(void *opaque);
void kvm_arch_exit(void);
637

638 639 640 641 642 643 644
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu);
void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu);

void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu);
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu);
struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id);
645
int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);
646
int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu);
647
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
648

649
int kvm_arch_hardware_enable(void *garbage);
650 651 652 653
void kvm_arch_hardware_disable(void *garbage);
int kvm_arch_hardware_setup(void);
void kvm_arch_hardware_unsetup(void);
void kvm_arch_check_processor_compat(void *rtn);
654
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
655
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
656

657 658 659
void *kvm_kvzalloc(unsigned long size);
void kvm_kvfree(const void *addr);

660 661 662 663 664 665 666 667 668 669 670 671
#ifndef __KVM_HAVE_ARCH_VM_ALLOC
static inline struct kvm *kvm_arch_alloc_vm(void)
{
	return kzalloc(sizeof(struct kvm), GFP_KERNEL);
}

static inline void kvm_arch_free_vm(struct kvm *kvm)
{
	kfree(kvm);
}
#endif

672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
#ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA
void kvm_arch_register_noncoherent_dma(struct kvm *kvm);
void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm);
bool kvm_arch_has_noncoherent_dma(struct kvm *kvm);
#else
static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
{
}

static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
{
}

static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
{
	return false;
}
#endif

691 692
static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
{
693 694 695
#ifdef __KVM_HAVE_ARCH_WQP
	return vcpu->arch.wqp;
#else
696 697
	return &vcpu->wq;
#endif
698
}
699

700
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type);
701
void kvm_arch_destroy_vm(struct kvm *kvm);
702
void kvm_arch_sync_events(struct kvm *kvm);
703

704
int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
705
void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
706

X
Xiao Guangrong 已提交
707
bool kvm_is_mmio_pfn(pfn_t pfn);
708

B
Ben-Ami Yassour 已提交
709 710 711 712 713 714 715 716 717 718
struct kvm_irq_ack_notifier {
	struct hlist_node link;
	unsigned gsi;
	void (*irq_acked)(struct kvm_irq_ack_notifier *kian);
};

struct kvm_assigned_dev_kernel {
	struct kvm_irq_ack_notifier ack_notifier;
	struct list_head list;
	int assigned_dev_id;
719
	int host_segnr;
B
Ben-Ami Yassour 已提交
720 721
	int host_busnr;
	int host_devfn;
S
Sheng Yang 已提交
722
	unsigned int entries_nr;
B
Ben-Ami Yassour 已提交
723
	int host_irq;
724
	bool host_irq_disabled;
725
	bool pci_2_3;
S
Sheng Yang 已提交
726
	struct msix_entry *host_msix_entries;
B
Ben-Ami Yassour 已提交
727
	int guest_irq;
728
	struct msix_entry *guest_msix_entries;
729
	unsigned long irq_requested_type;
730
	int irq_source_id;
731
	int flags;
B
Ben-Ami Yassour 已提交
732 733
	struct pci_dev *dev;
	struct kvm *kvm;
734
	spinlock_t intx_lock;
735
	spinlock_t intx_mask_lock;
736
	char irq_name[32];
737
	struct pci_saved_state *pci_saved_state;
B
Ben-Ami Yassour 已提交
738
};
739 740 741 742 743 744 745 746 747 748 749

struct kvm_irq_mask_notifier {
	void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
	int irq;
	struct hlist_node link;
};

void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
				    struct kvm_irq_mask_notifier *kimn);
void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
				      struct kvm_irq_mask_notifier *kimn);
750 751
void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
			     bool mask);
752

753 754
int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
		bool line_status);
M
Michael S. Tsirkin 已提交
755
int kvm_set_irq_inatomic(struct kvm *kvm, int irq_source_id, u32 irq, int level);
756
int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm,
757
		int irq_source_id, int level, bool line_status);
758
bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin);
759
void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin);
760 761
void kvm_register_irq_ack_notifier(struct kvm *kvm,
				   struct kvm_irq_ack_notifier *kian);
762 763
void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
				   struct kvm_irq_ack_notifier *kian);
764 765
int kvm_request_irq_source_id(struct kvm *kvm);
void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
B
Ben-Ami Yassour 已提交
766

767
#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
768
int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
769
void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
W
Weidong Han 已提交
770
int kvm_iommu_map_guest(struct kvm *kvm);
B
Ben-Ami Yassour 已提交
771
int kvm_iommu_unmap_guest(struct kvm *kvm);
W
Weidong Han 已提交
772 773
int kvm_assign_device(struct kvm *kvm,
		      struct kvm_assigned_dev_kernel *assigned_dev);
W
Weidong Han 已提交
774 775
int kvm_deassign_device(struct kvm *kvm,
			struct kvm_assigned_dev_kernel *assigned_dev);
776
#else
B
Ben-Ami Yassour 已提交
777
static inline int kvm_iommu_map_pages(struct kvm *kvm,
778
				      struct kvm_memory_slot *slot)
B
Ben-Ami Yassour 已提交
779 780 781 782
{
	return 0;
}

783 784 785 786 787
static inline void kvm_iommu_unmap_pages(struct kvm *kvm,
					 struct kvm_memory_slot *slot)
{
}

B
Ben-Ami Yassour 已提交
788 789 790 791
static inline int kvm_iommu_unmap_guest(struct kvm *kvm)
{
	return 0;
}
792
#endif
B
Ben-Ami Yassour 已提交
793

794 795 796 797 798 799 800 801 802 803
static inline void kvm_guest_enter(void)
{
	unsigned long flags;

	BUG_ON(preemptible());

	local_irq_save(flags);
	guest_enter();
	local_irq_restore(flags);

804 805
	/* KVM does not hold any references to rcu protected data when it
	 * switches CPU into a guest mode. In fact switching to a guest mode
M
Michael S. Tsirkin 已提交
806
	 * is very similar to exiting to userspace from rcu point of view. In
807 808 809 810 811
	 * addition CPU may stay in a guest mode for quite a long time (up to
	 * one time slice). Lets treat guest mode as quiescent state, just like
	 * we do with user-mode execution.
	 */
	rcu_virt_note_context_switch(smp_processor_id());
812 813 814 815
}

static inline void kvm_guest_exit(void)
{
816 817 818 819 820
	unsigned long flags;

	local_irq_save(flags);
	guest_exit();
	local_irq_restore(flags);
821 822
}

823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
/*
 * search_memslots() and __gfn_to_memslot() are here because they are
 * used in non-modular code in arch/powerpc/kvm/book3s_hv_rm_mmu.c.
 * gfn_to_memslot() itself isn't here as an inline because that would
 * bloat other code too much.
 */
static inline struct kvm_memory_slot *
search_memslots(struct kvm_memslots *slots, gfn_t gfn)
{
	struct kvm_memory_slot *memslot;

	kvm_for_each_memslot(memslot, slots)
		if (gfn >= memslot->base_gfn &&
		      gfn < memslot->base_gfn + memslot->npages)
			return memslot;

	return NULL;
}

static inline struct kvm_memory_slot *
__gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn)
{
	return search_memslots(slots, gfn);
}

848 849 850 851 852 853
static inline unsigned long
__gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
{
	return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE;
}

854 855 856 857 858
static inline int memslot_id(struct kvm *kvm, gfn_t gfn)
{
	return gfn_to_memslot(kvm, gfn)->id;
}

859 860
static inline gfn_t
hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot)
861
{
862 863 864
	gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT;

	return slot->base_gfn + gfn_offset;
865 866
}

A
Avi Kivity 已提交
867 868 869 870
static inline gpa_t gfn_to_gpa(gfn_t gfn)
{
	return (gpa_t)gfn << PAGE_SHIFT;
}
A
Avi Kivity 已提交
871

872 873 874 875 876
static inline gfn_t gpa_to_gfn(gpa_t gpa)
{
	return (gfn_t)(gpa >> PAGE_SHIFT);
}

B
Ben-Ami Yassour 已提交
877 878 879 880 881
static inline hpa_t pfn_to_hpa(pfn_t pfn)
{
	return (hpa_t)pfn << PAGE_SHIFT;
}

M
Marcelo Tosatti 已提交
882
static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
883 884 885 886
{
	set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
}

887 888 889 890 891
enum kvm_stat_kind {
	KVM_STAT_VM,
	KVM_STAT_VCPU,
};

892 893 894
struct kvm_stats_debugfs_item {
	const char *name;
	int offset;
895
	enum kvm_stat_kind kind;
896 897 898
	struct dentry *dentry;
};
extern struct kvm_stats_debugfs_item debugfs_entries[];
899
extern struct dentry *kvm_debugfs_dir;
900

901
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
902
static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq)
903
{
904
	if (unlikely(kvm->mmu_notifier_count))
905 906
		return 1;
	/*
907 908 909 910 911 912 913 914
	 * Ensure the read of mmu_notifier_count happens before the read
	 * of mmu_notifier_seq.  This interacts with the smp_wmb() in
	 * mmu_notifier_invalidate_range_end to make sure that the caller
	 * either sees the old (non-zero) value of mmu_notifier_count or
	 * the new (incremented) value of mmu_notifier_seq.
	 * PowerPC Book3s HV KVM calls this under a per-page lock
	 * rather than under kvm->mmu_lock, for scalability, so
	 * can't rely on kvm->mmu_lock to keep things ordered.
915
	 */
916
	smp_rmb();
917
	if (kvm->mmu_notifier_seq != mmu_seq)
918 919 920 921 922
		return 1;
	return 0;
}
#endif

923
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
924

925 926 927
#ifdef CONFIG_S390
#define KVM_MAX_IRQ_ROUTES 4096 //FIXME: we can have more than that...
#else
928
#define KVM_MAX_IRQ_ROUTES 1024
929
#endif
930 931 932 933 934 935

int kvm_setup_default_irq_routing(struct kvm *kvm);
int kvm_set_irq_routing(struct kvm *kvm,
			const struct kvm_irq_routing_entry *entries,
			unsigned nr,
			unsigned flags);
936 937 938
int kvm_set_routing_entry(struct kvm_irq_routing_table *rt,
			  struct kvm_kernel_irq_routing_entry *e,
			  const struct kvm_irq_routing_entry *ue);
939 940
void kvm_free_irq_routing(struct kvm *kvm);

941 942
int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);

943 944 945 946 947 948
#else

static inline void kvm_free_irq_routing(struct kvm *kvm) {}

#endif

G
Gregory Haskins 已提交
949 950
#ifdef CONFIG_HAVE_KVM_EVENTFD

G
Gregory Haskins 已提交
951
void kvm_eventfd_init(struct kvm *kvm);
952 953 954
int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args);

#ifdef CONFIG_HAVE_KVM_IRQCHIP
955
int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args);
G
Gregory Haskins 已提交
956
void kvm_irqfd_release(struct kvm *kvm);
957
void kvm_irq_routing_update(struct kvm *, struct kvm_irq_routing_table *);
958 959 960 961 962 963 964 965
#else
static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
{
	return -EINVAL;
}

static inline void kvm_irqfd_release(struct kvm *kvm) {}
#endif
G
Gregory Haskins 已提交
966 967 968

#else

G
Gregory Haskins 已提交
969
static inline void kvm_eventfd_init(struct kvm *kvm) {}
970

971
static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
G
Gregory Haskins 已提交
972 973 974 975 976
{
	return -EINVAL;
}

static inline void kvm_irqfd_release(struct kvm *kvm) {}
977

A
Alexander Graf 已提交
978
#ifdef CONFIG_HAVE_KVM_IRQCHIP
979 980 981 982 983
static inline void kvm_irq_routing_update(struct kvm *kvm,
					  struct kvm_irq_routing_table *irq_rt)
{
	rcu_assign_pointer(kvm->irq_routing, irq_rt);
}
A
Alexander Graf 已提交
984
#endif
985

G
Gregory Haskins 已提交
986 987 988 989
static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
{
	return -ENOSYS;
}
G
Gregory Haskins 已提交
990 991 992

#endif /* CONFIG_HAVE_KVM_EVENTFD */

993
#ifdef CONFIG_KVM_APIC_ARCHITECTURE
994 995
static inline bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
{
996
	return vcpu->kvm->bsp_vcpu_id == vcpu->vcpu_id;
997
}
998 999 1000 1001 1002 1003 1004

bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu);

#else

static inline bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu) { return true; }

A
Avi Kivity 已提交
1005
#endif
1006

1007
#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
1008 1009 1010 1011

long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
				  unsigned long arg);

1012 1013
void kvm_free_all_assigned_devices(struct kvm *kvm);

1014 1015 1016 1017 1018 1019 1020 1021
#else

static inline long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
						unsigned long arg)
{
	return -ENOTTY;
}

1022 1023
static inline void kvm_free_all_assigned_devices(struct kvm *kvm) {}

1024
#endif
1025

1026 1027 1028 1029 1030 1031 1032
static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu)
{
	set_bit(req, &vcpu->requests);
}

static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
{
1033 1034 1035 1036 1037 1038
	if (test_bit(req, &vcpu->requests)) {
		clear_bit(req, &vcpu->requests);
		return true;
	} else {
		return false;
	}
1039 1040
}

1041 1042
extern bool kvm_rebooting;

S
Scott Wood 已提交
1043 1044 1045 1046 1047 1048
struct kvm_device_ops;

struct kvm_device {
	struct kvm_device_ops *ops;
	struct kvm *kvm;
	void *private;
1049
	struct list_head vm_node;
S
Scott Wood 已提交
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
};

/* create, destroy, and name are mandatory */
struct kvm_device_ops {
	const char *name;
	int (*create)(struct kvm_device *dev, u32 type);

	/*
	 * Destroy is responsible for freeing dev.
	 *
	 * Destroy may be called before or after destructors are called
	 * on emulated I/O regions, depending on whether a reference is
	 * held by a vcpu or other kvm component that gets destroyed
	 * after the emulated I/O.
	 */
	void (*destroy)(struct kvm_device *dev);

	int (*set_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
	int (*get_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
	int (*has_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
	long (*ioctl)(struct kvm_device *dev, unsigned int ioctl,
		      unsigned long arg);
};

void kvm_device_get(struct kvm_device *dev);
void kvm_device_put(struct kvm_device *dev);
struct kvm_device *kvm_device_from_filp(struct file *filp);

1078
extern struct kvm_device_ops kvm_mpic_ops;
1079
extern struct kvm_device_ops kvm_xics_ops;
A
Alex Williamson 已提交
1080
extern struct kvm_device_ops kvm_vfio_ops;
1081
extern struct kvm_device_ops kvm_arm_vgic_v2_ops;
1082
extern struct kvm_device_ops kvm_flic_ops;
1083

1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT

static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
{
	vcpu->spin_loop.in_spin_loop = val;
}
static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
{
	vcpu->spin_loop.dy_eligible = val;
}

#else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */

static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
{
}

static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
{
}
#endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
1105
#endif
1106