emulate.c 107.4 KB
Newer Older
A
Avi Kivity 已提交
1
/******************************************************************************
2
 * emulate.c
A
Avi Kivity 已提交
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
A
Avi Kivity 已提交
10 11
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
A
Avi Kivity 已提交
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
A
Avi Kivity 已提交
27

28
#include "x86.h"
29
#include "tss.h"
30

31 32 33
/*
 * Operand types
 */
34 35 36 37 38 39 40 41 42
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
43 44 45 46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
#define OpImm             12ull  /* Sign extended immediate */
47 48 49 50 51 52 53
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
54 55 56 57 58 59
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
60 61

#define OpBits             5  /* Width of operand field */
62
#define OpMask             ((1ull << OpBits) - 1)
63

A
Avi Kivity 已提交
64 65 66 67 68 69 70 71 72 73
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
74
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
75
/* Destination operand type. */
76 77 78 79 80 81 82 83 84 85
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
#define DstMask     (OpMask << DstShift)
A
Avi Kivity 已提交
86
/* Source operand type. */
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
#define SrcDX       (OpDX << SrcShift)
#define SrcMask     (OpMask << SrcShift)
105 106 107 108 109 110 111 112 113 114
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
#define Sse         (1<<18)     /* SSE Vector instruction */
115 116 117 118
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
119
/* Misc flags */
120
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
121
#define VendorSpecific (1<<22) /* Vendor specific instruction */
122
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124
#define Undefined   (1<<25) /* No Such Instruction */
125
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
126
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
127
#define No64	    (1<<28)
128
#define PageTable   (1 << 29)   /* instruction used to write page table */
129
/* Source 2 operand type */
130
#define Src2Shift   (30)
131 132 133 134 135
#define Src2None    (OpNone << Src2Shift)
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
136 137 138 139 140 141
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
142
#define Src2Mask    (OpMask << Src2Shift)
A
Avi Kivity 已提交
143

144 145 146 147 148 149 150 151
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
152

153
struct opcode {
154 155
	u64 flags : 56;
	u64 intercept : 8;
156
	union {
157
		int (*execute)(struct x86_emulate_ctxt *ctxt);
158 159
		struct opcode *group;
		struct group_dual *gdual;
160
		struct gprefix *gprefix;
161
	} u;
162
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
163 164 165 166 167
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
168 169
};

170 171 172 173 174 175 176
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

A
Avi Kivity 已提交
177
/* EFLAGS bit definitions. */
178 179 180 181
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
182 183
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
184 185
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
A
Avi Kivity 已提交
186 187
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
188
#define EFLG_IF (1<<9)
189
#define EFLG_TF (1<<8)
A
Avi Kivity 已提交
190 191 192 193 194 195
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

196 197 198
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

A
Avi Kivity 已提交
199 200 201 202 203 204 205
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

206
#if defined(CONFIG_X86_64)
A
Avi Kivity 已提交
207 208 209 210 211 212 213 214 215 216 217 218 219 220
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
A
Avi Kivity 已提交
236 237 238 239 240 241 242 243 244

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

245 246 247 248 249 250
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

251
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
252 253 254 255 256
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
257 258
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
259
			  "=&r" (_tmp)					\
260
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
261
	} while (0)
262 263


A
Avi Kivity 已提交
264
/* Raw emulation: instruction has two explicit operands. */
265
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
266 267 268
	do {								\
		unsigned long _tmp;					\
									\
269
		switch ((ctxt)->dst.bytes) {				\
270
		case 2:							\
271
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
272 273
			break;						\
		case 4:							\
274
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
275 276
			break;						\
		case 8:							\
277
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
278 279
			break;						\
		}							\
A
Avi Kivity 已提交
280 281
	} while (0)

282
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
A
Avi Kivity 已提交
283
	do {								     \
284
		unsigned long _tmp;					     \
285
		switch ((ctxt)->dst.bytes) {				     \
A
Avi Kivity 已提交
286
		case 1:							     \
287
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
A
Avi Kivity 已提交
288 289
			break;						     \
		default:						     \
290
			__emulate_2op_nobyte(ctxt, _op,			     \
A
Avi Kivity 已提交
291 292 293 294 295 296
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
297 298
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
A
Avi Kivity 已提交
299 300

/* Source operand is byte, word, long or quad sized. */
301 302
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
303 304

/* Source operand is word, long or quad sized. */
305 306
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
307

308
/* Instruction has three operands and one operand is stored in ECX register */
309
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
310 311
	do {								\
		unsigned long _tmp;					\
312 313 314
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
315 316 317 318 319
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
320
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
321 322 323
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
324 325 326
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
327 328
	} while (0)

329
#define emulate_2op_cl(ctxt, _op)					\
330
	do {								\
331
		switch ((ctxt)->dst.bytes) {				\
332
		case 2:							\
333
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
334 335
			break;						\
		case 4:							\
336
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
337 338
			break;						\
		case 8:							\
339
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
340 341
			break;						\
		}							\
342 343
	} while (0)

344
#define __emulate_1op(ctxt, _op, _suffix)				\
A
Avi Kivity 已提交
345 346 347
	do {								\
		unsigned long _tmp;					\
									\
348 349 350 351
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
352
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
353 354 355 356 357
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
358
#define emulate_1op(ctxt, _op)						\
359
	do {								\
360 361 362 363 364
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
A
Avi Kivity 已提交
365 366 367
		}							\
	} while (0)

368
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
369 370
	do {								\
		unsigned long _tmp;					\
371 372
		ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX];		\
		ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX];		\
373 374 375 376 377 378 379 380 381 382 383 384
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
385 386 387 388
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val),	\
			  "a" (*rax), "d" (*rdx));			\
389 390
	} while (0)

391
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
392
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
393
	do {								\
394
		switch((ctxt)->src.bytes) {				\
395
		case 1:							\
396
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
397 398
			break;						\
		case 2:							\
399
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
400 401
			break;						\
		case 4:							\
402
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
403 404
			break;						\
		case 8: ON64(						\
405
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
406 407 408 409
			break;						\
		}							\
	} while (0)

410 411 412 413 414 415
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
416 417 418 419 420 421 422 423
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
424 425 426
		.next_rip   = ctxt->eip,
	};

427
	return ctxt->ops->intercept(ctxt, &info, stage);
428 429
}

430
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
431
{
432
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
433 434
}

A
Avi Kivity 已提交
435
/* Access/update address held in a register, based on addressing mode. */
436
static inline unsigned long
437
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
438
{
439
	if (ctxt->ad_bytes == sizeof(unsigned long))
440 441
		return reg;
	else
442
		return reg & ad_mask(ctxt);
443 444 445
}

static inline unsigned long
446
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
447
{
448
	return address_mask(ctxt, reg);
449 450
}

451
static inline void
452
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
453
{
454
	if (ctxt->ad_bytes == sizeof(unsigned long))
455 456
		*reg += inc;
	else
457
		*reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
458
}
A
Avi Kivity 已提交
459

460
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
461
{
462
	register_address_increment(ctxt, &ctxt->_eip, rel);
463
}
464

465 466 467 468 469 470 471
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

472
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
473
{
474 475
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
476 477
}

478
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
479 480 481 482
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

483
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
484 485
}

486
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
487
{
488
	if (!ctxt->has_seg_override)
489 490
		return 0;

491
	return ctxt->seg_override;
492 493
}

494 495
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
496
{
497 498 499
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
500
	return X86EMUL_PROPAGATE_FAULT;
501 502
}

503 504 505 506 507
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

508
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
509
{
510
	return emulate_exception(ctxt, GP_VECTOR, err, true);
511 512
}

513 514 515 516 517
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

518
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
519
{
520
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
521 522
}

523
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
524
{
525
	return emulate_exception(ctxt, TS_VECTOR, err, true);
526 527
}

528 529
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
530
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
531 532
}

A
Avi Kivity 已提交
533 534 535 536 537
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

558
static int __linearize(struct x86_emulate_ctxt *ctxt,
559
		     struct segmented_address addr,
560
		     unsigned size, bool write, bool fetch,
561 562
		     ulong *linear)
{
563 564
	struct desc_struct desc;
	bool usable;
565
	ulong la;
566
	u32 lim;
567
	u16 sel;
568
	unsigned cpl, rpl;
569

570
	la = seg_base(ctxt, addr.seg) + addr.ea;
571 572 573 574 575 576 577 578
	switch (ctxt->mode) {
	case X86EMUL_MODE_REAL:
		break;
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
579 580
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
581 582 583 584 585 586
		if (!usable)
			goto bad;
		/* code segment or read-only data segment */
		if (((desc.type & 8) || !(desc.type & 2)) && write)
			goto bad;
		/* unreadable code segment */
587
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
588 589 590 591 592 593 594 595 596 597 598 599 600 601
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
			/* exapand-down segment */
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
602
		cpl = ctxt->ops->cpl(ctxt);
603
		rpl = sel & 3;
604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
		cpl = max(cpl, rpl);
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
620
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
621 622 623
		la &= (u32)-1;
	*linear = la;
	return X86EMUL_CONTINUE;
624 625 626 627 628
bad:
	if (addr.seg == VCPU_SREG_SS)
		return emulate_ss(ctxt, addr.seg);
	else
		return emulate_gp(ctxt, addr.seg);
629 630
}

631 632 633 634 635 636 637 638 639
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


640 641 642 643 644
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
645 646 647
	int rc;
	ulong linear;

648
	rc = linearize(ctxt, addr, size, false, &linear);
649 650
	if (rc != X86EMUL_CONTINUE)
		return rc;
651
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
652 653
}

654 655 656 657 658 659 660 661
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
662
{
663
	struct fetch_cache *fc = &ctxt->fetch;
664
	int rc;
665
	int size, cur_size;
666

667
	if (ctxt->_eip == fc->end) {
668
		unsigned long linear;
669 670
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
671
		cur_size = fc->end - fc->start;
672 673
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
674
		rc = __linearize(ctxt, addr, size, false, true, &linear);
675
		if (unlikely(rc != X86EMUL_CONTINUE))
676
			return rc;
677 678
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
679
		if (unlikely(rc != X86EMUL_CONTINUE))
680
			return rc;
681
		fc->end += size;
682
	}
683 684
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
685
	return X86EMUL_CONTINUE;
686 687 688
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
689
			 void *dest, unsigned size)
690
{
691
	int rc;
692

693
	/* x86 instructions are limited to 15 bytes. */
694
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
695
		return X86EMUL_UNHANDLEABLE;
696
	while (size--) {
697
		rc = do_insn_fetch_byte(ctxt, dest++);
698
		if (rc != X86EMUL_CONTINUE)
699 700
			return rc;
	}
701
	return X86EMUL_CONTINUE;
702 703
}

704
/* Fetch next part of the instruction being emulated. */
705
#define insn_fetch(_type, _ctxt)					\
706
({	unsigned long _x;						\
707
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
708 709 710 711 712
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

713 714
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
715 716 717 718
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

719 720 721 722 723 724 725
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
static void *decode_register(u8 modrm_reg, unsigned long *regs,
			     int highbyte_regs)
A
Avi Kivity 已提交
726 727 728 729 730 731 732 733 734 735
{
	void *p;

	p = &regs[modrm_reg];
	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
		p = (unsigned char *)&regs[modrm_reg & 3] + 1;
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
736
			   struct segmented_address addr,
A
Avi Kivity 已提交
737 738 739 740 741 742 743
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
744
	rc = segmented_read_std(ctxt, addr, size, 2);
745
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
746
		return rc;
747
	addr.ea += 2;
748
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
749 750 751
	return rc;
}

752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786
static int test_cc(unsigned int condition, unsigned int flags)
{
	int rc = 0;

	switch ((condition & 15) >> 1) {
	case 0: /* o */
		rc |= (flags & EFLG_OF);
		break;
	case 1: /* b/c/nae */
		rc |= (flags & EFLG_CF);
		break;
	case 2: /* z/e */
		rc |= (flags & EFLG_ZF);
		break;
	case 3: /* be/na */
		rc |= (flags & (EFLG_CF|EFLG_ZF));
		break;
	case 4: /* s */
		rc |= (flags & EFLG_SF);
		break;
	case 5: /* p/pe */
		rc |= (flags & EFLG_PF);
		break;
	case 7: /* le/ng */
		rc |= (flags & EFLG_ZF);
		/* fall through */
	case 6: /* l/nge */
		rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
		break;
	}

	/* Odd condition identifiers (lsb == 1) have inverted sense. */
	return (!!rc ^ (condition & 1));
}

787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

A
Avi Kivity 已提交
805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
				    struct operand *op,
862 863
				    int inhibit_bytereg)
{
864 865
	unsigned reg = ctxt->modrm_reg;
	int highbyte_regs = ctxt->rex_prefix == 0;
866

867 868
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
869

870
	if (ctxt->d & Sse) {
A
Avi Kivity 已提交
871 872 873 874 875 876 877
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}

878
	op->type = OP_REG;
879 880
	if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
		op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
881 882
		op->bytes = 1;
	} else {
883 884
		op->addr.reg = decode_register(reg, ctxt->regs, 0);
		op->bytes = ctxt->op_bytes;
885
	}
886
	fetch_register_operand(op);
887 888 889
	op->orig_val = op->val;
}

890
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
891
			struct operand *op)
892 893
{
	u8 sib;
894
	int index_reg = 0, base_reg = 0, scale;
895
	int rc = X86EMUL_CONTINUE;
896
	ulong modrm_ea = 0;
897

898 899 900 901
	if (ctxt->rex_prefix) {
		ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1;	/* REX.R */
		index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
		ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
902 903
	}

904
	ctxt->modrm = insn_fetch(u8, ctxt);
905 906 907 908
	ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
	ctxt->modrm_rm |= (ctxt->modrm & 0x07);
	ctxt->modrm_seg = VCPU_SREG_DS;
909

910
	if (ctxt->modrm_mod == 3) {
911
		op->type = OP_REG;
912 913 914 915
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.reg = decode_register(ctxt->modrm_rm,
					       ctxt->regs, ctxt->d & ByteOp);
		if (ctxt->d & Sse) {
A
Avi Kivity 已提交
916 917
			op->type = OP_XMM;
			op->bytes = 16;
918 919
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
A
Avi Kivity 已提交
920 921
			return rc;
		}
922
		fetch_register_operand(op);
923 924 925
		return rc;
	}

926 927
	op->type = OP_MEM;

928 929 930 931 932
	if (ctxt->ad_bytes == 2) {
		unsigned bx = ctxt->regs[VCPU_REGS_RBX];
		unsigned bp = ctxt->regs[VCPU_REGS_RBP];
		unsigned si = ctxt->regs[VCPU_REGS_RSI];
		unsigned di = ctxt->regs[VCPU_REGS_RDI];
933 934

		/* 16-bit ModR/M decode. */
935
		switch (ctxt->modrm_mod) {
936
		case 0:
937
			if (ctxt->modrm_rm == 6)
938
				modrm_ea += insn_fetch(u16, ctxt);
939 940
			break;
		case 1:
941
			modrm_ea += insn_fetch(s8, ctxt);
942 943
			break;
		case 2:
944
			modrm_ea += insn_fetch(u16, ctxt);
945 946
			break;
		}
947
		switch (ctxt->modrm_rm) {
948
		case 0:
949
			modrm_ea += bx + si;
950 951
			break;
		case 1:
952
			modrm_ea += bx + di;
953 954
			break;
		case 2:
955
			modrm_ea += bp + si;
956 957
			break;
		case 3:
958
			modrm_ea += bp + di;
959 960
			break;
		case 4:
961
			modrm_ea += si;
962 963
			break;
		case 5:
964
			modrm_ea += di;
965 966
			break;
		case 6:
967
			if (ctxt->modrm_mod != 0)
968
				modrm_ea += bp;
969 970
			break;
		case 7:
971
			modrm_ea += bx;
972 973
			break;
		}
974 975 976
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
977
		modrm_ea = (u16)modrm_ea;
978 979
	} else {
		/* 32/64-bit ModR/M decode. */
980
		if ((ctxt->modrm_rm & 7) == 4) {
981
			sib = insn_fetch(u8, ctxt);
982 983 984 985
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

986
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
987
				modrm_ea += insn_fetch(s32, ctxt);
988
			else
989
				modrm_ea += ctxt->regs[base_reg];
990
			if (index_reg != 4)
991 992
				modrm_ea += ctxt->regs[index_reg] << scale;
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
993
			if (ctxt->mode == X86EMUL_MODE_PROT64)
994
				ctxt->rip_relative = 1;
995
		} else
996 997
			modrm_ea += ctxt->regs[ctxt->modrm_rm];
		switch (ctxt->modrm_mod) {
998
		case 0:
999
			if (ctxt->modrm_rm == 5)
1000
				modrm_ea += insn_fetch(s32, ctxt);
1001 1002
			break;
		case 1:
1003
			modrm_ea += insn_fetch(s8, ctxt);
1004 1005
			break;
		case 2:
1006
			modrm_ea += insn_fetch(s32, ctxt);
1007 1008 1009
			break;
		}
	}
1010
	op->addr.mem.ea = modrm_ea;
1011 1012 1013 1014 1015
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
1016
		      struct operand *op)
1017
{
1018
	int rc = X86EMUL_CONTINUE;
1019

1020
	op->type = OP_MEM;
1021
	switch (ctxt->ad_bytes) {
1022
	case 2:
1023
		op->addr.mem.ea = insn_fetch(u16, ctxt);
1024 1025
		break;
	case 4:
1026
		op->addr.mem.ea = insn_fetch(u32, ctxt);
1027 1028
		break;
	case 8:
1029
		op->addr.mem.ea = insn_fetch(u64, ctxt);
1030 1031 1032 1033 1034 1035
		break;
	}
done:
	return rc;
}

1036
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1037
{
1038
	long sv = 0, mask;
1039

1040 1041
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
		mask = ~(ctxt->dst.bytes * 8 - 1);
1042

1043 1044 1045 1046
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1047

1048
		ctxt->dst.addr.mem.ea += (sv >> 3);
1049
	}
1050 1051

	/* only subword offset */
1052
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1053 1054
}

1055 1056
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1057
{
1058
	int rc;
1059
	struct read_cache *mc = &ctxt->mem_read;
A
Avi Kivity 已提交
1060

1061 1062 1063 1064 1065
	while (size) {
		int n = min(size, 8u);
		size -= n;
		if (mc->pos < mc->end)
			goto read_cached;
1066

1067 1068
		rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
					      &ctxt->exception);
1069 1070 1071
		if (rc != X86EMUL_CONTINUE)
			return rc;
		mc->end += n;
A
Avi Kivity 已提交
1072

1073 1074 1075 1076 1077
	read_cached:
		memcpy(dest, mc->data + mc->pos, n);
		mc->pos += n;
		dest += n;
		addr += n;
A
Avi Kivity 已提交
1078
	}
1079 1080
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1081

1082 1083 1084 1085 1086
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1087 1088 1089
	int rc;
	ulong linear;

1090
	rc = linearize(ctxt, addr, size, false, &linear);
1091 1092
	if (rc != X86EMUL_CONTINUE)
		return rc;
1093
	return read_emulated(ctxt, linear, data, size);
1094 1095 1096 1097 1098 1099 1100
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1101 1102 1103
	int rc;
	ulong linear;

1104
	rc = linearize(ctxt, addr, size, true, &linear);
1105 1106
	if (rc != X86EMUL_CONTINUE)
		return rc;
1107 1108
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1109 1110 1111 1112 1113 1114 1115
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1116 1117 1118
	int rc;
	ulong linear;

1119
	rc = linearize(ctxt, addr, size, true, &linear);
1120 1121
	if (rc != X86EMUL_CONTINUE)
		return rc;
1122 1123
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1124 1125
}

1126 1127 1128 1129
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1130
	struct read_cache *rc = &ctxt->io_read;
1131

1132 1133
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1134 1135
		unsigned int count = ctxt->rep_prefix ?
			address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1136
		in_page = (ctxt->eflags & EFLG_DF) ?
1137 1138
			offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
			PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1139 1140 1141 1142 1143
		n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
			count);
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1144
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1145 1146
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1147 1148
	}

1149 1150 1151 1152
	memcpy(dest, rc->data + rc->pos, size);
	rc->pos += size;
	return 1;
}
A
Avi Kivity 已提交
1153

1154 1155 1156
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1157 1158
	struct x86_emulate_ops *ops = ctxt->ops;

1159 1160
	if (selector & 1 << 2) {
		struct desc_struct desc;
1161 1162
		u16 sel;

1163
		memset (dt, 0, sizeof *dt);
1164
		if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1165
			return;
1166

1167 1168 1169
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
		dt->address = get_desc_base(&desc);
	} else
1170
		ops->get_gdt(ctxt, dt);
1171
}
1172

1173 1174 1175 1176 1177 1178 1179
/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1180

1181
	get_descriptor_table_ptr(ctxt, selector, &dt);
1182

1183 1184
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1185

1186 1187 1188
	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
1189
}
1190

1191 1192 1193 1194 1195 1196 1197
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
A
Avi Kivity 已提交
1198

1199
	get_descriptor_table_ptr(ctxt, selector, &dt);
1200

1201 1202
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
A
Avi Kivity 已提交
1203

1204
	addr = dt.address + index * 8;
1205 1206
	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
				    &ctxt->exception);
1207
}
1208

1209
/* Does not support long mode */
1210 1211 1212 1213 1214 1215 1216 1217 1218
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
	struct desc_struct seg_desc;
	u8 dpl, rpl, cpl;
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
	int ret;
1219

1220
	memset(&seg_desc, 0, sizeof seg_desc);
1221

1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
	if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
	    || ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
		goto load;
	}

	/* NULL selector is not valid for TR, CS and SS */
	if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
	    && null_selector)
		goto exception;

	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

	if (null_selector) /* for NULL selector skip all following checks */
		goto load;

1245
	ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
	err_vec = GP_VECTOR;

	/* can't load system descriptor into segment selecor */
	if (seg <= VCPU_SREG_GS && !seg_desc.s)
		goto exception;

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	rpl = selector & 3;
	dpl = seg_desc.dpl;
1263
	cpl = ctxt->ops->cpl(ctxt);
1264 1265 1266 1267 1268 1269 1270 1271 1272

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1273
		break;
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1289
		break;
1290 1291 1292 1293 1294 1295 1296 1297 1298
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1299
		/*
1300 1301 1302
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1303
		 */
1304 1305 1306 1307
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1308
		break;
1309 1310 1311 1312 1313
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
		seg_desc.type |= 1;
1314
		ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1315 1316 1317 1318
		if (ret != X86EMUL_CONTINUE)
			return ret;
	}
load:
1319
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1320 1321 1322 1323 1324 1325
	return X86EMUL_CONTINUE;
exception:
	emulate_exception(ctxt, err_vec, err_code, true);
	return X86EMUL_PROPAGATE_FAULT;
}

1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
static void write_register_operand(struct operand *op)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (op->bytes) {
	case 1:
		*(u8 *)op->addr.reg = (u8)op->val;
		break;
	case 2:
		*(u16 *)op->addr.reg = (u16)op->val;
		break;
	case 4:
		*op->addr.reg = (u32)op->val;
		break;	/* 64b: zero-extend */
	case 8:
		*op->addr.reg = op->val;
		break;
	}
}

1345
static int writeback(struct x86_emulate_ctxt *ctxt)
1346 1347 1348
{
	int rc;

1349
	switch (ctxt->dst.type) {
1350
	case OP_REG:
1351
		write_register_operand(&ctxt->dst);
A
Avi Kivity 已提交
1352
		break;
1353
	case OP_MEM:
1354
		if (ctxt->lock_prefix)
1355
			rc = segmented_cmpxchg(ctxt,
1356 1357 1358 1359
					       ctxt->dst.addr.mem,
					       &ctxt->dst.orig_val,
					       &ctxt->dst.val,
					       ctxt->dst.bytes);
1360
		else
1361
			rc = segmented_write(ctxt,
1362 1363 1364
					     ctxt->dst.addr.mem,
					     &ctxt->dst.val,
					     ctxt->dst.bytes);
1365 1366
		if (rc != X86EMUL_CONTINUE)
			return rc;
1367
		break;
A
Avi Kivity 已提交
1368
	case OP_XMM:
1369
		write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
A
Avi Kivity 已提交
1370
		break;
1371 1372
	case OP_NONE:
		/* no writeback */
1373
		break;
1374
	default:
1375
		break;
A
Avi Kivity 已提交
1376
	}
1377 1378
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1379

1380
static int em_push(struct x86_emulate_ctxt *ctxt)
1381
{
1382
	struct segmented_address addr;
1383

1384 1385
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1386 1387 1388
	addr.seg = VCPU_SREG_SS;

	/* Disable writeback. */
1389 1390
	ctxt->dst.type = OP_NONE;
	return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1391
}
1392

1393 1394 1395 1396
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1397
	struct segmented_address addr;
1398

1399
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1400
	addr.seg = VCPU_SREG_SS;
1401
	rc = segmented_read(ctxt, addr, dest, len);
1402 1403 1404
	if (rc != X86EMUL_CONTINUE)
		return rc;

1405
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1406
	return rc;
1407 1408
}

1409 1410
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1411
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1412 1413
}

1414
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1415
			void *dest, int len)
1416 1417
{
	int rc;
1418 1419
	unsigned long val, change_mask;
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1420
	int cpl = ctxt->ops->cpl(ctxt);
1421

1422
	rc = emulate_pop(ctxt, &val, len);
1423 1424
	if (rc != X86EMUL_CONTINUE)
		return rc;
1425

1426 1427
	change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
		| EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1428

1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
			change_mask |= EFLG_IOPL;
		if (cpl <= iopl)
			change_mask |= EFLG_IF;
		break;
	case X86EMUL_MODE_VM86:
1439 1440
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1441 1442 1443 1444 1445
		change_mask |= EFLG_IF;
		break;
	default: /* real mode */
		change_mask |= (EFLG_IOPL | EFLG_IF);
		break;
1446
	}
1447 1448 1449 1450 1451

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1452 1453
}

1454 1455
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1456 1457 1458 1459
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1460 1461
}

1462
static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1463
{
1464 1465
	int seg = ctxt->src2.val;

1466
	ctxt->src.val = get_segment_selector(ctxt, seg);
1467

1468
	return em_push(ctxt);
1469 1470
}

1471
static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1472
{
1473
	int seg = ctxt->src2.val;
1474 1475
	unsigned long selector;
	int rc;
1476

1477
	rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1478 1479 1480
	if (rc != X86EMUL_CONTINUE)
		return rc;

1481
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1482
	return rc;
1483 1484
}

1485
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1486
{
1487
	unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1488 1489
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1490

1491 1492
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
1493
		(ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1494

1495
		rc = em_push(ctxt);
1496 1497
		if (rc != X86EMUL_CONTINUE)
			return rc;
1498

1499
		++reg;
1500 1501
	}

1502
	return rc;
1503 1504
}

1505 1506
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
1507
	ctxt->src.val =  (unsigned long)ctxt->eflags;
1508 1509 1510
	return em_push(ctxt);
}

1511
static int em_popa(struct x86_emulate_ctxt *ctxt)
1512
{
1513 1514
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
1515

1516 1517
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
1518 1519
			register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
							ctxt->op_bytes);
1520 1521
			--reg;
		}
1522

1523
		rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1524 1525 1526
		if (rc != X86EMUL_CONTINUE)
			break;
		--reg;
1527
	}
1528
	return rc;
1529 1530
}

1531
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1532
{
1533
	struct x86_emulate_ops *ops = ctxt->ops;
1534
	int rc;
1535 1536 1537 1538 1539 1540
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
1541
	ctxt->src.val = ctxt->eflags;
1542
	rc = em_push(ctxt);
1543 1544
	if (rc != X86EMUL_CONTINUE)
		return rc;
1545 1546 1547

	ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);

1548
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1549
	rc = em_push(ctxt);
1550 1551
	if (rc != X86EMUL_CONTINUE)
		return rc;
1552

1553
	ctxt->src.val = ctxt->_eip;
1554
	rc = em_push(ctxt);
1555 1556 1557
	if (rc != X86EMUL_CONTINUE)
		return rc;

1558
	ops->get_idt(ctxt, &dt);
1559 1560 1561 1562

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

1563
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1564 1565 1566
	if (rc != X86EMUL_CONTINUE)
		return rc;

1567
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1568 1569 1570
	if (rc != X86EMUL_CONTINUE)
		return rc;

1571
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1572 1573 1574
	if (rc != X86EMUL_CONTINUE)
		return rc;

1575
	ctxt->_eip = eip;
1576 1577 1578 1579

	return rc;
}

1580
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1581 1582 1583
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1584
		return emulate_int_real(ctxt, irq);
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

1595
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1596
{
1597 1598 1599 1600 1601 1602 1603 1604
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
	unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
			     EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
			     EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
	unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1605

1606
	/* TODO: Add stack limit check */
1607

1608
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1609

1610 1611
	if (rc != X86EMUL_CONTINUE)
		return rc;
1612

1613 1614
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
1615

1616
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1617

1618 1619
	if (rc != X86EMUL_CONTINUE)
		return rc;
1620

1621
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1622

1623 1624
	if (rc != X86EMUL_CONTINUE)
		return rc;
1625

1626
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1627

1628 1629
	if (rc != X86EMUL_CONTINUE)
		return rc;
1630

1631
	ctxt->_eip = temp_eip;
1632 1633


1634
	if (ctxt->op_bytes == 4)
1635
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1636
	else if (ctxt->op_bytes == 2) {
1637 1638
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
1639
	}
1640 1641 1642 1643 1644

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
	ctxt->eflags |= EFLG_RESERVED_ONE_MASK;

	return rc;
1645 1646
}

1647
static int em_iret(struct x86_emulate_ctxt *ctxt)
1648
{
1649 1650
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1651
		return emulate_iret_real(ctxt);
1652 1653 1654 1655
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
1656
	default:
1657 1658
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
1659 1660 1661
	}
}

1662 1663 1664 1665 1666
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned short sel;

1667
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1668

1669
	rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1670 1671 1672
	if (rc != X86EMUL_CONTINUE)
		return rc;

1673 1674
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1675 1676 1677
	return X86EMUL_CONTINUE;
}

1678
static int em_grp2(struct x86_emulate_ctxt *ctxt)
1679
{
1680
	switch (ctxt->modrm_reg) {
1681
	case 0:	/* rol */
1682
		emulate_2op_SrcB(ctxt, "rol");
1683 1684
		break;
	case 1:	/* ror */
1685
		emulate_2op_SrcB(ctxt, "ror");
1686 1687
		break;
	case 2:	/* rcl */
1688
		emulate_2op_SrcB(ctxt, "rcl");
1689 1690
		break;
	case 3:	/* rcr */
1691
		emulate_2op_SrcB(ctxt, "rcr");
1692 1693 1694
		break;
	case 4:	/* sal/shl */
	case 6:	/* sal/shl */
1695
		emulate_2op_SrcB(ctxt, "sal");
1696 1697
		break;
	case 5:	/* shr */
1698
		emulate_2op_SrcB(ctxt, "shr");
1699 1700
		break;
	case 7:	/* sar */
1701
		emulate_2op_SrcB(ctxt, "sar");
1702 1703
		break;
	}
1704
	return X86EMUL_CONTINUE;
1705 1706
}

1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735
static int em_not(struct x86_emulate_ctxt *ctxt)
{
	ctxt->dst.val = ~ctxt->dst.val;
	return X86EMUL_CONTINUE;
}

static int em_neg(struct x86_emulate_ctxt *ctxt)
{
	emulate_1op(ctxt, "neg");
	return X86EMUL_CONTINUE;
}

static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "mul", ex);
	return X86EMUL_CONTINUE;
}

static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "imul", ex);
	return X86EMUL_CONTINUE;
}

static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1736
{
1737
	u8 de = 0;
1738

1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
	emulate_1op_rax_rdx(ctxt, "div", de);
	if (de)
		return emulate_de(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 de = 0;

	emulate_1op_rax_rdx(ctxt, "idiv", de);
1750 1751
	if (de)
		return emulate_de(ctxt);
1752
	return X86EMUL_CONTINUE;
1753 1754
}

1755
static int em_grp45(struct x86_emulate_ctxt *ctxt)
1756
{
1757
	int rc = X86EMUL_CONTINUE;
1758

1759
	switch (ctxt->modrm_reg) {
1760
	case 0:	/* inc */
1761
		emulate_1op(ctxt, "inc");
1762 1763
		break;
	case 1:	/* dec */
1764
		emulate_1op(ctxt, "dec");
1765
		break;
1766 1767
	case 2: /* call near abs */ {
		long int old_eip;
1768 1769 1770
		old_eip = ctxt->_eip;
		ctxt->_eip = ctxt->src.val;
		ctxt->src.val = old_eip;
1771
		rc = em_push(ctxt);
1772 1773
		break;
	}
1774
	case 4: /* jmp abs */
1775
		ctxt->_eip = ctxt->src.val;
1776
		break;
1777 1778 1779
	case 5: /* jmp far */
		rc = em_jmp_far(ctxt);
		break;
1780
	case 6:	/* push */
1781
		rc = em_push(ctxt);
1782 1783
		break;
	}
1784
	return rc;
1785 1786
}

1787
static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1788
{
1789
	u64 old = ctxt->dst.orig_val64;
1790

1791 1792 1793 1794
	if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
	    ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
		ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
		ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1795
		ctxt->eflags &= ~EFLG_ZF;
1796
	} else {
1797 1798
		ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
			(u32) ctxt->regs[VCPU_REGS_RBX];
1799

1800
		ctxt->eflags |= EFLG_ZF;
1801
	}
1802
	return X86EMUL_CONTINUE;
1803 1804
}

1805 1806
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
1807 1808 1809
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
1810 1811 1812
	return em_pop(ctxt);
}

1813
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1814 1815 1816 1817
{
	int rc;
	unsigned long cs;

1818
	rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1819
	if (rc != X86EMUL_CONTINUE)
1820
		return rc;
1821 1822 1823
	if (ctxt->op_bytes == 4)
		ctxt->_eip = (u32)ctxt->_eip;
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1824
	if (rc != X86EMUL_CONTINUE)
1825
		return rc;
1826
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1827 1828 1829
	return rc;
}

1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847
static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
{
	/* Save real source value, then compare EAX against destination. */
	ctxt->src.orig_val = ctxt->src.val;
	ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
	emulate_2op_SrcV(ctxt, "cmp");

	if (ctxt->eflags & EFLG_ZF) {
		/* Success: write back to memory. */
		ctxt->dst.val = ctxt->src.orig_val;
	} else {
		/* Failure: write the value we saw to EAX. */
		ctxt->dst.type = OP_REG;
		ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
	}
	return X86EMUL_CONTINUE;
}

1848
static int em_lseg(struct x86_emulate_ctxt *ctxt)
1849
{
1850
	int seg = ctxt->src2.val;
1851 1852 1853
	unsigned short sel;
	int rc;

1854
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1855

1856
	rc = load_segment_descriptor(ctxt, sel, seg);
1857 1858 1859
	if (rc != X86EMUL_CONTINUE)
		return rc;

1860
	ctxt->dst.val = ctxt->src.val;
1861 1862 1863
	return rc;
}

1864
static void
1865
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1866
			struct desc_struct *cs, struct desc_struct *ss)
1867
{
1868 1869
	u16 selector;

1870
	memset(cs, 0, sizeof(struct desc_struct));
1871
	ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1872
	memset(ss, 0, sizeof(struct desc_struct));
1873 1874

	cs->l = 0;		/* will be adjusted later */
1875
	set_desc_base(cs, 0);	/* flat segment */
1876
	cs->g = 1;		/* 4kb granularity */
1877
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
1878 1879 1880
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
1881 1882
	cs->p = 1;
	cs->d = 1;
1883

1884 1885
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
1886 1887 1888
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
1889
	ss->d = 1;		/* 32bit stack segment */
1890
	ss->dpl = 0;
1891
	ss->p = 1;
1892 1893
}

1894
static int em_syscall(struct x86_emulate_ctxt *ctxt)
1895
{
1896
	struct x86_emulate_ops *ops = ctxt->ops;
1897
	struct desc_struct cs, ss;
1898
	u64 msr_data;
1899
	u16 cs_sel, ss_sel;
1900
	u64 efer = 0;
1901 1902

	/* syscall is not available in real mode */
1903
	if (ctxt->mode == X86EMUL_MODE_REAL ||
1904 1905
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
1906

1907
	ops->get_msr(ctxt, MSR_EFER, &efer);
1908
	setup_syscalls_segments(ctxt, &cs, &ss);
1909

1910
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
1911
	msr_data >>= 32;
1912 1913
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
1914

1915
	if (efer & EFER_LMA) {
1916
		cs.d = 0;
1917 1918
		cs.l = 1;
	}
1919 1920
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1921

1922
	ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1923
	if (efer & EFER_LMA) {
1924
#ifdef CONFIG_X86_64
1925
		ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1926

1927
		ops->get_msr(ctxt,
1928 1929
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
1930
		ctxt->_eip = msr_data;
1931

1932
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1933 1934 1935 1936
		ctxt->eflags &= ~(msr_data | EFLG_RF);
#endif
	} else {
		/* legacy mode */
1937
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
1938
		ctxt->_eip = (u32)msr_data;
1939 1940 1941 1942

		ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
	}

1943
	return X86EMUL_CONTINUE;
1944 1945
}

1946
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1947
{
1948
	struct x86_emulate_ops *ops = ctxt->ops;
1949
	struct desc_struct cs, ss;
1950
	u64 msr_data;
1951
	u16 cs_sel, ss_sel;
1952
	u64 efer = 0;
1953

1954
	ops->get_msr(ctxt, MSR_EFER, &efer);
1955
	/* inject #GP if in real mode */
1956 1957
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
1958 1959 1960 1961

	/* XXX sysenter/sysexit have not been tested in 64bit mode.
	* Therefore, we inject an #UD.
	*/
1962 1963
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return emulate_ud(ctxt);
1964

1965
	setup_syscalls_segments(ctxt, &cs, &ss);
1966

1967
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1968 1969
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT32:
1970 1971
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
1972 1973
		break;
	case X86EMUL_MODE_PROT64:
1974 1975
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
1976 1977 1978 1979
		break;
	}

	ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1980 1981 1982 1983
	cs_sel = (u16)msr_data;
	cs_sel &= ~SELECTOR_RPL_MASK;
	ss_sel = cs_sel + 8;
	ss_sel &= ~SELECTOR_RPL_MASK;
1984
	if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1985
		cs.d = 0;
1986 1987 1988
		cs.l = 1;
	}

1989 1990
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1991

1992
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1993
	ctxt->_eip = msr_data;
1994

1995
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1996
	ctxt->regs[VCPU_REGS_RSP] = msr_data;
1997

1998
	return X86EMUL_CONTINUE;
1999 2000
}

2001
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2002
{
2003
	struct x86_emulate_ops *ops = ctxt->ops;
2004
	struct desc_struct cs, ss;
2005 2006
	u64 msr_data;
	int usermode;
X
Xiao Guangrong 已提交
2007
	u16 cs_sel = 0, ss_sel = 0;
2008

2009 2010
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2011 2012
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
2013

2014
	setup_syscalls_segments(ctxt, &cs, &ss);
2015

2016
	if ((ctxt->rex_prefix & 0x8) != 0x0)
2017 2018 2019 2020 2021 2022
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

	cs.dpl = 3;
	ss.dpl = 3;
2023
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2024 2025
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
2026
		cs_sel = (u16)(msr_data + 16);
2027 2028
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2029
		ss_sel = (u16)(msr_data + 24);
2030 2031
		break;
	case X86EMUL_MODE_PROT64:
2032
		cs_sel = (u16)(msr_data + 32);
2033 2034
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2035 2036
		ss_sel = cs_sel + 8;
		cs.d = 0;
2037 2038 2039
		cs.l = 1;
		break;
	}
2040 2041
	cs_sel |= SELECTOR_RPL_MASK;
	ss_sel |= SELECTOR_RPL_MASK;
2042

2043 2044
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2045

2046 2047
	ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
	ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2048

2049
	return X86EMUL_CONTINUE;
2050 2051
}

2052
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2053 2054 2055 2056 2057 2058 2059
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2060
	return ctxt->ops->cpl(ctxt) > iopl;
2061 2062 2063 2064 2065
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2066
	struct x86_emulate_ops *ops = ctxt->ops;
2067
	struct desc_struct tr_seg;
2068
	u32 base3;
2069
	int r;
2070
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2071
	unsigned mask = (1 << len) - 1;
2072
	unsigned long base;
2073

2074
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2075
	if (!tr_seg.p)
2076
		return false;
2077
	if (desc_limit_scaled(&tr_seg) < 103)
2078
		return false;
2079 2080 2081 2082
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2083
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2084 2085
	if (r != X86EMUL_CONTINUE)
		return false;
2086
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2087
		return false;
2088
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2089 2090 2091 2092 2093 2094 2095 2096 2097 2098
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2099 2100 2101
	if (ctxt->perm_ok)
		return true;

2102 2103
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2104
			return false;
2105 2106 2107

	ctxt->perm_ok = true;

2108 2109 2110
	return true;
}

2111 2112 2113
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
2114
	tss->ip = ctxt->_eip;
2115
	tss->flag = ctxt->eflags;
2116 2117 2118 2119 2120 2121 2122 2123
	tss->ax = ctxt->regs[VCPU_REGS_RAX];
	tss->cx = ctxt->regs[VCPU_REGS_RCX];
	tss->dx = ctxt->regs[VCPU_REGS_RDX];
	tss->bx = ctxt->regs[VCPU_REGS_RBX];
	tss->sp = ctxt->regs[VCPU_REGS_RSP];
	tss->bp = ctxt->regs[VCPU_REGS_RBP];
	tss->si = ctxt->regs[VCPU_REGS_RSI];
	tss->di = ctxt->regs[VCPU_REGS_RDI];
2124

2125 2126 2127 2128 2129
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2130 2131 2132 2133 2134 2135 2136
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;

2137
	ctxt->_eip = tss->ip;
2138
	ctxt->eflags = tss->flag | 2;
2139 2140 2141 2142 2143 2144 2145 2146
	ctxt->regs[VCPU_REGS_RAX] = tss->ax;
	ctxt->regs[VCPU_REGS_RCX] = tss->cx;
	ctxt->regs[VCPU_REGS_RDX] = tss->dx;
	ctxt->regs[VCPU_REGS_RBX] = tss->bx;
	ctxt->regs[VCPU_REGS_RSP] = tss->sp;
	ctxt->regs[VCPU_REGS_RBP] = tss->bp;
	ctxt->regs[VCPU_REGS_RSI] = tss->si;
	ctxt->regs[VCPU_REGS_RDI] = tss->di;
2147 2148 2149 2150 2151

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2152 2153 2154 2155 2156
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2157 2158 2159 2160 2161

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2162
	ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2163 2164
	if (ret != X86EMUL_CONTINUE)
		return ret;
2165
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2166 2167
	if (ret != X86EMUL_CONTINUE)
		return ret;
2168
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2169 2170
	if (ret != X86EMUL_CONTINUE)
		return ret;
2171
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2172 2173
	if (ret != X86EMUL_CONTINUE)
		return ret;
2174
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2175 2176 2177 2178 2179 2180 2181 2182 2183 2184
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2185
	struct x86_emulate_ops *ops = ctxt->ops;
2186 2187
	struct tss_segment_16 tss_seg;
	int ret;
2188
	u32 new_tss_base = get_desc_base(new_desc);
2189

2190
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2191
			    &ctxt->exception);
2192
	if (ret != X86EMUL_CONTINUE)
2193 2194 2195
		/* FIXME: need to provide precise fault address */
		return ret;

2196
	save_state_to_tss16(ctxt, &tss_seg);
2197

2198
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2199
			     &ctxt->exception);
2200
	if (ret != X86EMUL_CONTINUE)
2201 2202 2203
		/* FIXME: need to provide precise fault address */
		return ret;

2204
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2205
			    &ctxt->exception);
2206
	if (ret != X86EMUL_CONTINUE)
2207 2208 2209 2210 2211 2212
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2213
		ret = ops->write_std(ctxt, new_tss_base,
2214 2215
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2216
				     &ctxt->exception);
2217
		if (ret != X86EMUL_CONTINUE)
2218 2219 2220 2221
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2222
	return load_state_from_tss16(ctxt, &tss_seg);
2223 2224 2225 2226 2227
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
2228
	tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2229
	tss->eip = ctxt->_eip;
2230
	tss->eflags = ctxt->eflags;
2231 2232 2233 2234 2235 2236 2237 2238
	tss->eax = ctxt->regs[VCPU_REGS_RAX];
	tss->ecx = ctxt->regs[VCPU_REGS_RCX];
	tss->edx = ctxt->regs[VCPU_REGS_RDX];
	tss->ebx = ctxt->regs[VCPU_REGS_RBX];
	tss->esp = ctxt->regs[VCPU_REGS_RSP];
	tss->ebp = ctxt->regs[VCPU_REGS_RBP];
	tss->esi = ctxt->regs[VCPU_REGS_RSI];
	tss->edi = ctxt->regs[VCPU_REGS_RDI];
2239

2240 2241 2242 2243 2244 2245 2246
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
	tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2247 2248 2249 2250 2251 2252 2253
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;

2254
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2255
		return emulate_gp(ctxt, 0);
2256
	ctxt->_eip = tss->eip;
2257
	ctxt->eflags = tss->eflags | 2;
2258 2259 2260 2261 2262 2263 2264 2265
	ctxt->regs[VCPU_REGS_RAX] = tss->eax;
	ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
	ctxt->regs[VCPU_REGS_RDX] = tss->edx;
	ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
	ctxt->regs[VCPU_REGS_RSP] = tss->esp;
	ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
	ctxt->regs[VCPU_REGS_RSI] = tss->esi;
	ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2266 2267 2268 2269 2270

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2271 2272 2273 2274 2275 2276 2277
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2278 2279 2280 2281 2282

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2283
	ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2284 2285
	if (ret != X86EMUL_CONTINUE)
		return ret;
2286
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2287 2288
	if (ret != X86EMUL_CONTINUE)
		return ret;
2289
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2290 2291
	if (ret != X86EMUL_CONTINUE)
		return ret;
2292
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2293 2294
	if (ret != X86EMUL_CONTINUE)
		return ret;
2295
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2296 2297
	if (ret != X86EMUL_CONTINUE)
		return ret;
2298
	ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2299 2300
	if (ret != X86EMUL_CONTINUE)
		return ret;
2301
	ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2302 2303 2304 2305 2306 2307 2308 2309 2310 2311
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2312
	struct x86_emulate_ops *ops = ctxt->ops;
2313 2314
	struct tss_segment_32 tss_seg;
	int ret;
2315
	u32 new_tss_base = get_desc_base(new_desc);
2316

2317
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2318
			    &ctxt->exception);
2319
	if (ret != X86EMUL_CONTINUE)
2320 2321 2322
		/* FIXME: need to provide precise fault address */
		return ret;

2323
	save_state_to_tss32(ctxt, &tss_seg);
2324

2325
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2326
			     &ctxt->exception);
2327
	if (ret != X86EMUL_CONTINUE)
2328 2329 2330
		/* FIXME: need to provide precise fault address */
		return ret;

2331
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2332
			    &ctxt->exception);
2333
	if (ret != X86EMUL_CONTINUE)
2334 2335 2336 2337 2338 2339
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2340
		ret = ops->write_std(ctxt, new_tss_base,
2341 2342
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2343
				     &ctxt->exception);
2344
		if (ret != X86EMUL_CONTINUE)
2345 2346 2347 2348
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2349
	return load_state_from_tss32(ctxt, &tss_seg);
2350 2351 2352
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2353 2354
				   u16 tss_selector, int reason,
				   bool has_error_code, u32 error_code)
2355
{
2356
	struct x86_emulate_ops *ops = ctxt->ops;
2357 2358
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
2359
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2360
	ulong old_tss_base =
2361
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2362
	u32 desc_limit;
2363 2364 2365

	/* FIXME: old_tss_base == ~0 ? */

2366
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2367 2368
	if (ret != X86EMUL_CONTINUE)
		return ret;
2369
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2370 2371 2372 2373 2374 2375 2376
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

	if (reason != TASK_SWITCH_IRET) {
		if ((tss_selector & 3) > next_tss_desc.dpl ||
2377
		    ops->cpl(ctxt) > next_tss_desc.dpl)
2378
			return emulate_gp(ctxt, 0);
2379 2380
	}

2381 2382 2383 2384
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
2385
		emulate_ts(ctxt, tss_selector & 0xfffc);
2386 2387 2388 2389 2390
		return X86EMUL_PROPAGATE_FAULT;
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2391
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
	   note that old_tss_sel is not used afetr this point */
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
2403
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2404 2405
				     old_tss_base, &next_tss_desc);
	else
2406
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2407
				     old_tss_base, &next_tss_desc);
2408 2409
	if (ret != X86EMUL_CONTINUE)
		return ret;
2410 2411 2412 2413 2414 2415

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
2416
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2417 2418
	}

2419
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
2420
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2421

2422
	if (has_error_code) {
2423 2424 2425
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
2426
		ret = em_push(ctxt);
2427 2428
	}

2429 2430 2431 2432
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2433 2434
			 u16 tss_selector, int reason,
			 bool has_error_code, u32 error_code)
2435 2436 2437
{
	int rc;

2438 2439
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
2440

2441
	rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2442
				     has_error_code, error_code);
2443

2444
	if (rc == X86EMUL_CONTINUE)
2445
		ctxt->eip = ctxt->_eip;
2446

2447
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2448 2449
}

2450
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2451
			    int reg, struct operand *op)
2452 2453 2454
{
	int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;

2455 2456
	register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2457
	op->addr.mem.seg = seg;
2458 2459
}

2460 2461 2462 2463 2464 2465
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
2466
	al = ctxt->dst.val;
2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

2484
	ctxt->dst.val = al;
2485
	/* Set PF, ZF, SF */
2486 2487 2488
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
2489
	emulate_2op_SrcV(ctxt, "or");
2490 2491 2492 2493 2494 2495 2496 2497
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

2498 2499 2500 2501 2502 2503 2504 2505 2506
static int em_call(struct x86_emulate_ctxt *ctxt)
{
	long rel = ctxt->src.val;

	ctxt->src.val = (unsigned long)ctxt->_eip;
	jmp_rel(ctxt, rel);
	return em_push(ctxt);
}

2507 2508 2509 2510 2511 2512
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;

2513
	old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2514
	old_eip = ctxt->_eip;
2515

2516
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2517
	if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2518 2519
		return X86EMUL_CONTINUE;

2520 2521
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2522

2523
	ctxt->src.val = old_cs;
2524
	rc = em_push(ctxt);
2525 2526 2527
	if (rc != X86EMUL_CONTINUE)
		return rc;

2528
	ctxt->src.val = old_eip;
2529
	return em_push(ctxt);
2530 2531
}

2532 2533 2534 2535
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2536 2537 2538 2539
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
	rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2540 2541
	if (rc != X86EMUL_CONTINUE)
		return rc;
2542
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2543 2544 2545
	return X86EMUL_CONTINUE;
}

2546 2547
static int em_add(struct x86_emulate_ctxt *ctxt)
{
2548
	emulate_2op_SrcV(ctxt, "add");
2549 2550 2551 2552 2553
	return X86EMUL_CONTINUE;
}

static int em_or(struct x86_emulate_ctxt *ctxt)
{
2554
	emulate_2op_SrcV(ctxt, "or");
2555 2556 2557 2558 2559
	return X86EMUL_CONTINUE;
}

static int em_adc(struct x86_emulate_ctxt *ctxt)
{
2560
	emulate_2op_SrcV(ctxt, "adc");
2561 2562 2563 2564 2565
	return X86EMUL_CONTINUE;
}

static int em_sbb(struct x86_emulate_ctxt *ctxt)
{
2566
	emulate_2op_SrcV(ctxt, "sbb");
2567 2568 2569 2570 2571
	return X86EMUL_CONTINUE;
}

static int em_and(struct x86_emulate_ctxt *ctxt)
{
2572
	emulate_2op_SrcV(ctxt, "and");
2573 2574 2575 2576 2577
	return X86EMUL_CONTINUE;
}

static int em_sub(struct x86_emulate_ctxt *ctxt)
{
2578
	emulate_2op_SrcV(ctxt, "sub");
2579 2580 2581 2582 2583
	return X86EMUL_CONTINUE;
}

static int em_xor(struct x86_emulate_ctxt *ctxt)
{
2584
	emulate_2op_SrcV(ctxt, "xor");
2585 2586 2587 2588 2589
	return X86EMUL_CONTINUE;
}

static int em_cmp(struct x86_emulate_ctxt *ctxt)
{
2590
	emulate_2op_SrcV(ctxt, "cmp");
2591
	/* Disable writeback. */
2592
	ctxt->dst.type = OP_NONE;
2593 2594 2595
	return X86EMUL_CONTINUE;
}

2596 2597
static int em_test(struct x86_emulate_ctxt *ctxt)
{
2598
	emulate_2op_SrcV(ctxt, "test");
2599 2600
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
2601 2602 2603
	return X86EMUL_CONTINUE;
}

2604 2605 2606
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
2607 2608
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
2609 2610

	/* Write back the memory destination with implicit LOCK prefix. */
2611 2612
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
2613 2614 2615
	return X86EMUL_CONTINUE;
}

2616
static int em_imul(struct x86_emulate_ctxt *ctxt)
2617
{
2618
	emulate_2op_SrcV_nobyte(ctxt, "imul");
2619 2620 2621
	return X86EMUL_CONTINUE;
}

2622 2623
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
2624
	ctxt->dst.val = ctxt->src2.val;
2625 2626 2627
	return em_imul(ctxt);
}

2628 2629
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
2630 2631 2632 2633
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
	ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2634 2635 2636 2637

	return X86EMUL_CONTINUE;
}

2638 2639 2640 2641
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

2642
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2643 2644
	ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
	ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2645 2646 2647
	return X86EMUL_CONTINUE;
}

2648 2649
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
2650
	ctxt->dst.val = ctxt->src.val;
2651 2652 2653
	return X86EMUL_CONTINUE;
}

2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681
static int em_cr_write(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_dr_write(struct x86_emulate_ctxt *ctxt)
{
	unsigned long val;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		val = ctxt->src.val & ~0ULL;
	else
		val = ctxt->src.val & ~0U;

	/* #UD condition is already handled. */
	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705
static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

	msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
		| ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
	if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

	if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
		return emulate_gp(ctxt, 0);

	ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
	ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
	return X86EMUL_CONTINUE;
}

2706 2707
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
2708
	if (ctxt->modrm_reg > VCPU_SREG_GS)
2709 2710
		return emulate_ud(ctxt);

2711
	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2712 2713 2714 2715 2716
	return X86EMUL_CONTINUE;
}

static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
2717
	u16 sel = ctxt->src.val;
2718

2719
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2720 2721
		return emulate_ud(ctxt);

2722
	if (ctxt->modrm_reg == VCPU_SREG_SS)
2723 2724 2725
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
2726 2727
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2728 2729
}

2730 2731
static int em_movdqu(struct x86_emulate_ctxt *ctxt)
{
2732
	memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2733 2734 2735
	return X86EMUL_CONTINUE;
}

2736 2737
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
2738 2739 2740
	int rc;
	ulong linear;

2741
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2742
	if (rc == X86EMUL_CONTINUE)
2743
		ctxt->ops->invlpg(ctxt, linear);
2744
	/* Disable writeback. */
2745
	ctxt->dst.type = OP_NONE;
2746 2747 2748
	return X86EMUL_CONTINUE;
}

2749 2750 2751 2752 2753 2754 2755 2756 2757 2758
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

2759 2760 2761 2762
static int em_vmcall(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2763
	if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2764 2765 2766 2767 2768 2769 2770
		return X86EMUL_UNHANDLEABLE;

	rc = ctxt->ops->fix_hypercall(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
2771
	ctxt->_eip = ctxt->eip;
2772
	/* Disable writeback. */
2773
	ctxt->dst.type = OP_NONE;
2774 2775 2776 2777 2778 2779 2780 2781
	return X86EMUL_CONTINUE;
}

static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

2782
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2783
			     &desc_ptr.size, &desc_ptr.address,
2784
			     ctxt->op_bytes);
2785 2786 2787 2788
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_gdt(ctxt, &desc_ptr);
	/* Disable writeback. */
2789
	ctxt->dst.type = OP_NONE;
2790 2791 2792
	return X86EMUL_CONTINUE;
}

2793
static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2794 2795 2796
{
	int rc;

2797 2798
	rc = ctxt->ops->fix_hypercall(ctxt);

2799
	/* Disable writeback. */
2800
	ctxt->dst.type = OP_NONE;
2801 2802 2803 2804 2805 2806 2807 2808
	return rc;
}

static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

2809
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2810
			     &desc_ptr.size, &desc_ptr.address,
2811
			     ctxt->op_bytes);
2812 2813 2814 2815
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_idt(ctxt, &desc_ptr);
	/* Disable writeback. */
2816
	ctxt->dst.type = OP_NONE;
2817 2818 2819 2820 2821
	return X86EMUL_CONTINUE;
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
2822 2823
	ctxt->dst.bytes = 2;
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2824 2825 2826 2827 2828 2829
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2830 2831
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
2832 2833 2834
	return X86EMUL_CONTINUE;
}

2835 2836
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
2837 2838 2839 2840
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
	if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
		jmp_rel(ctxt, ctxt->src.val);
2841 2842 2843 2844 2845 2846

	return X86EMUL_CONTINUE;
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
2847 2848
	if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
		jmp_rel(ctxt, ctxt->src.val);
2849 2850 2851 2852

	return X86EMUL_CONTINUE;
}

2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870
static int em_in(struct x86_emulate_ctxt *ctxt)
{
	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
			     &ctxt->dst.val))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
}

static int em_out(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				    &ctxt->src.val, 1);
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918
static int em_bt(struct x86_emulate_ctxt *ctxt)
{
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	/* only subword offset */
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;

	emulate_2op_SrcV_nobyte(ctxt, "bt");
	return X86EMUL_CONTINUE;
}

static int em_bts(struct x86_emulate_ctxt *ctxt)
{
	emulate_2op_SrcV_nobyte(ctxt, "bts");
	return X86EMUL_CONTINUE;
}

static int em_btr(struct x86_emulate_ctxt *ctxt)
{
	emulate_2op_SrcV_nobyte(ctxt, "btr");
	return X86EMUL_CONTINUE;
}

static int em_btc(struct x86_emulate_ctxt *ctxt)
{
	emulate_2op_SrcV_nobyte(ctxt, "btc");
	return X86EMUL_CONTINUE;
}

2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952
static int em_bsf(struct x86_emulate_ctxt *ctxt)
{
	u8 zf;

	__asm__ ("bsf %2, %0; setz %1"
		 : "=r"(ctxt->dst.val), "=q"(zf)
		 : "r"(ctxt->src.val));

	ctxt->eflags &= ~X86_EFLAGS_ZF;
	if (zf) {
		ctxt->eflags |= X86_EFLAGS_ZF;
		/* Disable writeback. */
		ctxt->dst.type = OP_NONE;
	}
	return X86EMUL_CONTINUE;
}

static int em_bsr(struct x86_emulate_ctxt *ctxt)
{
	u8 zf;

	__asm__ ("bsr %2, %0; setz %1"
		 : "=r"(ctxt->dst.val), "=q"(zf)
		 : "r"(ctxt->src.val));

	ctxt->eflags &= ~X86_EFLAGS_ZF;
	if (zf) {
		ctxt->eflags |= X86_EFLAGS_ZF;
		/* Disable writeback. */
		ctxt->dst.type = OP_NONE;
	}
	return X86EMUL_CONTINUE;
}

2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
2967
	if (!valid_cr(ctxt->modrm_reg))
2968 2969 2970 2971 2972 2973 2974
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
2975 2976
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
2977
	u64 efer = 0;
2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
2995
		u64 cr4;
2996 2997 2998 2999
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

3000 3001
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3002 3003 3004 3005 3006 3007 3008 3009 3010 3011

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

3012 3013
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
3014
			rsvd = CR3_L_MODE_RESERVED_BITS;
3015
		else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3016
			rsvd = CR3_PAE_RESERVED_BITS;
3017
		else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3018 3019 3020 3021 3022 3023 3024 3025
			rsvd = CR3_NONPAE_RESERVED_BITS;

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
3026
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

3038 3039 3040 3041
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

3042
	ctxt->ops->get_dr(ctxt, 7, &dr7);
3043 3044 3045 3046 3047 3048 3049

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
3050
	int dr = ctxt->modrm_reg;
3051 3052 3053 3054 3055
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

3056
	cr4 = ctxt->ops->get_cr(ctxt, 4);
3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

	if (check_dr7_gd(ctxt))
		return emulate_db(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
3068 3069
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
3070 3071 3072 3073 3074 3075 3076

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

3077 3078 3079 3080
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
	u64 efer;

3081
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3082 3083 3084 3085 3086 3087 3088 3089 3090

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
3091
	u64 rax = ctxt->regs[VCPU_REGS_RAX];
3092 3093

	/* Valid physical address? */
3094
	if (rax & 0xffff000000000000ULL)
3095 3096 3097 3098 3099
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

3100 3101
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
3102
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3103

3104
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3105 3106 3107 3108 3109
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

3110 3111
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
3112
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3113
	u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3114

3115
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3116 3117 3118 3119 3120 3121
	    (rcx > 3))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3122 3123
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
3124 3125
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3126 3127 3128 3129 3130 3131 3132
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
3133 3134
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3135 3136 3137 3138 3139
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3140
#define D(_y) { .flags = (_y) }
3141
#define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3142 3143
#define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
		      .check_perm = (_p) }
3144
#define N    D(0)
3145
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3146
#define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3147
#define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3148
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3149 3150
#define II(_f, _e, _i) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3151 3152 3153
#define IIP(_f, _e, _i, _p) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
	  .check_perm = (_p) }
3154
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3155

3156
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
3157
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3158
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
3159 3160
#define I2bvIP(_f, _e, _i, _p) \
	IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3161

3162 3163 3164
#define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3165

3166 3167 3168 3169 3170 3171
static struct opcode group7_rm1[] = {
	DI(SrcNone | ModRM | Priv, monitor),
	DI(SrcNone | ModRM | Priv, mwait),
	N, N, N, N, N, N,
};

3172 3173
static struct opcode group7_rm3[] = {
	DIP(SrcNone | ModRM | Prot | Priv, vmrun,   check_svme_pa),
3174
	II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3175 3176 3177 3178 3179 3180 3181
	DIP(SrcNone | ModRM | Prot | Priv, vmload,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, vmsave,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, stgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, clgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, skinit,  check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
};
3182

3183 3184 3185 3186 3187
static struct opcode group7_rm7[] = {
	N,
	DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
	N, N, N, N, N, N,
};
3188

3189
static struct opcode group1[] = {
3190
	I(Lock, em_add),
3191
	I(Lock | PageTable, em_or),
3192 3193
	I(Lock, em_adc),
	I(Lock, em_sbb),
3194
	I(Lock | PageTable, em_and),
3195 3196 3197
	I(Lock, em_sub),
	I(Lock, em_xor),
	I(0, em_cmp),
3198 3199 3200
};

static struct opcode group1A[] = {
3201
	I(DstMem | SrcNone | ModRM | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3202 3203 3204
};

static struct opcode group3[] = {
3205 3206 3207 3208 3209 3210 3211 3212
	I(DstMem | SrcImm | ModRM, em_test),
	I(DstMem | SrcImm | ModRM, em_test),
	I(DstMem | SrcNone | ModRM | Lock, em_not),
	I(DstMem | SrcNone | ModRM | Lock, em_neg),
	I(SrcMem | ModRM, em_mul_ex),
	I(SrcMem | ModRM, em_imul_ex),
	I(SrcMem | ModRM, em_div_ex),
	I(SrcMem | ModRM, em_idiv_ex),
3213 3214 3215
};

static struct opcode group4[] = {
3216 3217
	I(ByteOp | DstMem | SrcNone | ModRM | Lock, em_grp45),
	I(ByteOp | DstMem | SrcNone | ModRM | Lock, em_grp45),
3218 3219 3220 3221
	N, N, N, N, N, N,
};

static struct opcode group5[] = {
3222 3223 3224
	I(DstMem | SrcNone | ModRM | Lock, em_grp45),
	I(DstMem | SrcNone | ModRM | Lock, em_grp45),
	I(SrcMem | ModRM | Stack, em_grp45),
3225
	I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3226 3227 3228
	I(SrcMem | ModRM | Stack, em_grp45),
	I(SrcMemFAddr | ModRM | ImplicitOps, em_grp45),
	I(SrcMem | ModRM | Stack, em_grp45), N,
3229 3230
};

3231 3232 3233 3234 3235 3236 3237 3238
static struct opcode group6[] = {
	DI(ModRM | Prot,        sldt),
	DI(ModRM | Prot,        str),
	DI(ModRM | Prot | Priv, lldt),
	DI(ModRM | Prot | Priv, ltr),
	N, N, N, N,
};

3239
static struct group_dual group7 = { {
3240 3241
	DI(ModRM | Mov | DstMem | Priv, sgdt),
	DI(ModRM | Mov | DstMem | Priv, sidt),
3242 3243 3244 3245 3246
	II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
	II(ModRM | SrcMem | Priv, em_lidt, lidt),
	II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
	II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
	II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3247
}, {
3248 3249
	I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
	EXT(0, group7_rm1),
3250
	N, EXT(0, group7_rm3),
3251 3252
	II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
	II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3253 3254 3255 3256
} };

static struct opcode group8[] = {
	N, N, N, N,
3257 3258 3259 3260
	I(DstMem | SrcImmByte | ModRM, em_bt),
	I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_bts),
	I(DstMem | SrcImmByte | ModRM | Lock, em_btr),
	I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_btc),
3261 3262 3263
};

static struct group_dual group9 = { {
3264
	N, I(DstMem64 | ModRM | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3265 3266 3267 3268
}, {
	N, N, N, N, N, N, N, N,
} };

3269
static struct opcode group11[] = {
3270 3271
	I(DstMem | SrcImm | ModRM | Mov | PageTable, em_mov),
	X7(D(Undefined)),
3272 3273
};

3274 3275 3276 3277
static struct gprefix pfx_0f_6f_0f_7f = {
	N, N, N, I(Sse, em_movdqu),
};

3278 3279
static struct opcode opcode_table[256] = {
	/* 0x00 - 0x07 */
3280
	I6ALU(Lock, em_add),
3281 3282
	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3283
	/* 0x08 - 0x0F */
3284
	I6ALU(Lock | PageTable, em_or),
3285 3286
	I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
	N,
3287
	/* 0x10 - 0x17 */
3288
	I6ALU(Lock, em_adc),
3289 3290
	I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3291
	/* 0x18 - 0x1F */
3292
	I6ALU(Lock, em_sbb),
3293 3294
	I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3295
	/* 0x20 - 0x27 */
3296
	I6ALU(Lock | PageTable, em_and), N, N,
3297
	/* 0x28 - 0x2F */
3298
	I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3299
	/* 0x30 - 0x37 */
3300
	I6ALU(Lock, em_xor), N, N,
3301
	/* 0x38 - 0x3F */
3302
	I6ALU(0, em_cmp), N, N,
3303 3304 3305
	/* 0x40 - 0x4F */
	X16(D(DstReg)),
	/* 0x50 - 0x57 */
3306
	X8(I(SrcReg | Stack, em_push)),
3307
	/* 0x58 - 0x5F */
3308
	X8(I(DstReg | Stack, em_pop)),
3309
	/* 0x60 - 0x67 */
3310 3311
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
3312 3313 3314
	N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
	N, N, N, N,
	/* 0x68 - 0x6F */
3315 3316
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3317 3318
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3319 3320
	I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
	I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3321 3322 3323 3324 3325 3326 3327
	/* 0x70 - 0x7F */
	X16(D(SrcImmByte)),
	/* 0x80 - 0x87 */
	G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
	G(DstMem | SrcImm | ModRM | Group, group1),
	G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
	G(DstMem | SrcImmByte | ModRM | Group, group1),
3328
	I2bv(DstMem | SrcReg | ModRM, em_test),
3329
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3330
	/* 0x88 - 0x8F */
3331
	I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3332
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3333
	I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3334 3335 3336
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
3337
	/* 0x90 - 0x97 */
3338
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3339
	/* 0x98 - 0x9F */
3340
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3341
	I(SrcImmFAddr | No64, em_call_far), N,
3342 3343
	II(ImplicitOps | Stack, em_pushf, pushf),
	II(ImplicitOps | Stack, em_popf, popf), N, N,
3344
	/* 0xA0 - 0xA7 */
3345
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3346
	I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3347
	I2bv(SrcSI | DstDI | Mov | String, em_mov),
3348
	I2bv(SrcSI | DstDI | String, em_cmp),
3349
	/* 0xA8 - 0xAF */
3350
	I2bv(DstAcc | SrcImm, em_test),
3351 3352
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3353
	I2bv(SrcAcc | DstDI | String, em_cmp),
3354
	/* 0xB0 - 0xB7 */
3355
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3356
	/* 0xB8 - 0xBF */
3357
	X8(I(DstReg | SrcImm | Mov, em_mov)),
3358
	/* 0xC0 - 0xC7 */
3359
	D2bv(DstMem | SrcImmByte | ModRM),
3360
	I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3361
	I(ImplicitOps | Stack, em_ret),
3362 3363
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3364
	G(ByteOp, group11), G(0, group11),
3365
	/* 0xC8 - 0xCF */
3366
	N, N, N, I(ImplicitOps | Stack, em_ret_far),
3367
	D(ImplicitOps), DI(SrcImmByte, intn),
3368
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3369
	/* 0xD0 - 0xD7 */
3370
	D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3371 3372 3373 3374
	N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
3375 3376
	X3(I(SrcImmByte, em_loop)),
	I(SrcImmByte, em_jcxz),
3377 3378
	I2bvIP(SrcImmUByte | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3379
	/* 0xE8 - 0xEF */
3380
	I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3381
	I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3382 3383
	I2bvIP(SrcDX | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3384
	/* 0xF0 - 0xF7 */
3385
	N, DI(ImplicitOps, icebp), N, N,
3386 3387
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
3388
	/* 0xF8 - 0xFF */
3389 3390
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3391 3392 3393 3394 3395
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

static struct opcode twobyte_table[256] = {
	/* 0x00 - 0x0F */
3396
	G(0, group6), GD(0, &group7), N, N,
3397 3398
	N, I(ImplicitOps | VendorSpecific, em_syscall),
	II(ImplicitOps | Priv, em_clts, clts), N,
3399
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3400 3401 3402 3403
	N, D(ImplicitOps | ModRM), N, N,
	/* 0x10 - 0x1F */
	N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
	/* 0x20 - 0x2F */
3404
	DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3405
	DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3406 3407
	IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
	IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3408 3409 3410
	N, N, N, N,
	N, N, N, N, N, N, N, N,
	/* 0x30 - 0x3F */
3411
	II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3412
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3413
	II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3414
	DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3415 3416
	I(ImplicitOps | VendorSpecific, em_sysenter),
	I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3417
	N, N,
3418 3419 3420 3421 3422 3423
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
	X16(D(DstReg | SrcMem | ModRM | Mov)),
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
3424 3425 3426 3427
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3428
	/* 0x70 - 0x7F */
3429 3430 3431 3432
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3433 3434 3435
	/* 0x80 - 0x8F */
	X16(D(SrcImm)),
	/* 0x90 - 0x9F */
3436
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3437
	/* 0xA0 - 0xA7 */
3438
	I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3439
	DI(ImplicitOps, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3440 3441 3442
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM), N, N,
	/* 0xA8 - 0xAF */
3443
	I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3444
	DI(ImplicitOps, rsm),
3445
	I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3446 3447
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM),
3448
	D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3449
	/* 0xB0 - 0xB7 */
3450
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3451
	I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3452
	I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3453 3454
	I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3455
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3456 3457
	/* 0xB8 - 0xBF */
	N, N,
3458 3459
	G(BitOp, group8),
	I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3460
	I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3461
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3462
	/* 0xC0 - 0xCF */
3463
	D2bv(DstMem | SrcReg | ModRM | Lock),
3464
	N, D(DstMem | SrcReg | ModRM | Mov),
3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479
	N, N, N, GD(0, &group9),
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

#undef D
#undef N
#undef G
#undef GD
#undef I
3480
#undef GP
3481
#undef EXT
3482

3483
#undef D2bv
3484
#undef D2bvIP
3485
#undef I2bv
3486
#undef I2bvIP
3487
#undef I6ALU
3488

3489
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3490 3491 3492
{
	unsigned size;

3493
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
3506
	op->addr.mem.ea = ctxt->_eip;
3507 3508 3509
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
3510
		op->val = insn_fetch(s8, ctxt);
3511 3512
		break;
	case 2:
3513
		op->val = insn_fetch(s16, ctxt);
3514 3515
		break;
	case 4:
3516
		op->val = insn_fetch(s32, ctxt);
3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535
		break;
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

3536 3537 3538 3539 3540 3541 3542 3543
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
			  unsigned d)
{
	int rc = X86EMUL_CONTINUE;

	switch (d) {
	case OpReg:
		decode_register_operand(ctxt, op,
3544
			 op == &ctxt->dst &&
3545 3546 3547
			 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
		break;
	case OpImmUByte:
3548
		rc = decode_imm(ctxt, op, 1, false);
3549 3550
		break;
	case OpMem:
3551
		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3552 3553 3554 3555
	mem_common:
		*op = ctxt->memop;
		ctxt->memopp = op;
		if ((ctxt->d & BitOp) && op == &ctxt->dst)
3556 3557 3558
			fetch_bit_operand(ctxt);
		op->orig_val = op->val;
		break;
3559 3560 3561
	case OpMem64:
		ctxt->memop.bytes = 8;
		goto mem_common;
3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582
	case OpAcc:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
	case OpDI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
		op->addr.mem.seg = VCPU_SREG_ES;
		op->val = 0;
		break;
	case OpDX:
		op->type = OP_REG;
		op->bytes = 2;
		op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
		fetch_register_operand(op);
		break;
3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596
	case OpCL:
		op->bytes = 1;
		op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
		break;
	case OpImmByte:
		rc = decode_imm(ctxt, op, 1, true);
		break;
	case OpOne:
		op->bytes = 1;
		op->val = 1;
		break;
	case OpImm:
		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
		break;
3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625
	case OpMem16:
		ctxt->memop.bytes = 2;
		goto mem_common;
	case OpMem32:
		ctxt->memop.bytes = 4;
		goto mem_common;
	case OpImmU16:
		rc = decode_imm(ctxt, op, 2, false);
		break;
	case OpImmU:
		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
		break;
	case OpSI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
		op->addr.mem.seg = seg_override(ctxt);
		op->val = 0;
		break;
	case OpImmFAddr:
		op->type = OP_IMM;
		op->addr.mem.ea = ctxt->_eip;
		op->bytes = ctxt->op_bytes + 2;
		insn_fetch_arr(op->valptr, op->bytes, ctxt);
		break;
	case OpMemFAddr:
		ctxt->memop.bytes = ctxt->op_bytes + 2;
		goto mem_common;
3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643
	case OpES:
		op->val = VCPU_SREG_ES;
		break;
	case OpCS:
		op->val = VCPU_SREG_CS;
		break;
	case OpSS:
		op->val = VCPU_SREG_SS;
		break;
	case OpDS:
		op->val = VCPU_SREG_DS;
		break;
	case OpFS:
		op->val = VCPU_SREG_FS;
		break;
	case OpGS:
		op->val = VCPU_SREG_GS;
		break;
3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654
	case OpImplicit:
		/* Special instructions do their own operand decoding. */
	default:
		op->type = OP_NONE; /* Disable writeback. */
		break;
	}

done:
	return rc;
}

3655
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3656 3657 3658
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
3659
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3660
	bool op_prefix = false;
3661
	struct opcode opcode;
3662

3663 3664
	ctxt->memop.type = OP_NONE;
	ctxt->memopp = NULL;
3665 3666 3667
	ctxt->_eip = ctxt->eip;
	ctxt->fetch.start = ctxt->_eip;
	ctxt->fetch.end = ctxt->fetch.start + insn_len;
3668
	if (insn_len > 0)
3669
		memcpy(ctxt->fetch.data, insn, insn_len);
3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
3687
		return EMULATION_FAILED;
3688 3689
	}

3690 3691
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
3692 3693 3694

	/* Legacy prefixes. */
	for (;;) {
3695
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
3696
		case 0x66:	/* operand-size override */
3697
			op_prefix = true;
3698
			/* switch between 2/4 bytes */
3699
			ctxt->op_bytes = def_op_bytes ^ 6;
3700 3701 3702 3703
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
3704
				ctxt->ad_bytes = def_ad_bytes ^ 12;
3705 3706
			else
				/* switch between 2/4 bytes */
3707
				ctxt->ad_bytes = def_ad_bytes ^ 6;
3708 3709 3710 3711 3712
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
3713
			set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3714 3715 3716
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
3717
			set_seg_override(ctxt, ctxt->b & 7);
3718 3719 3720 3721
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
3722
			ctxt->rex_prefix = ctxt->b;
3723 3724
			continue;
		case 0xf0:	/* LOCK */
3725
			ctxt->lock_prefix = 1;
3726 3727 3728
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
3729
			ctxt->rep_prefix = ctxt->b;
3730 3731 3732 3733 3734 3735 3736
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

3737
		ctxt->rex_prefix = 0;
3738 3739 3740 3741 3742
	}

done_prefixes:

	/* REX prefix. */
3743 3744
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
3745 3746

	/* Opcode byte(s). */
3747
	opcode = opcode_table[ctxt->b];
3748
	/* Two-byte opcode? */
3749 3750
	if (ctxt->b == 0x0f) {
		ctxt->twobyte = 1;
3751
		ctxt->b = insn_fetch(u8, ctxt);
3752
		opcode = twobyte_table[ctxt->b];
3753
	}
3754
	ctxt->d = opcode.flags;
3755

3756 3757
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
3758
		case Group:
3759
			ctxt->modrm = insn_fetch(u8, ctxt);
3760 3761
			--ctxt->_eip;
			goffset = (ctxt->modrm >> 3) & 7;
3762 3763 3764
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
3765
			ctxt->modrm = insn_fetch(u8, ctxt);
3766 3767 3768
			--ctxt->_eip;
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
3769 3770 3771 3772 3773
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
3774
			goffset = ctxt->modrm & 7;
3775
			opcode = opcode.u.group[goffset];
3776 3777
			break;
		case Prefix:
3778
			if (ctxt->rep_prefix && op_prefix)
3779
				return EMULATION_FAILED;
3780
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3781 3782 3783 3784 3785 3786 3787 3788
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
		default:
3789
			return EMULATION_FAILED;
3790
		}
3791

3792
		ctxt->d &= ~(u64)GroupMask;
3793
		ctxt->d |= opcode.flags;
3794 3795
	}

3796 3797 3798
	ctxt->execute = opcode.u.execute;
	ctxt->check_perm = opcode.check_perm;
	ctxt->intercept = opcode.intercept;
3799 3800

	/* Unrecognised? */
3801
	if (ctxt->d == 0 || (ctxt->d & Undefined))
3802
		return EMULATION_FAILED;
3803

3804
	if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3805
		return EMULATION_FAILED;
3806

3807 3808
	if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
		ctxt->op_bytes = 8;
3809

3810
	if (ctxt->d & Op3264) {
3811
		if (mode == X86EMUL_MODE_PROT64)
3812
			ctxt->op_bytes = 8;
3813
		else
3814
			ctxt->op_bytes = 4;
3815 3816
	}

3817 3818
	if (ctxt->d & Sse)
		ctxt->op_bytes = 16;
A
Avi Kivity 已提交
3819

3820
	/* ModRM and SIB bytes. */
3821
	if (ctxt->d & ModRM) {
3822
		rc = decode_modrm(ctxt, &ctxt->memop);
3823 3824 3825
		if (!ctxt->has_seg_override)
			set_seg_override(ctxt, ctxt->modrm_seg);
	} else if (ctxt->d & MemAbs)
3826
		rc = decode_abs(ctxt, &ctxt->memop);
3827 3828 3829
	if (rc != X86EMUL_CONTINUE)
		goto done;

3830 3831
	if (!ctxt->has_seg_override)
		set_seg_override(ctxt, VCPU_SREG_DS);
3832

3833
	ctxt->memop.addr.mem.seg = seg_override(ctxt);
3834

3835 3836
	if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3837 3838 3839 3840 3841

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
3842
	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3843 3844 3845
	if (rc != X86EMUL_CONTINUE)
		goto done;

3846 3847 3848 3849
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
3850
	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3851 3852 3853
	if (rc != X86EMUL_CONTINUE)
		goto done;

3854
	/* Decode and fetch the destination operand: register or memory. */
3855
	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3856 3857

done:
3858 3859
	if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
		ctxt->memopp->addr.mem.ea += ctxt->_eip;
3860

3861
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3862 3863
}

3864 3865 3866 3867 3868
bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
{
	return ctxt->d & PageTable;
}

3869 3870 3871 3872 3873 3874 3875 3876 3877
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
3878 3879 3880
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
3881
		 ((ctxt->eflags & EFLG_ZF) == 0))
3882
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
3883 3884 3885 3886 3887 3888
		    ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
		return true;

	return false;
}

3889
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3890
{
3891
	struct x86_emulate_ops *ops = ctxt->ops;
3892
	int rc = X86EMUL_CONTINUE;
3893
	int saved_dst_type = ctxt->dst.type;
3894

3895
	ctxt->mem_read.pos = 0;
3896

3897
	if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3898
		rc = emulate_ud(ctxt);
3899 3900 3901
		goto done;
	}

3902
	/* LOCK prefix is allowed only with some instructions */
3903
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3904
		rc = emulate_ud(ctxt);
3905 3906 3907
		goto done;
	}

3908
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3909
		rc = emulate_ud(ctxt);
3910 3911 3912
		goto done;
	}

3913
	if ((ctxt->d & Sse)
3914 3915
	    && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
		|| !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
A
Avi Kivity 已提交
3916 3917 3918 3919
		rc = emulate_ud(ctxt);
		goto done;
	}

3920
	if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
A
Avi Kivity 已提交
3921 3922 3923 3924
		rc = emulate_nm(ctxt);
		goto done;
	}

3925 3926
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3927
					      X86_ICPT_PRE_EXCEPT);
3928 3929 3930 3931
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3932
	/* Privileged instruction can be executed only in CPL=0 */
3933
	if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3934
		rc = emulate_gp(ctxt, 0);
3935 3936 3937
		goto done;
	}

3938
	/* Instruction can only be executed in protected mode */
3939
	if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3940 3941 3942 3943
		rc = emulate_ud(ctxt);
		goto done;
	}

3944
	/* Do instruction specific permission checks */
3945 3946
	if (ctxt->check_perm) {
		rc = ctxt->check_perm(ctxt);
3947 3948 3949 3950
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3951 3952
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3953
					      X86_ICPT_POST_EXCEPT);
3954 3955 3956 3957
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3958
	if (ctxt->rep_prefix && (ctxt->d & String)) {
3959
		/* All REP prefixes have the same first termination condition */
3960 3961
		if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
			ctxt->eip = ctxt->_eip;
3962 3963 3964 3965
			goto done;
		}
	}

3966 3967 3968
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
3969
		if (rc != X86EMUL_CONTINUE)
3970
			goto done;
3971
		ctxt->src.orig_val64 = ctxt->src.val64;
3972 3973
	}

3974 3975 3976
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
3977 3978 3979 3980
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3981
	if ((ctxt->d & DstMask) == ImplicitOps)
3982 3983 3984
		goto special_insn;


3985
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3986
		/* optimisation - avoid slow emulated read if Mov */
3987 3988
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
3989 3990
		if (rc != X86EMUL_CONTINUE)
			goto done;
3991
	}
3992
	ctxt->dst.orig_val = ctxt->dst.val;
3993

3994 3995
special_insn:

3996 3997
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3998
					      X86_ICPT_POST_MEMACCESS);
3999 4000 4001 4002
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4003 4004
	if (ctxt->execute) {
		rc = ctxt->execute(ctxt);
4005 4006 4007 4008 4009
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

4010
	if (ctxt->twobyte)
A
Avi Kivity 已提交
4011 4012
		goto twobyte_insn;

4013
	switch (ctxt->b) {
4014
	case 0x40 ... 0x47: /* inc r16/r32 */
4015
		emulate_1op(ctxt, "inc");
4016 4017
		break;
	case 0x48 ... 0x4f: /* dec r16/r32 */
4018
		emulate_1op(ctxt, "dec");
4019
		break;
A
Avi Kivity 已提交
4020
	case 0x63:		/* movsxd */
4021
		if (ctxt->mode != X86EMUL_MODE_PROT64)
A
Avi Kivity 已提交
4022
			goto cannot_emulate;
4023
		ctxt->dst.val = (s32) ctxt->src.val;
A
Avi Kivity 已提交
4024
		break;
4025
	case 0x70 ... 0x7f: /* jcc (short) */
4026 4027
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
4028
		break;
N
Nitin A Kamble 已提交
4029
	case 0x8d: /* lea r16/r32, m */
4030
		ctxt->dst.val = ctxt->src.addr.mem.ea;
N
Nitin A Kamble 已提交
4031
		break;
4032
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
4033
		if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
4034
			break;
4035 4036
		rc = em_xchg(ctxt);
		break;
4037
	case 0x98: /* cbw/cwde/cdqe */
4038 4039 4040 4041
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4042 4043
		}
		break;
4044
	case 0xc0 ... 0xc1:
4045
		rc = em_grp2(ctxt);
4046
		break;
4047
	case 0xcc:		/* int3 */
4048 4049
		rc = emulate_int(ctxt, 3);
		break;
4050
	case 0xcd:		/* int n */
4051
		rc = emulate_int(ctxt, ctxt->src.val);
4052 4053
		break;
	case 0xce:		/* into */
4054 4055
		if (ctxt->eflags & EFLG_OF)
			rc = emulate_int(ctxt, 4);
4056
		break;
4057
	case 0xd0 ... 0xd1:	/* Grp2 */
4058
		rc = em_grp2(ctxt);
4059 4060
		break;
	case 0xd2 ... 0xd3:	/* Grp2 */
4061
		ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4062
		rc = em_grp2(ctxt);
4063
		break;
4064
	case 0xe9: /* jmp rel */
4065
	case 0xeb: /* jmp rel short */
4066 4067
		jmp_rel(ctxt, ctxt->src.val);
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
4068
		break;
4069
	case 0xf4:              /* hlt */
4070
		ctxt->ops->halt(ctxt);
4071
		break;
4072 4073 4074 4075 4076 4077 4078
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
		ctxt->eflags ^= EFLG_CF;
		break;
	case 0xf8: /* clc */
		ctxt->eflags &= ~EFLG_CF;
		break;
4079 4080 4081
	case 0xf9: /* stc */
		ctxt->eflags |= EFLG_CF;
		break;
4082 4083 4084 4085 4086 4087
	case 0xfc: /* cld */
		ctxt->eflags &= ~EFLG_DF;
		break;
	case 0xfd: /* std */
		ctxt->eflags |= EFLG_DF;
		break;
4088 4089
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4090
	}
4091

4092 4093 4094
	if (rc != X86EMUL_CONTINUE)
		goto done;

4095
writeback:
4096
	rc = writeback(ctxt);
4097
	if (rc != X86EMUL_CONTINUE)
4098 4099
		goto done;

4100 4101 4102 4103
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
4104
	ctxt->dst.type = saved_dst_type;
4105

4106 4107 4108
	if ((ctxt->d & SrcMask) == SrcSI)
		string_addr_inc(ctxt, seg_override(ctxt),
				VCPU_REGS_RSI, &ctxt->src);
4109

4110
	if ((ctxt->d & DstMask) == DstDI)
4111
		string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4112
				&ctxt->dst);
4113

4114 4115 4116
	if (ctxt->rep_prefix && (ctxt->d & String)) {
		struct read_cache *r = &ctxt->io_read;
		register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4117

4118 4119 4120 4121 4122
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
4123
			if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4124 4125 4126 4127 4128 4129
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
4130
				ctxt->mem_read.end = 0;
4131 4132 4133
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
4134
		}
4135
	}
4136

4137
	ctxt->eip = ctxt->_eip;
4138 4139

done:
4140 4141
	if (rc == X86EMUL_PROPAGATE_FAULT)
		ctxt->have_exception = true;
4142 4143 4144
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

4145
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
4146 4147

twobyte_insn:
4148
	switch (ctxt->b) {
4149
	case 0x09:		/* wbinvd */
4150
		(ctxt->ops->wbinvd)(ctxt);
4151 4152
		break;
	case 0x08:		/* invd */
4153 4154 4155 4156
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
		break;
	case 0x20: /* mov cr, reg */
4157
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4158
		break;
A
Avi Kivity 已提交
4159
	case 0x21: /* mov from dr to reg */
4160
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
A
Avi Kivity 已提交
4161 4162
		break;
	case 0x40 ... 0x4f:	/* cmov */
4163 4164 4165
		ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
		if (!test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
4166
		break;
4167
	case 0x80 ... 0x8f: /* jnz rel, etc*/
4168 4169
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
4170
		break;
4171
	case 0x90 ... 0x9f:     /* setcc r/m8 */
4172
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4173
		break;
4174 4175
	case 0xa4: /* shld imm8, r, r/m */
	case 0xa5: /* shld cl, r, r/m */
4176
		emulate_2op_cl(ctxt, "shld");
4177 4178 4179
		break;
	case 0xac: /* shrd imm8, r, r/m */
	case 0xad: /* shrd cl, r, r/m */
4180
		emulate_2op_cl(ctxt, "shrd");
4181
		break;
4182 4183
	case 0xae:              /* clflush */
		break;
A
Avi Kivity 已提交
4184
	case 0xb6 ... 0xb7:	/* movzx */
4185 4186 4187
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
						       : (u16) ctxt->src.val;
A
Avi Kivity 已提交
4188 4189
		break;
	case 0xbe ... 0xbf:	/* movsx */
4190 4191 4192
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
							(s16) ctxt->src.val;
A
Avi Kivity 已提交
4193
		break;
4194
	case 0xc0 ... 0xc1:	/* xadd */
4195
		emulate_2op_SrcV(ctxt, "add");
4196
		/* Write back the register source. */
4197 4198
		ctxt->src.val = ctxt->dst.orig_val;
		write_register_operand(&ctxt->src);
4199
		break;
4200
	case 0xc3:		/* movnti */
4201 4202 4203
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
							(u64) ctxt->src.val;
4204
		break;
4205 4206
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4207
	}
4208 4209 4210 4211

	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
4212 4213 4214
	goto writeback;

cannot_emulate:
4215
	return EMULATION_FAILED;
A
Avi Kivity 已提交
4216
}