emulate.c 114.5 KB
Newer Older
A
Avi Kivity 已提交
1
/******************************************************************************
2
 * emulate.c
A
Avi Kivity 已提交
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
A
Avi Kivity 已提交
10 11
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
A
Avi Kivity 已提交
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
A
Avi Kivity 已提交
27

28
#include "x86.h"
29
#include "tss.h"
30

31 32 33
/*
 * Operand types
 */
34 35 36 37 38 39 40 41 42
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
43 44 45 46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
#define OpImm             12ull  /* Sign extended immediate */
47 48 49 50 51 52 53
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
54 55 56 57 58 59
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
60
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
61 62

#define OpBits             5  /* Width of operand field */
63
#define OpMask             ((1ull << OpBits) - 1)
64

A
Avi Kivity 已提交
65 66 67 68 69 70 71 72 73 74
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
75
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
76
/* Destination operand type. */
77 78 79 80 81 82 83 84 85 86
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
#define DstMask     (OpMask << DstShift)
A
Avi Kivity 已提交
87
/* Source operand type. */
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
#define SrcDX       (OpDX << SrcShift)
105
#define SrcMem8     (OpMem8 << SrcShift)
106
#define SrcMask     (OpMask << SrcShift)
107 108 109 110 111 112 113 114 115 116
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
#define Sse         (1<<18)     /* SSE Vector instruction */
117 118 119 120
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
121
/* Misc flags */
122
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
123
#define VendorSpecific (1<<22) /* Vendor specific instruction */
124
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
125
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
126
#define Undefined   (1<<25) /* No Such Instruction */
127
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
128
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
129
#define No64	    (1<<28)
130
#define PageTable   (1 << 29)   /* instruction used to write page table */
131
/* Source 2 operand type */
132
#define Src2Shift   (30)
133 134 135 136 137
#define Src2None    (OpNone << Src2Shift)
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
138 139 140 141 142 143
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
144
#define Src2Mask    (OpMask << Src2Shift)
A
Avi Kivity 已提交
145
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
146 147 148
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
A
Avi Kivity 已提交
149

150 151 152 153 154 155 156 157
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
158

159
struct opcode {
160 161
	u64 flags : 56;
	u64 intercept : 8;
162
	union {
163
		int (*execute)(struct x86_emulate_ctxt *ctxt);
164 165
		struct opcode *group;
		struct group_dual *gdual;
166
		struct gprefix *gprefix;
167
	} u;
168
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
169 170 171 172 173
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
174 175
};

176 177 178 179 180 181 182
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

A
Avi Kivity 已提交
183
/* EFLAGS bit definitions. */
184 185 186 187
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
188 189
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
190 191
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
A
Avi Kivity 已提交
192 193
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
194
#define EFLG_IF (1<<9)
195
#define EFLG_TF (1<<8)
A
Avi Kivity 已提交
196 197 198 199 200 201
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

202 203 204
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

A
Avi Kivity 已提交
205 206 207 208 209 210 211
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

212
#if defined(CONFIG_X86_64)
A
Avi Kivity 已提交
213 214 215 216 217 218 219 220 221 222 223 224 225 226
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
A
Avi Kivity 已提交
242 243 244 245 246 247 248 249 250

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

251 252 253 254 255 256
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

257
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
258 259 260 261 262
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
263 264
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
265
			  "=&r" (_tmp)					\
266
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
267
	} while (0)
268 269


A
Avi Kivity 已提交
270
/* Raw emulation: instruction has two explicit operands. */
271
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
272 273 274
	do {								\
		unsigned long _tmp;					\
									\
275
		switch ((ctxt)->dst.bytes) {				\
276
		case 2:							\
277
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
278 279
			break;						\
		case 4:							\
280
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
281 282
			break;						\
		case 8:							\
283
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
284 285
			break;						\
		}							\
A
Avi Kivity 已提交
286 287
	} while (0)

288
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
A
Avi Kivity 已提交
289
	do {								     \
290
		unsigned long _tmp;					     \
291
		switch ((ctxt)->dst.bytes) {				     \
A
Avi Kivity 已提交
292
		case 1:							     \
293
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
A
Avi Kivity 已提交
294 295
			break;						     \
		default:						     \
296
			__emulate_2op_nobyte(ctxt, _op,			     \
A
Avi Kivity 已提交
297 298 299 300 301 302
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
303 304
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
A
Avi Kivity 已提交
305 306

/* Source operand is byte, word, long or quad sized. */
307 308
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
309 310

/* Source operand is word, long or quad sized. */
311 312
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
313

314
/* Instruction has three operands and one operand is stored in ECX register */
315
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
316 317
	do {								\
		unsigned long _tmp;					\
318 319 320
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
321 322 323 324 325
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
326
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
327 328 329
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
330 331 332
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
333 334
	} while (0)

335
#define emulate_2op_cl(ctxt, _op)					\
336
	do {								\
337
		switch ((ctxt)->dst.bytes) {				\
338
		case 2:							\
339
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
340 341
			break;						\
		case 4:							\
342
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
343 344
			break;						\
		case 8:							\
345
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
346 347
			break;						\
		}							\
348 349
	} while (0)

350
#define __emulate_1op(ctxt, _op, _suffix)				\
A
Avi Kivity 已提交
351 352 353
	do {								\
		unsigned long _tmp;					\
									\
354 355 356 357
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
358
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
359 360 361 362 363
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
364
#define emulate_1op(ctxt, _op)						\
365
	do {								\
366 367 368 369 370
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
A
Avi Kivity 已提交
371 372 373
		}							\
	} while (0)

374
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
375 376
	do {								\
		unsigned long _tmp;					\
377 378
		ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX];		\
		ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX];		\
379 380 381 382 383 384 385 386 387 388 389 390
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
391 392 393 394
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val),	\
			  "a" (*rax), "d" (*rdx));			\
395 396
	} while (0)

397
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
398
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
399
	do {								\
400
		switch((ctxt)->src.bytes) {				\
401
		case 1:							\
402
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
403 404
			break;						\
		case 2:							\
405
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
406 407
			break;						\
		case 4:							\
408
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
409 410
			break;						\
		case 8: ON64(						\
411
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
412 413 414 415
			break;						\
		}							\
	} while (0)

416 417 418 419 420 421
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
422 423 424 425 426 427 428 429
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
430 431 432
		.next_rip   = ctxt->eip,
	};

433
	return ctxt->ops->intercept(ctxt, &info, stage);
434 435
}

436
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
437
{
438
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
439 440
}

A
Avi Kivity 已提交
441
/* Access/update address held in a register, based on addressing mode. */
442
static inline unsigned long
443
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
444
{
445
	if (ctxt->ad_bytes == sizeof(unsigned long))
446 447
		return reg;
	else
448
		return reg & ad_mask(ctxt);
449 450 451
}

static inline unsigned long
452
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
453
{
454
	return address_mask(ctxt, reg);
455 456
}

457
static inline void
458
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
459
{
460
	if (ctxt->ad_bytes == sizeof(unsigned long))
461 462
		*reg += inc;
	else
463
		*reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
464
}
A
Avi Kivity 已提交
465

466
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
467
{
468
	register_address_increment(ctxt, &ctxt->_eip, rel);
469
}
470

471 472 473 474 475 476 477
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

478
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
479
{
480 481
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
482 483
}

484
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
485 486 487 488
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

489
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
490 491
}

492
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
493
{
494
	if (!ctxt->has_seg_override)
495 496
		return 0;

497
	return ctxt->seg_override;
498 499
}

500 501
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
502
{
503 504 505
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
506
	return X86EMUL_PROPAGATE_FAULT;
507 508
}

509 510 511 512 513
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

514
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
515
{
516
	return emulate_exception(ctxt, GP_VECTOR, err, true);
517 518
}

519 520 521 522 523
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

524
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
525
{
526
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
527 528
}

529
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
530
{
531
	return emulate_exception(ctxt, TS_VECTOR, err, true);
532 533
}

534 535
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
536
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
537 538
}

A
Avi Kivity 已提交
539 540 541 542 543
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.
 */
static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
{
	if (likely(size < 16))
		return false;

	if (ctxt->d & Aligned)
		return true;
	else if (ctxt->d & Unaligned)
		return false;
	else if (ctxt->d & Avx)
		return false;
	else
		return true;
}

587
static int __linearize(struct x86_emulate_ctxt *ctxt,
588
		     struct segmented_address addr,
589
		     unsigned size, bool write, bool fetch,
590 591
		     ulong *linear)
{
592 593
	struct desc_struct desc;
	bool usable;
594
	ulong la;
595
	u32 lim;
596
	u16 sel;
597
	unsigned cpl, rpl;
598

599
	la = seg_base(ctxt, addr.seg) + addr.ea;
600 601 602 603 604 605 606 607
	switch (ctxt->mode) {
	case X86EMUL_MODE_REAL:
		break;
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
608 609
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
610 611 612 613 614 615
		if (!usable)
			goto bad;
		/* code segment or read-only data segment */
		if (((desc.type & 8) || !(desc.type & 2)) && write)
			goto bad;
		/* unreadable code segment */
616
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
617 618 619 620 621 622 623 624 625 626 627 628 629 630
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
			/* exapand-down segment */
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
631
		cpl = ctxt->ops->cpl(ctxt);
632
		rpl = sel & 3;
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
		cpl = max(cpl, rpl);
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
649
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
650
		la &= (u32)-1;
651 652
	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
		return emulate_gp(ctxt, 0);
653 654
	*linear = la;
	return X86EMUL_CONTINUE;
655 656 657 658 659
bad:
	if (addr.seg == VCPU_SREG_SS)
		return emulate_ss(ctxt, addr.seg);
	else
		return emulate_gp(ctxt, addr.seg);
660 661
}

662 663 664 665 666 667 668 669 670
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


671 672 673 674 675
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
676 677 678
	int rc;
	ulong linear;

679
	rc = linearize(ctxt, addr, size, false, &linear);
680 681
	if (rc != X86EMUL_CONTINUE)
		return rc;
682
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
683 684
}

685 686 687 688 689 690 691 692
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
693
{
694
	struct fetch_cache *fc = &ctxt->fetch;
695
	int rc;
696
	int size, cur_size;
697

698
	if (ctxt->_eip == fc->end) {
699
		unsigned long linear;
700 701
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
702
		cur_size = fc->end - fc->start;
703 704
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
705
		rc = __linearize(ctxt, addr, size, false, true, &linear);
706
		if (unlikely(rc != X86EMUL_CONTINUE))
707
			return rc;
708 709
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
710
		if (unlikely(rc != X86EMUL_CONTINUE))
711
			return rc;
712
		fc->end += size;
713
	}
714 715
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
716
	return X86EMUL_CONTINUE;
717 718 719
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
720
			 void *dest, unsigned size)
721
{
722
	int rc;
723

724
	/* x86 instructions are limited to 15 bytes. */
725
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
726
		return X86EMUL_UNHANDLEABLE;
727
	while (size--) {
728
		rc = do_insn_fetch_byte(ctxt, dest++);
729
		if (rc != X86EMUL_CONTINUE)
730 731
			return rc;
	}
732
	return X86EMUL_CONTINUE;
733 734
}

735
/* Fetch next part of the instruction being emulated. */
736
#define insn_fetch(_type, _ctxt)					\
737
({	unsigned long _x;						\
738
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
739 740 741 742 743
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

744 745
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
746 747 748 749
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

750 751 752 753 754 755 756
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
static void *decode_register(u8 modrm_reg, unsigned long *regs,
			     int highbyte_regs)
A
Avi Kivity 已提交
757 758 759 760 761 762 763 764 765 766
{
	void *p;

	p = &regs[modrm_reg];
	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
		p = (unsigned char *)&regs[modrm_reg & 3] + 1;
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
767
			   struct segmented_address addr,
A
Avi Kivity 已提交
768 769 770 771 772 773 774
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
775
	rc = segmented_read_std(ctxt, addr, size, 2);
776
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
777
		return rc;
778
	addr.ea += 2;
779
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
780 781 782
	return rc;
}

783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
static int test_cc(unsigned int condition, unsigned int flags)
{
	int rc = 0;

	switch ((condition & 15) >> 1) {
	case 0: /* o */
		rc |= (flags & EFLG_OF);
		break;
	case 1: /* b/c/nae */
		rc |= (flags & EFLG_CF);
		break;
	case 2: /* z/e */
		rc |= (flags & EFLG_ZF);
		break;
	case 3: /* be/na */
		rc |= (flags & (EFLG_CF|EFLG_ZF));
		break;
	case 4: /* s */
		rc |= (flags & EFLG_SF);
		break;
	case 5: /* p/pe */
		rc |= (flags & EFLG_PF);
		break;
	case 7: /* le/ng */
		rc |= (flags & EFLG_ZF);
		/* fall through */
	case 6: /* l/nge */
		rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
		break;
	}

	/* Odd condition identifiers (lsb == 1) have inverted sense. */
	return (!!rc ^ (condition & 1));
}

818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

A
Avi Kivity 已提交
836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

A
Avi Kivity 已提交
891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924
static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
	case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
	case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
	case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
	case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
	case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
	case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
	case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
	case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
	case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
	case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

A
Avi Kivity 已提交
925
static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
926
				    struct operand *op)
927
{
928 929
	unsigned reg = ctxt->modrm_reg;
	int highbyte_regs = ctxt->rex_prefix == 0;
930

931 932
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
933

934
	if (ctxt->d & Sse) {
A
Avi Kivity 已提交
935 936 937 938 939 940
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}
A
Avi Kivity 已提交
941 942 943 944 945 946 947
	if (ctxt->d & Mmx) {
		reg &= 7;
		op->type = OP_MM;
		op->bytes = 8;
		op->addr.mm = reg;
		return;
	}
A
Avi Kivity 已提交
948

949
	op->type = OP_REG;
950
	if (ctxt->d & ByteOp) {
951
		op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
952 953
		op->bytes = 1;
	} else {
954 955
		op->addr.reg = decode_register(reg, ctxt->regs, 0);
		op->bytes = ctxt->op_bytes;
956
	}
957
	fetch_register_operand(op);
958 959 960
	op->orig_val = op->val;
}

961
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
962
			struct operand *op)
963 964
{
	u8 sib;
965
	int index_reg = 0, base_reg = 0, scale;
966
	int rc = X86EMUL_CONTINUE;
967
	ulong modrm_ea = 0;
968

969 970 971 972
	if (ctxt->rex_prefix) {
		ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1;	/* REX.R */
		index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
		ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
973 974
	}

975 976 977 978
	ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
	ctxt->modrm_rm |= (ctxt->modrm & 0x07);
	ctxt->modrm_seg = VCPU_SREG_DS;
979

980
	if (ctxt->modrm_mod == 3) {
981
		op->type = OP_REG;
982 983 984 985
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.reg = decode_register(ctxt->modrm_rm,
					       ctxt->regs, ctxt->d & ByteOp);
		if (ctxt->d & Sse) {
A
Avi Kivity 已提交
986 987
			op->type = OP_XMM;
			op->bytes = 16;
988 989
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
A
Avi Kivity 已提交
990 991
			return rc;
		}
A
Avi Kivity 已提交
992 993 994 995 996 997
		if (ctxt->d & Mmx) {
			op->type = OP_MM;
			op->bytes = 8;
			op->addr.xmm = ctxt->modrm_rm & 7;
			return rc;
		}
998
		fetch_register_operand(op);
999 1000 1001
		return rc;
	}

1002 1003
	op->type = OP_MEM;

1004 1005 1006 1007 1008
	if (ctxt->ad_bytes == 2) {
		unsigned bx = ctxt->regs[VCPU_REGS_RBX];
		unsigned bp = ctxt->regs[VCPU_REGS_RBP];
		unsigned si = ctxt->regs[VCPU_REGS_RSI];
		unsigned di = ctxt->regs[VCPU_REGS_RDI];
1009 1010

		/* 16-bit ModR/M decode. */
1011
		switch (ctxt->modrm_mod) {
1012
		case 0:
1013
			if (ctxt->modrm_rm == 6)
1014
				modrm_ea += insn_fetch(u16, ctxt);
1015 1016
			break;
		case 1:
1017
			modrm_ea += insn_fetch(s8, ctxt);
1018 1019
			break;
		case 2:
1020
			modrm_ea += insn_fetch(u16, ctxt);
1021 1022
			break;
		}
1023
		switch (ctxt->modrm_rm) {
1024
		case 0:
1025
			modrm_ea += bx + si;
1026 1027
			break;
		case 1:
1028
			modrm_ea += bx + di;
1029 1030
			break;
		case 2:
1031
			modrm_ea += bp + si;
1032 1033
			break;
		case 3:
1034
			modrm_ea += bp + di;
1035 1036
			break;
		case 4:
1037
			modrm_ea += si;
1038 1039
			break;
		case 5:
1040
			modrm_ea += di;
1041 1042
			break;
		case 6:
1043
			if (ctxt->modrm_mod != 0)
1044
				modrm_ea += bp;
1045 1046
			break;
		case 7:
1047
			modrm_ea += bx;
1048 1049
			break;
		}
1050 1051 1052
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
1053
		modrm_ea = (u16)modrm_ea;
1054 1055
	} else {
		/* 32/64-bit ModR/M decode. */
1056
		if ((ctxt->modrm_rm & 7) == 4) {
1057
			sib = insn_fetch(u8, ctxt);
1058 1059 1060 1061
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

1062
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1063
				modrm_ea += insn_fetch(s32, ctxt);
1064
			else
1065
				modrm_ea += ctxt->regs[base_reg];
1066
			if (index_reg != 4)
1067 1068
				modrm_ea += ctxt->regs[index_reg] << scale;
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1069
			if (ctxt->mode == X86EMUL_MODE_PROT64)
1070
				ctxt->rip_relative = 1;
1071
		} else
1072 1073
			modrm_ea += ctxt->regs[ctxt->modrm_rm];
		switch (ctxt->modrm_mod) {
1074
		case 0:
1075
			if (ctxt->modrm_rm == 5)
1076
				modrm_ea += insn_fetch(s32, ctxt);
1077 1078
			break;
		case 1:
1079
			modrm_ea += insn_fetch(s8, ctxt);
1080 1081
			break;
		case 2:
1082
			modrm_ea += insn_fetch(s32, ctxt);
1083 1084 1085
			break;
		}
	}
1086
	op->addr.mem.ea = modrm_ea;
1087 1088 1089 1090 1091
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
1092
		      struct operand *op)
1093
{
1094
	int rc = X86EMUL_CONTINUE;
1095

1096
	op->type = OP_MEM;
1097
	switch (ctxt->ad_bytes) {
1098
	case 2:
1099
		op->addr.mem.ea = insn_fetch(u16, ctxt);
1100 1101
		break;
	case 4:
1102
		op->addr.mem.ea = insn_fetch(u32, ctxt);
1103 1104
		break;
	case 8:
1105
		op->addr.mem.ea = insn_fetch(u64, ctxt);
1106 1107 1108 1109 1110 1111
		break;
	}
done:
	return rc;
}

1112
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1113
{
1114
	long sv = 0, mask;
1115

1116 1117
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
		mask = ~(ctxt->dst.bytes * 8 - 1);
1118

1119 1120 1121 1122
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1123

1124
		ctxt->dst.addr.mem.ea += (sv >> 3);
1125
	}
1126 1127

	/* only subword offset */
1128
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1129 1130
}

1131 1132
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1133
{
1134
	int rc;
1135
	struct read_cache *mc = &ctxt->mem_read;
A
Avi Kivity 已提交
1136

1137 1138 1139 1140 1141
	while (size) {
		int n = min(size, 8u);
		size -= n;
		if (mc->pos < mc->end)
			goto read_cached;
1142

1143 1144
		rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
					      &ctxt->exception);
1145 1146 1147
		if (rc != X86EMUL_CONTINUE)
			return rc;
		mc->end += n;
A
Avi Kivity 已提交
1148

1149 1150 1151 1152 1153
	read_cached:
		memcpy(dest, mc->data + mc->pos, n);
		mc->pos += n;
		dest += n;
		addr += n;
A
Avi Kivity 已提交
1154
	}
1155 1156
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1157

1158 1159 1160 1161 1162
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1163 1164 1165
	int rc;
	ulong linear;

1166
	rc = linearize(ctxt, addr, size, false, &linear);
1167 1168
	if (rc != X86EMUL_CONTINUE)
		return rc;
1169
	return read_emulated(ctxt, linear, data, size);
1170 1171 1172 1173 1174 1175 1176
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1177 1178 1179
	int rc;
	ulong linear;

1180
	rc = linearize(ctxt, addr, size, true, &linear);
1181 1182
	if (rc != X86EMUL_CONTINUE)
		return rc;
1183 1184
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1185 1186 1187 1188 1189 1190 1191
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1192 1193 1194
	int rc;
	ulong linear;

1195
	rc = linearize(ctxt, addr, size, true, &linear);
1196 1197
	if (rc != X86EMUL_CONTINUE)
		return rc;
1198 1199
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1200 1201
}

1202 1203 1204 1205
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1206
	struct read_cache *rc = &ctxt->io_read;
1207

1208 1209
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1210 1211
		unsigned int count = ctxt->rep_prefix ?
			address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1212
		in_page = (ctxt->eflags & EFLG_DF) ?
1213 1214
			offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
			PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1215 1216 1217 1218 1219
		n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
			count);
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1220
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1221 1222
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1223 1224
	}

1225 1226 1227 1228
	memcpy(dest, rc->data + rc->pos, size);
	rc->pos += size;
	return 1;
}
A
Avi Kivity 已提交
1229

1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
				     u16 index, struct desc_struct *desc)
{
	struct desc_ptr dt;
	ulong addr;

	ctxt->ops->get_idt(ctxt, &dt);

	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, index << 3 | 0x2);

	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
}

1246 1247 1248
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1249 1250
	struct x86_emulate_ops *ops = ctxt->ops;

1251 1252
	if (selector & 1 << 2) {
		struct desc_struct desc;
1253 1254
		u16 sel;

1255
		memset (dt, 0, sizeof *dt);
1256
		if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1257
			return;
1258

1259 1260 1261
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
		dt->address = get_desc_base(&desc);
	} else
1262
		ops->get_gdt(ctxt, dt);
1263
}
1264

1265 1266 1267 1268 1269 1270 1271
/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1272

1273
	get_descriptor_table_ptr(ctxt, selector, &dt);
1274

1275 1276
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1277

1278 1279 1280
	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
1281
}
1282

1283 1284 1285 1286 1287 1288 1289
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
A
Avi Kivity 已提交
1290

1291
	get_descriptor_table_ptr(ctxt, selector, &dt);
1292

1293 1294
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
A
Avi Kivity 已提交
1295

1296
	addr = dt.address + index * 8;
1297 1298
	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
				    &ctxt->exception);
1299
}
1300

1301
/* Does not support long mode */
1302 1303 1304 1305 1306 1307 1308 1309 1310
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
	struct desc_struct seg_desc;
	u8 dpl, rpl, cpl;
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
	int ret;
1311

1312
	memset(&seg_desc, 0, sizeof seg_desc);
1313

1314 1315 1316 1317 1318 1319 1320 1321
	if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
	    || ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
1322 1323
		if (ctxt->mode == X86EMUL_MODE_VM86)
			seg_desc.dpl = 3;
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338
		goto load;
	}

	/* NULL selector is not valid for TR, CS and SS */
	if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
	    && null_selector)
		goto exception;

	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

	if (null_selector) /* for NULL selector skip all following checks */
		goto load;

1339
	ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
	err_vec = GP_VECTOR;

	/* can't load system descriptor into segment selecor */
	if (seg <= VCPU_SREG_GS && !seg_desc.s)
		goto exception;

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	rpl = selector & 3;
	dpl = seg_desc.dpl;
1357
	cpl = ctxt->ops->cpl(ctxt);
1358 1359 1360 1361 1362 1363 1364 1365 1366

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1367
		break;
1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1383
		break;
1384 1385 1386 1387 1388 1389 1390 1391 1392
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1393
		/*
1394 1395 1396
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1397
		 */
1398 1399 1400 1401
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1402
		break;
1403 1404 1405 1406 1407
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
		seg_desc.type |= 1;
1408
		ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1409 1410 1411 1412
		if (ret != X86EMUL_CONTINUE)
			return ret;
	}
load:
1413
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1414 1415 1416 1417 1418 1419
	return X86EMUL_CONTINUE;
exception:
	emulate_exception(ctxt, err_vec, err_code, true);
	return X86EMUL_PROPAGATE_FAULT;
}

1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
static void write_register_operand(struct operand *op)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (op->bytes) {
	case 1:
		*(u8 *)op->addr.reg = (u8)op->val;
		break;
	case 2:
		*(u16 *)op->addr.reg = (u16)op->val;
		break;
	case 4:
		*op->addr.reg = (u32)op->val;
		break;	/* 64b: zero-extend */
	case 8:
		*op->addr.reg = op->val;
		break;
	}
}

1439
static int writeback(struct x86_emulate_ctxt *ctxt)
1440 1441 1442
{
	int rc;

1443
	switch (ctxt->dst.type) {
1444
	case OP_REG:
1445
		write_register_operand(&ctxt->dst);
A
Avi Kivity 已提交
1446
		break;
1447
	case OP_MEM:
1448
		if (ctxt->lock_prefix)
1449
			rc = segmented_cmpxchg(ctxt,
1450 1451 1452 1453
					       ctxt->dst.addr.mem,
					       &ctxt->dst.orig_val,
					       &ctxt->dst.val,
					       ctxt->dst.bytes);
1454
		else
1455
			rc = segmented_write(ctxt,
1456 1457 1458
					     ctxt->dst.addr.mem,
					     &ctxt->dst.val,
					     ctxt->dst.bytes);
1459 1460
		if (rc != X86EMUL_CONTINUE)
			return rc;
1461
		break;
A
Avi Kivity 已提交
1462
	case OP_XMM:
1463
		write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
A
Avi Kivity 已提交
1464
		break;
A
Avi Kivity 已提交
1465 1466 1467
	case OP_MM:
		write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
		break;
1468 1469
	case OP_NONE:
		/* no writeback */
1470
		break;
1471
	default:
1472
		break;
A
Avi Kivity 已提交
1473
	}
1474 1475
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1476

1477
static int em_push(struct x86_emulate_ctxt *ctxt)
1478
{
1479
	struct segmented_address addr;
1480

1481 1482
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1483 1484 1485
	addr.seg = VCPU_SREG_SS;

	/* Disable writeback. */
1486 1487
	ctxt->dst.type = OP_NONE;
	return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1488
}
1489

1490 1491 1492 1493
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1494
	struct segmented_address addr;
1495

1496
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1497
	addr.seg = VCPU_SREG_SS;
1498
	rc = segmented_read(ctxt, addr, dest, len);
1499 1500 1501
	if (rc != X86EMUL_CONTINUE)
		return rc;

1502
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1503
	return rc;
1504 1505
}

1506 1507
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1508
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1509 1510
}

1511
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1512
			void *dest, int len)
1513 1514
{
	int rc;
1515 1516
	unsigned long val, change_mask;
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1517
	int cpl = ctxt->ops->cpl(ctxt);
1518

1519
	rc = emulate_pop(ctxt, &val, len);
1520 1521
	if (rc != X86EMUL_CONTINUE)
		return rc;
1522

1523 1524
	change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
		| EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1525

1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
			change_mask |= EFLG_IOPL;
		if (cpl <= iopl)
			change_mask |= EFLG_IF;
		break;
	case X86EMUL_MODE_VM86:
1536 1537
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1538 1539 1540 1541 1542
		change_mask |= EFLG_IF;
		break;
	default: /* real mode */
		change_mask |= (EFLG_IOPL | EFLG_IF);
		break;
1543
	}
1544 1545 1546 1547 1548

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1549 1550
}

1551 1552
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1553 1554 1555 1556
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1557 1558
}

1559
static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1560
{
1561 1562
	int seg = ctxt->src2.val;

1563
	ctxt->src.val = get_segment_selector(ctxt, seg);
1564

1565
	return em_push(ctxt);
1566 1567
}

1568
static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1569
{
1570
	int seg = ctxt->src2.val;
1571 1572
	unsigned long selector;
	int rc;
1573

1574
	rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1575 1576 1577
	if (rc != X86EMUL_CONTINUE)
		return rc;

1578
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1579
	return rc;
1580 1581
}

1582
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1583
{
1584
	unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1585 1586
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1587

1588 1589
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
1590
		(ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1591

1592
		rc = em_push(ctxt);
1593 1594
		if (rc != X86EMUL_CONTINUE)
			return rc;
1595

1596
		++reg;
1597 1598
	}

1599
	return rc;
1600 1601
}

1602 1603
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
1604
	ctxt->src.val =  (unsigned long)ctxt->eflags;
1605 1606 1607
	return em_push(ctxt);
}

1608
static int em_popa(struct x86_emulate_ctxt *ctxt)
1609
{
1610 1611
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
1612

1613 1614
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
1615 1616
			register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
							ctxt->op_bytes);
1617 1618
			--reg;
		}
1619

1620
		rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1621 1622 1623
		if (rc != X86EMUL_CONTINUE)
			break;
		--reg;
1624
	}
1625
	return rc;
1626 1627
}

1628
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1629
{
1630
	struct x86_emulate_ops *ops = ctxt->ops;
1631
	int rc;
1632 1633 1634 1635 1636 1637
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
1638
	ctxt->src.val = ctxt->eflags;
1639
	rc = em_push(ctxt);
1640 1641
	if (rc != X86EMUL_CONTINUE)
		return rc;
1642 1643 1644

	ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);

1645
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1646
	rc = em_push(ctxt);
1647 1648
	if (rc != X86EMUL_CONTINUE)
		return rc;
1649

1650
	ctxt->src.val = ctxt->_eip;
1651
	rc = em_push(ctxt);
1652 1653 1654
	if (rc != X86EMUL_CONTINUE)
		return rc;

1655
	ops->get_idt(ctxt, &dt);
1656 1657 1658 1659

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

1660
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1661 1662 1663
	if (rc != X86EMUL_CONTINUE)
		return rc;

1664
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1665 1666 1667
	if (rc != X86EMUL_CONTINUE)
		return rc;

1668
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1669 1670 1671
	if (rc != X86EMUL_CONTINUE)
		return rc;

1672
	ctxt->_eip = eip;
1673 1674 1675 1676

	return rc;
}

1677
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1678 1679 1680
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1681
		return emulate_int_real(ctxt, irq);
1682 1683 1684 1685 1686 1687 1688 1689 1690 1691
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

1692
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1693
{
1694 1695 1696 1697 1698 1699 1700 1701
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
	unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
			     EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
			     EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
	unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1702

1703
	/* TODO: Add stack limit check */
1704

1705
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1706

1707 1708
	if (rc != X86EMUL_CONTINUE)
		return rc;
1709

1710 1711
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
1712

1713
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1714

1715 1716
	if (rc != X86EMUL_CONTINUE)
		return rc;
1717

1718
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1719

1720 1721
	if (rc != X86EMUL_CONTINUE)
		return rc;
1722

1723
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1724

1725 1726
	if (rc != X86EMUL_CONTINUE)
		return rc;
1727

1728
	ctxt->_eip = temp_eip;
1729 1730


1731
	if (ctxt->op_bytes == 4)
1732
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1733
	else if (ctxt->op_bytes == 2) {
1734 1735
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
1736
	}
1737 1738 1739 1740 1741

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
	ctxt->eflags |= EFLG_RESERVED_ONE_MASK;

	return rc;
1742 1743
}

1744
static int em_iret(struct x86_emulate_ctxt *ctxt)
1745
{
1746 1747
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1748
		return emulate_iret_real(ctxt);
1749 1750 1751 1752
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
1753
	default:
1754 1755
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
1756 1757 1758
	}
}

1759 1760 1761 1762 1763
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned short sel;

1764
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1765

1766
	rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1767 1768 1769
	if (rc != X86EMUL_CONTINUE)
		return rc;

1770 1771
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1772 1773 1774
	return X86EMUL_CONTINUE;
}

1775
static int em_grp2(struct x86_emulate_ctxt *ctxt)
1776
{
1777
	switch (ctxt->modrm_reg) {
1778
	case 0:	/* rol */
1779
		emulate_2op_SrcB(ctxt, "rol");
1780 1781
		break;
	case 1:	/* ror */
1782
		emulate_2op_SrcB(ctxt, "ror");
1783 1784
		break;
	case 2:	/* rcl */
1785
		emulate_2op_SrcB(ctxt, "rcl");
1786 1787
		break;
	case 3:	/* rcr */
1788
		emulate_2op_SrcB(ctxt, "rcr");
1789 1790 1791
		break;
	case 4:	/* sal/shl */
	case 6:	/* sal/shl */
1792
		emulate_2op_SrcB(ctxt, "sal");
1793 1794
		break;
	case 5:	/* shr */
1795
		emulate_2op_SrcB(ctxt, "shr");
1796 1797
		break;
	case 7:	/* sar */
1798
		emulate_2op_SrcB(ctxt, "sar");
1799 1800
		break;
	}
1801
	return X86EMUL_CONTINUE;
1802 1803
}

1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
static int em_not(struct x86_emulate_ctxt *ctxt)
{
	ctxt->dst.val = ~ctxt->dst.val;
	return X86EMUL_CONTINUE;
}

static int em_neg(struct x86_emulate_ctxt *ctxt)
{
	emulate_1op(ctxt, "neg");
	return X86EMUL_CONTINUE;
}

static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "mul", ex);
	return X86EMUL_CONTINUE;
}

static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "imul", ex);
	return X86EMUL_CONTINUE;
}

static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1833
{
1834
	u8 de = 0;
1835

1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846
	emulate_1op_rax_rdx(ctxt, "div", de);
	if (de)
		return emulate_de(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 de = 0;

	emulate_1op_rax_rdx(ctxt, "idiv", de);
1847 1848
	if (de)
		return emulate_de(ctxt);
1849
	return X86EMUL_CONTINUE;
1850 1851
}

1852
static int em_grp45(struct x86_emulate_ctxt *ctxt)
1853
{
1854
	int rc = X86EMUL_CONTINUE;
1855

1856
	switch (ctxt->modrm_reg) {
1857
	case 0:	/* inc */
1858
		emulate_1op(ctxt, "inc");
1859 1860
		break;
	case 1:	/* dec */
1861
		emulate_1op(ctxt, "dec");
1862
		break;
1863 1864
	case 2: /* call near abs */ {
		long int old_eip;
1865 1866 1867
		old_eip = ctxt->_eip;
		ctxt->_eip = ctxt->src.val;
		ctxt->src.val = old_eip;
1868
		rc = em_push(ctxt);
1869 1870
		break;
	}
1871
	case 4: /* jmp abs */
1872
		ctxt->_eip = ctxt->src.val;
1873
		break;
1874 1875 1876
	case 5: /* jmp far */
		rc = em_jmp_far(ctxt);
		break;
1877
	case 6:	/* push */
1878
		rc = em_push(ctxt);
1879 1880
		break;
	}
1881
	return rc;
1882 1883
}

1884
static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1885
{
1886
	u64 old = ctxt->dst.orig_val64;
1887

1888 1889 1890 1891
	if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
	    ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
		ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
		ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1892
		ctxt->eflags &= ~EFLG_ZF;
1893
	} else {
1894 1895
		ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
			(u32) ctxt->regs[VCPU_REGS_RBX];
1896

1897
		ctxt->eflags |= EFLG_ZF;
1898
	}
1899
	return X86EMUL_CONTINUE;
1900 1901
}

1902 1903
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
1904 1905 1906
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
1907 1908 1909
	return em_pop(ctxt);
}

1910
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1911 1912 1913 1914
{
	int rc;
	unsigned long cs;

1915
	rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1916
	if (rc != X86EMUL_CONTINUE)
1917
		return rc;
1918 1919 1920
	if (ctxt->op_bytes == 4)
		ctxt->_eip = (u32)ctxt->_eip;
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1921
	if (rc != X86EMUL_CONTINUE)
1922
		return rc;
1923
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1924 1925 1926
	return rc;
}

1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944
static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
{
	/* Save real source value, then compare EAX against destination. */
	ctxt->src.orig_val = ctxt->src.val;
	ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
	emulate_2op_SrcV(ctxt, "cmp");

	if (ctxt->eflags & EFLG_ZF) {
		/* Success: write back to memory. */
		ctxt->dst.val = ctxt->src.orig_val;
	} else {
		/* Failure: write the value we saw to EAX. */
		ctxt->dst.type = OP_REG;
		ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
	}
	return X86EMUL_CONTINUE;
}

1945
static int em_lseg(struct x86_emulate_ctxt *ctxt)
1946
{
1947
	int seg = ctxt->src2.val;
1948 1949 1950
	unsigned short sel;
	int rc;

1951
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1952

1953
	rc = load_segment_descriptor(ctxt, sel, seg);
1954 1955 1956
	if (rc != X86EMUL_CONTINUE)
		return rc;

1957
	ctxt->dst.val = ctxt->src.val;
1958 1959 1960
	return rc;
}

1961
static void
1962
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1963
			struct desc_struct *cs, struct desc_struct *ss)
1964
{
1965 1966
	u16 selector;

1967
	memset(cs, 0, sizeof(struct desc_struct));
1968
	ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1969
	memset(ss, 0, sizeof(struct desc_struct));
1970 1971

	cs->l = 0;		/* will be adjusted later */
1972
	set_desc_base(cs, 0);	/* flat segment */
1973
	cs->g = 1;		/* 4kb granularity */
1974
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
1975 1976 1977
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
1978 1979
	cs->p = 1;
	cs->d = 1;
1980

1981 1982
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
1983 1984 1985
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
1986
	ss->d = 1;		/* 32bit stack segment */
1987
	ss->dpl = 0;
1988
	ss->p = 1;
1989 1990
}

1991 1992 1993 1994 1995
static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

	eax = ecx = 0;
1996 1997
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
1998 1999 2000 2001
		&& ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
		&& edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
}

2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015
static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
{
	struct x86_emulate_ops *ops = ctxt->ops;
	u32 eax, ebx, ecx, edx;

	/*
	 * syscall should always be enabled in longmode - so only become
	 * vendor specific (cpuid) if other modes are active...
	 */
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return true;

	eax = 0x00000000;
	ecx = 0x00000000;
2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040
	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	/*
	 * Intel ("GenuineIntel")
	 * remark: Intel CPUs only support "syscall" in 64bit
	 * longmode. Also an 64bit guest with a
	 * 32bit compat-app running will #UD !! While this
	 * behaviour can be fixed (by emulating) into AMD
	 * response - CPUs of AMD can't behave like Intel.
	 */
	if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
		return false;

	/* AMD ("AuthenticAMD") */
	if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
		return true;

	/* AMD ("AMDisbetter!") */
	if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
		return true;
2041 2042 2043 2044 2045

	/* default: (not Intel, not AMD), apply Intel's stricter rules... */
	return false;
}

2046
static int em_syscall(struct x86_emulate_ctxt *ctxt)
2047
{
2048
	struct x86_emulate_ops *ops = ctxt->ops;
2049
	struct desc_struct cs, ss;
2050
	u64 msr_data;
2051
	u16 cs_sel, ss_sel;
2052
	u64 efer = 0;
2053 2054

	/* syscall is not available in real mode */
2055
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2056 2057
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
2058

2059 2060 2061
	if (!(em_syscall_is_enabled(ctxt)))
		return emulate_ud(ctxt);

2062
	ops->get_msr(ctxt, MSR_EFER, &efer);
2063
	setup_syscalls_segments(ctxt, &cs, &ss);
2064

2065 2066 2067
	if (!(efer & EFER_SCE))
		return emulate_ud(ctxt);

2068
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
2069
	msr_data >>= 32;
2070 2071
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
2072

2073
	if (efer & EFER_LMA) {
2074
		cs.d = 0;
2075 2076
		cs.l = 1;
	}
2077 2078
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2079

2080
	ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
2081
	if (efer & EFER_LMA) {
2082
#ifdef CONFIG_X86_64
2083
		ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2084

2085
		ops->get_msr(ctxt,
2086 2087
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
2088
		ctxt->_eip = msr_data;
2089

2090
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2091 2092 2093 2094
		ctxt->eflags &= ~(msr_data | EFLG_RF);
#endif
	} else {
		/* legacy mode */
2095
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
2096
		ctxt->_eip = (u32)msr_data;
2097 2098 2099 2100

		ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
	}

2101
	return X86EMUL_CONTINUE;
2102 2103
}

2104
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2105
{
2106
	struct x86_emulate_ops *ops = ctxt->ops;
2107
	struct desc_struct cs, ss;
2108
	u64 msr_data;
2109
	u16 cs_sel, ss_sel;
2110
	u64 efer = 0;
2111

2112
	ops->get_msr(ctxt, MSR_EFER, &efer);
2113
	/* inject #GP if in real mode */
2114 2115
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
2116

2117 2118 2119 2120 2121 2122 2123 2124
	/*
	 * Not recognized on AMD in compat mode (but is recognized in legacy
	 * mode).
	 */
	if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
	    && !vendor_intel(ctxt))
		return emulate_ud(ctxt);

2125 2126 2127
	/* XXX sysenter/sysexit have not been tested in 64bit mode.
	* Therefore, we inject an #UD.
	*/
2128 2129
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return emulate_ud(ctxt);
2130

2131
	setup_syscalls_segments(ctxt, &cs, &ss);
2132

2133
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2134 2135
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT32:
2136 2137
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2138 2139
		break;
	case X86EMUL_MODE_PROT64:
2140 2141
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2142 2143 2144 2145
		break;
	}

	ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2146 2147 2148 2149
	cs_sel = (u16)msr_data;
	cs_sel &= ~SELECTOR_RPL_MASK;
	ss_sel = cs_sel + 8;
	ss_sel &= ~SELECTOR_RPL_MASK;
2150
	if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2151
		cs.d = 0;
2152 2153 2154
		cs.l = 1;
	}

2155 2156
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2157

2158
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2159
	ctxt->_eip = msr_data;
2160

2161
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2162
	ctxt->regs[VCPU_REGS_RSP] = msr_data;
2163

2164
	return X86EMUL_CONTINUE;
2165 2166
}

2167
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2168
{
2169
	struct x86_emulate_ops *ops = ctxt->ops;
2170
	struct desc_struct cs, ss;
2171 2172
	u64 msr_data;
	int usermode;
X
Xiao Guangrong 已提交
2173
	u16 cs_sel = 0, ss_sel = 0;
2174

2175 2176
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2177 2178
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
2179

2180
	setup_syscalls_segments(ctxt, &cs, &ss);
2181

2182
	if ((ctxt->rex_prefix & 0x8) != 0x0)
2183 2184 2185 2186 2187 2188
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

	cs.dpl = 3;
	ss.dpl = 3;
2189
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2190 2191
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
2192
		cs_sel = (u16)(msr_data + 16);
2193 2194
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2195
		ss_sel = (u16)(msr_data + 24);
2196 2197
		break;
	case X86EMUL_MODE_PROT64:
2198
		cs_sel = (u16)(msr_data + 32);
2199 2200
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2201 2202
		ss_sel = cs_sel + 8;
		cs.d = 0;
2203 2204 2205
		cs.l = 1;
		break;
	}
2206 2207
	cs_sel |= SELECTOR_RPL_MASK;
	ss_sel |= SELECTOR_RPL_MASK;
2208

2209 2210
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2211

2212 2213
	ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
	ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2214

2215
	return X86EMUL_CONTINUE;
2216 2217
}

2218
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2219 2220 2221 2222 2223 2224 2225
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2226
	return ctxt->ops->cpl(ctxt) > iopl;
2227 2228 2229 2230 2231
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2232
	struct x86_emulate_ops *ops = ctxt->ops;
2233
	struct desc_struct tr_seg;
2234
	u32 base3;
2235
	int r;
2236
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2237
	unsigned mask = (1 << len) - 1;
2238
	unsigned long base;
2239

2240
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2241
	if (!tr_seg.p)
2242
		return false;
2243
	if (desc_limit_scaled(&tr_seg) < 103)
2244
		return false;
2245 2246 2247 2248
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2249
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2250 2251
	if (r != X86EMUL_CONTINUE)
		return false;
2252
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2253
		return false;
2254
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2255 2256 2257 2258 2259 2260 2261 2262 2263 2264
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2265 2266 2267
	if (ctxt->perm_ok)
		return true;

2268 2269
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2270
			return false;
2271 2272 2273

	ctxt->perm_ok = true;

2274 2275 2276
	return true;
}

2277 2278 2279
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
2280
	tss->ip = ctxt->_eip;
2281
	tss->flag = ctxt->eflags;
2282 2283 2284 2285 2286 2287 2288 2289
	tss->ax = ctxt->regs[VCPU_REGS_RAX];
	tss->cx = ctxt->regs[VCPU_REGS_RCX];
	tss->dx = ctxt->regs[VCPU_REGS_RDX];
	tss->bx = ctxt->regs[VCPU_REGS_RBX];
	tss->sp = ctxt->regs[VCPU_REGS_RSP];
	tss->bp = ctxt->regs[VCPU_REGS_RBP];
	tss->si = ctxt->regs[VCPU_REGS_RSI];
	tss->di = ctxt->regs[VCPU_REGS_RDI];
2290

2291 2292 2293 2294 2295
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2296 2297 2298 2299 2300 2301 2302
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;

2303
	ctxt->_eip = tss->ip;
2304
	ctxt->eflags = tss->flag | 2;
2305 2306 2307 2308 2309 2310 2311 2312
	ctxt->regs[VCPU_REGS_RAX] = tss->ax;
	ctxt->regs[VCPU_REGS_RCX] = tss->cx;
	ctxt->regs[VCPU_REGS_RDX] = tss->dx;
	ctxt->regs[VCPU_REGS_RBX] = tss->bx;
	ctxt->regs[VCPU_REGS_RSP] = tss->sp;
	ctxt->regs[VCPU_REGS_RBP] = tss->bp;
	ctxt->regs[VCPU_REGS_RSI] = tss->si;
	ctxt->regs[VCPU_REGS_RDI] = tss->di;
2313 2314 2315 2316 2317

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2318 2319 2320 2321 2322
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2323 2324 2325 2326 2327

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2328
	ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2329 2330
	if (ret != X86EMUL_CONTINUE)
		return ret;
2331
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2332 2333
	if (ret != X86EMUL_CONTINUE)
		return ret;
2334
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2335 2336
	if (ret != X86EMUL_CONTINUE)
		return ret;
2337
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2338 2339
	if (ret != X86EMUL_CONTINUE)
		return ret;
2340
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2341 2342 2343 2344 2345 2346 2347 2348 2349 2350
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2351
	struct x86_emulate_ops *ops = ctxt->ops;
2352 2353
	struct tss_segment_16 tss_seg;
	int ret;
2354
	u32 new_tss_base = get_desc_base(new_desc);
2355

2356
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2357
			    &ctxt->exception);
2358
	if (ret != X86EMUL_CONTINUE)
2359 2360 2361
		/* FIXME: need to provide precise fault address */
		return ret;

2362
	save_state_to_tss16(ctxt, &tss_seg);
2363

2364
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2365
			     &ctxt->exception);
2366
	if (ret != X86EMUL_CONTINUE)
2367 2368 2369
		/* FIXME: need to provide precise fault address */
		return ret;

2370
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2371
			    &ctxt->exception);
2372
	if (ret != X86EMUL_CONTINUE)
2373 2374 2375 2376 2377 2378
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2379
		ret = ops->write_std(ctxt, new_tss_base,
2380 2381
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2382
				     &ctxt->exception);
2383
		if (ret != X86EMUL_CONTINUE)
2384 2385 2386 2387
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2388
	return load_state_from_tss16(ctxt, &tss_seg);
2389 2390 2391 2392 2393
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
2394
	tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2395
	tss->eip = ctxt->_eip;
2396
	tss->eflags = ctxt->eflags;
2397 2398 2399 2400 2401 2402 2403 2404
	tss->eax = ctxt->regs[VCPU_REGS_RAX];
	tss->ecx = ctxt->regs[VCPU_REGS_RCX];
	tss->edx = ctxt->regs[VCPU_REGS_RDX];
	tss->ebx = ctxt->regs[VCPU_REGS_RBX];
	tss->esp = ctxt->regs[VCPU_REGS_RSP];
	tss->ebp = ctxt->regs[VCPU_REGS_RBP];
	tss->esi = ctxt->regs[VCPU_REGS_RSI];
	tss->edi = ctxt->regs[VCPU_REGS_RDI];
2405

2406 2407 2408 2409 2410 2411 2412
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
	tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2413 2414 2415 2416 2417 2418 2419
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;

2420
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2421
		return emulate_gp(ctxt, 0);
2422
	ctxt->_eip = tss->eip;
2423
	ctxt->eflags = tss->eflags | 2;
2424 2425

	/* General purpose registers */
2426 2427 2428 2429 2430 2431 2432 2433
	ctxt->regs[VCPU_REGS_RAX] = tss->eax;
	ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
	ctxt->regs[VCPU_REGS_RDX] = tss->edx;
	ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
	ctxt->regs[VCPU_REGS_RSP] = tss->esp;
	ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
	ctxt->regs[VCPU_REGS_RSI] = tss->esi;
	ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2434 2435 2436 2437 2438

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2439 2440 2441 2442 2443 2444 2445
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2446

2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464
	/*
	 * If we're switching between Protected Mode and VM86, we need to make
	 * sure to update the mode before loading the segment descriptors so
	 * that the selectors are interpreted correctly.
	 *
	 * Need to get rflags to the vcpu struct immediately because it
	 * influences the CPL which is checked at least when loading the segment
	 * descriptors and when pushing an error code to the new kernel stack.
	 *
	 * TODO Introduce a separate ctxt->ops->set_cpl callback
	 */
	if (ctxt->eflags & X86_EFLAGS_VM)
		ctxt->mode = X86EMUL_MODE_VM86;
	else
		ctxt->mode = X86EMUL_MODE_PROT32;

	ctxt->ops->set_rflags(ctxt, ctxt->eflags);

2465 2466 2467 2468
	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2469
	ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2470 2471
	if (ret != X86EMUL_CONTINUE)
		return ret;
2472
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2473 2474
	if (ret != X86EMUL_CONTINUE)
		return ret;
2475
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2476 2477
	if (ret != X86EMUL_CONTINUE)
		return ret;
2478
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2479 2480
	if (ret != X86EMUL_CONTINUE)
		return ret;
2481
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2482 2483
	if (ret != X86EMUL_CONTINUE)
		return ret;
2484
	ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2485 2486
	if (ret != X86EMUL_CONTINUE)
		return ret;
2487
	ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2488 2489 2490 2491 2492 2493 2494 2495 2496 2497
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2498
	struct x86_emulate_ops *ops = ctxt->ops;
2499 2500
	struct tss_segment_32 tss_seg;
	int ret;
2501
	u32 new_tss_base = get_desc_base(new_desc);
2502

2503
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2504
			    &ctxt->exception);
2505
	if (ret != X86EMUL_CONTINUE)
2506 2507 2508
		/* FIXME: need to provide precise fault address */
		return ret;

2509
	save_state_to_tss32(ctxt, &tss_seg);
2510

2511
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2512
			     &ctxt->exception);
2513
	if (ret != X86EMUL_CONTINUE)
2514 2515 2516
		/* FIXME: need to provide precise fault address */
		return ret;

2517
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2518
			    &ctxt->exception);
2519
	if (ret != X86EMUL_CONTINUE)
2520 2521 2522 2523 2524 2525
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2526
		ret = ops->write_std(ctxt, new_tss_base,
2527 2528
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2529
				     &ctxt->exception);
2530
		if (ret != X86EMUL_CONTINUE)
2531 2532 2533 2534
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2535
	return load_state_from_tss32(ctxt, &tss_seg);
2536 2537 2538
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2539
				   u16 tss_selector, int idt_index, int reason,
2540
				   bool has_error_code, u32 error_code)
2541
{
2542
	struct x86_emulate_ops *ops = ctxt->ops;
2543 2544
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
2545
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2546
	ulong old_tss_base =
2547
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2548
	u32 desc_limit;
2549 2550 2551

	/* FIXME: old_tss_base == ~0 ? */

2552
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2553 2554
	if (ret != X86EMUL_CONTINUE)
		return ret;
2555
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2556 2557 2558 2559 2560
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586
	/*
	 * Check privileges. The three cases are task switch caused by...
	 *
	 * 1. jmp/call/int to task gate: Check against DPL of the task gate
	 * 2. Exception/IRQ/iret: No check is performed
	 * 3. jmp/call to TSS: Check agains DPL of the TSS
	 */
	if (reason == TASK_SWITCH_GATE) {
		if (idt_index != -1) {
			/* Software interrupts */
			struct desc_struct task_gate_desc;
			int dpl;

			ret = read_interrupt_descriptor(ctxt, idt_index,
							&task_gate_desc);
			if (ret != X86EMUL_CONTINUE)
				return ret;

			dpl = task_gate_desc.dpl;
			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
				return emulate_gp(ctxt, (idt_index << 3) | 0x2);
		}
	} else if (reason != TASK_SWITCH_IRET) {
		int dpl = next_tss_desc.dpl;
		if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
			return emulate_gp(ctxt, tss_selector);
2587 2588
	}

2589

2590 2591 2592 2593
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
2594
		emulate_ts(ctxt, tss_selector & 0xfffc);
2595 2596 2597 2598 2599
		return X86EMUL_PROPAGATE_FAULT;
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2600
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
	   note that old_tss_sel is not used afetr this point */
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
2612
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2613 2614
				     old_tss_base, &next_tss_desc);
	else
2615
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2616
				     old_tss_base, &next_tss_desc);
2617 2618
	if (ret != X86EMUL_CONTINUE)
		return ret;
2619 2620 2621 2622 2623 2624

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
2625
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2626 2627
	}

2628
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
2629
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2630

2631
	if (has_error_code) {
2632 2633 2634
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
2635
		ret = em_push(ctxt);
2636 2637
	}

2638 2639 2640 2641
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2642
			 u16 tss_selector, int idt_index, int reason,
2643
			 bool has_error_code, u32 error_code)
2644 2645 2646
{
	int rc;

2647 2648
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
2649

2650
	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2651
				     has_error_code, error_code);
2652

2653
	if (rc == X86EMUL_CONTINUE)
2654
		ctxt->eip = ctxt->_eip;
2655

2656
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2657 2658
}

2659
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2660
			    int reg, struct operand *op)
2661 2662 2663
{
	int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;

2664 2665
	register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2666
	op->addr.mem.seg = seg;
2667 2668
}

2669 2670 2671 2672 2673 2674
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
2675
	al = ctxt->dst.val;
2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

2693
	ctxt->dst.val = al;
2694
	/* Set PF, ZF, SF */
2695 2696 2697
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
2698
	emulate_2op_SrcV(ctxt, "or");
2699 2700 2701 2702 2703 2704 2705 2706
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

2707 2708 2709 2710 2711 2712 2713 2714 2715
static int em_call(struct x86_emulate_ctxt *ctxt)
{
	long rel = ctxt->src.val;

	ctxt->src.val = (unsigned long)ctxt->_eip;
	jmp_rel(ctxt, rel);
	return em_push(ctxt);
}

2716 2717 2718 2719 2720 2721
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;

2722
	old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2723
	old_eip = ctxt->_eip;
2724

2725
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2726
	if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2727 2728
		return X86EMUL_CONTINUE;

2729 2730
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2731

2732
	ctxt->src.val = old_cs;
2733
	rc = em_push(ctxt);
2734 2735 2736
	if (rc != X86EMUL_CONTINUE)
		return rc;

2737
	ctxt->src.val = old_eip;
2738
	return em_push(ctxt);
2739 2740
}

2741 2742 2743 2744
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2745 2746 2747 2748
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
	rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2749 2750
	if (rc != X86EMUL_CONTINUE)
		return rc;
2751
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2752 2753 2754
	return X86EMUL_CONTINUE;
}

2755 2756
static int em_add(struct x86_emulate_ctxt *ctxt)
{
2757
	emulate_2op_SrcV(ctxt, "add");
2758 2759 2760 2761 2762
	return X86EMUL_CONTINUE;
}

static int em_or(struct x86_emulate_ctxt *ctxt)
{
2763
	emulate_2op_SrcV(ctxt, "or");
2764 2765 2766 2767 2768
	return X86EMUL_CONTINUE;
}

static int em_adc(struct x86_emulate_ctxt *ctxt)
{
2769
	emulate_2op_SrcV(ctxt, "adc");
2770 2771 2772 2773 2774
	return X86EMUL_CONTINUE;
}

static int em_sbb(struct x86_emulate_ctxt *ctxt)
{
2775
	emulate_2op_SrcV(ctxt, "sbb");
2776 2777 2778 2779 2780
	return X86EMUL_CONTINUE;
}

static int em_and(struct x86_emulate_ctxt *ctxt)
{
2781
	emulate_2op_SrcV(ctxt, "and");
2782 2783 2784 2785 2786
	return X86EMUL_CONTINUE;
}

static int em_sub(struct x86_emulate_ctxt *ctxt)
{
2787
	emulate_2op_SrcV(ctxt, "sub");
2788 2789 2790 2791 2792
	return X86EMUL_CONTINUE;
}

static int em_xor(struct x86_emulate_ctxt *ctxt)
{
2793
	emulate_2op_SrcV(ctxt, "xor");
2794 2795 2796 2797 2798
	return X86EMUL_CONTINUE;
}

static int em_cmp(struct x86_emulate_ctxt *ctxt)
{
2799
	emulate_2op_SrcV(ctxt, "cmp");
2800
	/* Disable writeback. */
2801
	ctxt->dst.type = OP_NONE;
2802 2803 2804
	return X86EMUL_CONTINUE;
}

2805 2806
static int em_test(struct x86_emulate_ctxt *ctxt)
{
2807
	emulate_2op_SrcV(ctxt, "test");
2808 2809
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
2810 2811 2812
	return X86EMUL_CONTINUE;
}

2813 2814 2815
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
2816 2817
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
2818 2819

	/* Write back the memory destination with implicit LOCK prefix. */
2820 2821
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
2822 2823 2824
	return X86EMUL_CONTINUE;
}

2825
static int em_imul(struct x86_emulate_ctxt *ctxt)
2826
{
2827
	emulate_2op_SrcV_nobyte(ctxt, "imul");
2828 2829 2830
	return X86EMUL_CONTINUE;
}

2831 2832
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
2833
	ctxt->dst.val = ctxt->src2.val;
2834 2835 2836
	return em_imul(ctxt);
}

2837 2838
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
2839 2840 2841 2842
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
	ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2843 2844 2845 2846

	return X86EMUL_CONTINUE;
}

2847 2848 2849 2850
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

2851
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2852 2853
	ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
	ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2854 2855 2856
	return X86EMUL_CONTINUE;
}

2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867
static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
{
	u64 pmc;

	if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
		return emulate_gp(ctxt, 0);
	ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
	ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
	return X86EMUL_CONTINUE;
}

2868 2869
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
S
Stefan Hajnoczi 已提交
2870
	memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2871 2872 2873
	return X86EMUL_CONTINUE;
}

2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901
static int em_cr_write(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_dr_write(struct x86_emulate_ctxt *ctxt)
{
	unsigned long val;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		val = ctxt->src.val & ~0ULL;
	else
		val = ctxt->src.val & ~0U;

	/* #UD condition is already handled. */
	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925
static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

	msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
		| ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
	if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

	if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
		return emulate_gp(ctxt, 0);

	ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
	ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
	return X86EMUL_CONTINUE;
}

2926 2927
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
2928
	if (ctxt->modrm_reg > VCPU_SREG_GS)
2929 2930
		return emulate_ud(ctxt);

2931
	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2932 2933 2934 2935 2936
	return X86EMUL_CONTINUE;
}

static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
2937
	u16 sel = ctxt->src.val;
2938

2939
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2940 2941
		return emulate_ud(ctxt);

2942
	if (ctxt->modrm_reg == VCPU_SREG_SS)
2943 2944 2945
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
2946 2947
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2948 2949
}

2950 2951
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
2952 2953 2954
	int rc;
	ulong linear;

2955
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2956
	if (rc == X86EMUL_CONTINUE)
2957
		ctxt->ops->invlpg(ctxt, linear);
2958
	/* Disable writeback. */
2959
	ctxt->dst.type = OP_NONE;
2960 2961 2962
	return X86EMUL_CONTINUE;
}

2963 2964 2965 2966 2967 2968 2969 2970 2971 2972
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

2973 2974 2975 2976
static int em_vmcall(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2977
	if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2978 2979 2980 2981 2982 2983 2984
		return X86EMUL_UNHANDLEABLE;

	rc = ctxt->ops->fix_hypercall(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
2985
	ctxt->_eip = ctxt->eip;
2986
	/* Disable writeback. */
2987
	ctxt->dst.type = OP_NONE;
2988 2989 2990 2991 2992 2993 2994 2995
	return X86EMUL_CONTINUE;
}

static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

2996
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2997
			     &desc_ptr.size, &desc_ptr.address,
2998
			     ctxt->op_bytes);
2999 3000 3001 3002
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_gdt(ctxt, &desc_ptr);
	/* Disable writeback. */
3003
	ctxt->dst.type = OP_NONE;
3004 3005 3006
	return X86EMUL_CONTINUE;
}

3007
static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3008 3009 3010
{
	int rc;

3011 3012
	rc = ctxt->ops->fix_hypercall(ctxt);

3013
	/* Disable writeback. */
3014
	ctxt->dst.type = OP_NONE;
3015 3016 3017 3018 3019 3020 3021 3022
	return rc;
}

static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

3023
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3024
			     &desc_ptr.size, &desc_ptr.address,
3025
			     ctxt->op_bytes);
3026 3027 3028 3029
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_idt(ctxt, &desc_ptr);
	/* Disable writeback. */
3030
	ctxt->dst.type = OP_NONE;
3031 3032 3033 3034 3035
	return X86EMUL_CONTINUE;
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
3036 3037
	ctxt->dst.bytes = 2;
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3038 3039 3040 3041 3042 3043
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3044 3045
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
3046 3047 3048
	return X86EMUL_CONTINUE;
}

3049 3050
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
3051 3052 3053 3054
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
	if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
		jmp_rel(ctxt, ctxt->src.val);
3055 3056 3057 3058 3059 3060

	return X86EMUL_CONTINUE;
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
3061 3062
	if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
		jmp_rel(ctxt, ctxt->src.val);
3063 3064 3065 3066

	return X86EMUL_CONTINUE;
}

3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084
static int em_in(struct x86_emulate_ctxt *ctxt)
{
	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
			     &ctxt->dst.val))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
}

static int em_out(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				    &ctxt->src.val, 1);
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132
static int em_bt(struct x86_emulate_ctxt *ctxt)
{
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	/* only subword offset */
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;

	emulate_2op_SrcV_nobyte(ctxt, "bt");
	return X86EMUL_CONTINUE;
}

static int em_bts(struct x86_emulate_ctxt *ctxt)
{
	emulate_2op_SrcV_nobyte(ctxt, "bts");
	return X86EMUL_CONTINUE;
}

static int em_btr(struct x86_emulate_ctxt *ctxt)
{
	emulate_2op_SrcV_nobyte(ctxt, "btr");
	return X86EMUL_CONTINUE;
}

static int em_btc(struct x86_emulate_ctxt *ctxt)
{
	emulate_2op_SrcV_nobyte(ctxt, "btc");
	return X86EMUL_CONTINUE;
}

3133 3134
static int em_bsf(struct x86_emulate_ctxt *ctxt)
{
3135
	emulate_2op_SrcV_nobyte(ctxt, "bsf");
3136 3137 3138 3139 3140
	return X86EMUL_CONTINUE;
}

static int em_bsr(struct x86_emulate_ctxt *ctxt)
{
3141
	emulate_2op_SrcV_nobyte(ctxt, "bsr");
3142 3143 3144
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158
static int em_cpuid(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

	eax = ctxt->regs[VCPU_REGS_RAX];
	ecx = ctxt->regs[VCPU_REGS_RCX];
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	ctxt->regs[VCPU_REGS_RAX] = eax;
	ctxt->regs[VCPU_REGS_RBX] = ebx;
	ctxt->regs[VCPU_REGS_RCX] = ecx;
	ctxt->regs[VCPU_REGS_RDX] = edx;
	return X86EMUL_CONTINUE;
}

3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
3173
	if (!valid_cr(ctxt->modrm_reg))
3174 3175 3176 3177 3178 3179 3180
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
3181 3182
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
3183
	u64 efer = 0;
3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
3201
		u64 cr4;
3202 3203 3204 3205
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

3206 3207
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3208 3209 3210 3211 3212 3213 3214 3215 3216 3217

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

3218 3219
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
3220
			rsvd = CR3_L_MODE_RESERVED_BITS;
3221
		else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3222
			rsvd = CR3_PAE_RESERVED_BITS;
3223
		else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3224 3225 3226 3227 3228 3229 3230 3231
			rsvd = CR3_NONPAE_RESERVED_BITS;

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
3232
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

3244 3245 3246 3247
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

3248
	ctxt->ops->get_dr(ctxt, 7, &dr7);
3249 3250 3251 3252 3253 3254 3255

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
3256
	int dr = ctxt->modrm_reg;
3257 3258 3259 3260 3261
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

3262
	cr4 = ctxt->ops->get_cr(ctxt, 4);
3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

	if (check_dr7_gd(ctxt))
		return emulate_db(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
3274 3275
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
3276 3277 3278 3279 3280 3281 3282

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

3283 3284 3285 3286
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
	u64 efer;

3287
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3288 3289 3290 3291 3292 3293 3294 3295 3296

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
3297
	u64 rax = ctxt->regs[VCPU_REGS_RAX];
3298 3299

	/* Valid physical address? */
3300
	if (rax & 0xffff000000000000ULL)
3301 3302 3303 3304 3305
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

3306 3307
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
3308
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3309

3310
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3311 3312 3313 3314 3315
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

3316 3317
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
3318
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3319
	u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3320

3321
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3322 3323 3324 3325 3326 3327
	    (rcx > 3))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3328 3329
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
3330 3331
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3332 3333 3334 3335 3336 3337 3338
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
3339 3340
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3341 3342 3343 3344 3345
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3346
#define D(_y) { .flags = (_y) }
3347
#define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3348 3349
#define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
		      .check_perm = (_p) }
3350
#define N    D(0)
3351
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3352 3353
#define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
#define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3354
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3355 3356
#define II(_f, _e, _i) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3357 3358 3359
#define IIP(_f, _e, _i, _p) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
	  .check_perm = (_p) }
3360
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3361

3362
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
3363
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3364
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
3365 3366
#define I2bvIP(_f, _e, _i, _p) \
	IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3367

3368 3369 3370
#define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3371

3372
static struct opcode group7_rm1[] = {
3373 3374
	DI(SrcNone | Priv, monitor),
	DI(SrcNone | Priv, mwait),
3375 3376 3377
	N, N, N, N, N, N,
};

3378
static struct opcode group7_rm3[] = {
3379 3380 3381 3382 3383 3384 3385 3386
	DIP(SrcNone | Prot | Priv,		vmrun,		check_svme_pa),
	II(SrcNone  | Prot | VendorSpecific,	em_vmmcall,	vmmcall),
	DIP(SrcNone | Prot | Priv,		vmload,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		vmsave,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		stgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		clgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		skinit,		check_svme),
	DIP(SrcNone | Prot | Priv,		invlpga,	check_svme),
3387
};
3388

3389 3390
static struct opcode group7_rm7[] = {
	N,
3391
	DIP(SrcNone, rdtscp, check_rdtsc),
3392 3393
	N, N, N, N, N, N,
};
3394

3395
static struct opcode group1[] = {
3396
	I(Lock, em_add),
3397
	I(Lock | PageTable, em_or),
3398 3399
	I(Lock, em_adc),
	I(Lock, em_sbb),
3400
	I(Lock | PageTable, em_and),
3401 3402 3403
	I(Lock, em_sub),
	I(Lock, em_xor),
	I(0, em_cmp),
3404 3405 3406
};

static struct opcode group1A[] = {
3407
	I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3408 3409 3410
};

static struct opcode group3[] = {
3411 3412 3413 3414 3415 3416 3417 3418
	I(DstMem | SrcImm, em_test),
	I(DstMem | SrcImm, em_test),
	I(DstMem | SrcNone | Lock, em_not),
	I(DstMem | SrcNone | Lock, em_neg),
	I(SrcMem, em_mul_ex),
	I(SrcMem, em_imul_ex),
	I(SrcMem, em_div_ex),
	I(SrcMem, em_idiv_ex),
3419 3420 3421
};

static struct opcode group4[] = {
3422 3423
	I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
	I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3424 3425 3426 3427
	N, N, N, N, N, N,
};

static struct opcode group5[] = {
3428 3429 3430 3431 3432 3433 3434
	I(DstMem | SrcNone | Lock,		em_grp45),
	I(DstMem | SrcNone | Lock,		em_grp45),
	I(SrcMem | Stack,			em_grp45),
	I(SrcMemFAddr | ImplicitOps | Stack,	em_call_far),
	I(SrcMem | Stack,			em_grp45),
	I(SrcMemFAddr | ImplicitOps,		em_grp45),
	I(SrcMem | Stack,			em_grp45), N,
3435 3436
};

3437
static struct opcode group6[] = {
3438 3439 3440 3441
	DI(Prot,	sldt),
	DI(Prot,	str),
	DI(Prot | Priv,	lldt),
	DI(Prot | Priv,	ltr),
3442 3443 3444
	N, N, N, N,
};

3445
static struct group_dual group7 = { {
3446 3447 3448 3449 3450 3451 3452
	DI(Mov | DstMem | Priv,			sgdt),
	DI(Mov | DstMem | Priv,			sidt),
	II(SrcMem | Priv,			em_lgdt, lgdt),
	II(SrcMem | Priv,			em_lidt, lidt),
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	II(SrcMem | ByteOp | Priv | NoAccess,	em_invlpg, invlpg),
3453
}, {
3454
	I(SrcNone | Priv | VendorSpecific,	em_vmcall),
3455
	EXT(0, group7_rm1),
3456
	N, EXT(0, group7_rm3),
3457 3458 3459
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	EXT(0, group7_rm7),
3460 3461 3462 3463
} };

static struct opcode group8[] = {
	N, N, N, N,
3464 3465 3466 3467
	I(DstMem | SrcImmByte,				em_bt),
	I(DstMem | SrcImmByte | Lock | PageTable,	em_bts),
	I(DstMem | SrcImmByte | Lock,			em_btr),
	I(DstMem | SrcImmByte | Lock | PageTable,	em_btc),
3468 3469 3470
};

static struct group_dual group9 = { {
3471
	N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3472 3473 3474 3475
}, {
	N, N, N, N, N, N, N, N,
} };

3476
static struct opcode group11[] = {
3477
	I(DstMem | SrcImm | Mov | PageTable, em_mov),
3478
	X7(D(Undefined)),
3479 3480
};

3481
static struct gprefix pfx_0f_6f_0f_7f = {
3482
	I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3483 3484
};

3485 3486 3487 3488
static struct gprefix pfx_vmovntpx = {
	I(0, em_mov), N, N, N,
};

3489 3490
static struct opcode opcode_table[256] = {
	/* 0x00 - 0x07 */
3491
	I6ALU(Lock, em_add),
3492 3493
	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3494
	/* 0x08 - 0x0F */
3495
	I6ALU(Lock | PageTable, em_or),
3496 3497
	I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
	N,
3498
	/* 0x10 - 0x17 */
3499
	I6ALU(Lock, em_adc),
3500 3501
	I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3502
	/* 0x18 - 0x1F */
3503
	I6ALU(Lock, em_sbb),
3504 3505
	I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3506
	/* 0x20 - 0x27 */
3507
	I6ALU(Lock | PageTable, em_and), N, N,
3508
	/* 0x28 - 0x2F */
3509
	I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3510
	/* 0x30 - 0x37 */
3511
	I6ALU(Lock, em_xor), N, N,
3512
	/* 0x38 - 0x3F */
3513
	I6ALU(0, em_cmp), N, N,
3514 3515 3516
	/* 0x40 - 0x4F */
	X16(D(DstReg)),
	/* 0x50 - 0x57 */
3517
	X8(I(SrcReg | Stack, em_push)),
3518
	/* 0x58 - 0x5F */
3519
	X8(I(DstReg | Stack, em_pop)),
3520
	/* 0x60 - 0x67 */
3521 3522
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
3523 3524 3525
	N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
	N, N, N, N,
	/* 0x68 - 0x6F */
3526 3527
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3528 3529
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3530 3531
	I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
	I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3532 3533 3534
	/* 0x70 - 0x7F */
	X16(D(SrcImmByte)),
	/* 0x80 - 0x87 */
3535 3536 3537 3538
	G(ByteOp | DstMem | SrcImm, group1),
	G(DstMem | SrcImm, group1),
	G(ByteOp | DstMem | SrcImm | No64, group1),
	G(DstMem | SrcImmByte, group1),
3539
	I2bv(DstMem | SrcReg | ModRM, em_test),
3540
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3541
	/* 0x88 - 0x8F */
3542
	I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3543
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3544
	I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3545 3546 3547
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
3548
	/* 0x90 - 0x97 */
3549
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3550
	/* 0x98 - 0x9F */
3551
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3552
	I(SrcImmFAddr | No64, em_call_far), N,
3553 3554
	II(ImplicitOps | Stack, em_pushf, pushf),
	II(ImplicitOps | Stack, em_popf, popf), N, N,
3555
	/* 0xA0 - 0xA7 */
3556
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3557
	I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3558
	I2bv(SrcSI | DstDI | Mov | String, em_mov),
3559
	I2bv(SrcSI | DstDI | String, em_cmp),
3560
	/* 0xA8 - 0xAF */
3561
	I2bv(DstAcc | SrcImm, em_test),
3562 3563
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3564
	I2bv(SrcAcc | DstDI | String, em_cmp),
3565
	/* 0xB0 - 0xB7 */
3566
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3567
	/* 0xB8 - 0xBF */
3568
	X8(I(DstReg | SrcImm | Mov, em_mov)),
3569
	/* 0xC0 - 0xC7 */
3570
	D2bv(DstMem | SrcImmByte | ModRM),
3571
	I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3572
	I(ImplicitOps | Stack, em_ret),
3573 3574
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3575
	G(ByteOp, group11), G(0, group11),
3576
	/* 0xC8 - 0xCF */
3577
	N, N, N, I(ImplicitOps | Stack, em_ret_far),
3578
	D(ImplicitOps), DI(SrcImmByte, intn),
3579
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3580
	/* 0xD0 - 0xD7 */
3581
	D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3582 3583 3584 3585
	N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
3586 3587
	X3(I(SrcImmByte, em_loop)),
	I(SrcImmByte, em_jcxz),
3588 3589
	I2bvIP(SrcImmUByte | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3590
	/* 0xE8 - 0xEF */
3591
	I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3592
	I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3593 3594
	I2bvIP(SrcDX | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3595
	/* 0xF0 - 0xF7 */
3596
	N, DI(ImplicitOps, icebp), N, N,
3597 3598
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
3599
	/* 0xF8 - 0xFF */
3600 3601
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3602 3603 3604 3605 3606
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

static struct opcode twobyte_table[256] = {
	/* 0x00 - 0x0F */
3607
	G(0, group6), GD(0, &group7), N, N,
3608 3609
	N, I(ImplicitOps | VendorSpecific, em_syscall),
	II(ImplicitOps | Priv, em_clts, clts), N,
3610
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3611 3612 3613 3614
	N, D(ImplicitOps | ModRM), N, N,
	/* 0x10 - 0x1F */
	N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
	/* 0x20 - 0x2F */
3615
	DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3616
	DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3617 3618
	IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
	IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3619
	N, N, N, N,
3620 3621
	N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
	N, N, N, N,
3622
	/* 0x30 - 0x3F */
3623
	II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3624
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3625
	II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3626
	IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3627 3628
	I(ImplicitOps | VendorSpecific, em_sysenter),
	I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3629
	N, N,
3630 3631 3632 3633 3634 3635
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
	X16(D(DstReg | SrcMem | ModRM | Mov)),
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
3636 3637 3638 3639
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3640
	/* 0x70 - 0x7F */
3641 3642 3643 3644
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3645 3646 3647
	/* 0x80 - 0x8F */
	X16(D(SrcImm)),
	/* 0x90 - 0x9F */
3648
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3649
	/* 0xA0 - 0xA7 */
3650
	I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
A
Avi Kivity 已提交
3651
	II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3652 3653 3654
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM), N, N,
	/* 0xA8 - 0xAF */
3655
	I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3656
	DI(ImplicitOps, rsm),
3657
	I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3658 3659
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM),
3660
	D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3661
	/* 0xB0 - 0xB7 */
3662
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3663
	I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3664
	I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3665 3666
	I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3667
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3668 3669
	/* 0xB8 - 0xBF */
	N, N,
3670 3671
	G(BitOp, group8),
	I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3672
	I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3673
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3674
	/* 0xC0 - 0xCF */
3675
	D2bv(DstMem | SrcReg | ModRM | Lock),
3676
	N, D(DstMem | SrcReg | ModRM | Mov),
3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691
	N, N, N, GD(0, &group9),
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

#undef D
#undef N
#undef G
#undef GD
#undef I
3692
#undef GP
3693
#undef EXT
3694

3695
#undef D2bv
3696
#undef D2bvIP
3697
#undef I2bv
3698
#undef I2bvIP
3699
#undef I6ALU
3700

3701
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3702 3703 3704
{
	unsigned size;

3705
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
3718
	op->addr.mem.ea = ctxt->_eip;
3719 3720 3721
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
3722
		op->val = insn_fetch(s8, ctxt);
3723 3724
		break;
	case 2:
3725
		op->val = insn_fetch(s16, ctxt);
3726 3727
		break;
	case 4:
3728
		op->val = insn_fetch(s32, ctxt);
3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747
		break;
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

3748 3749 3750 3751 3752 3753 3754
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
			  unsigned d)
{
	int rc = X86EMUL_CONTINUE;

	switch (d) {
	case OpReg:
3755
		decode_register_operand(ctxt, op);
3756 3757
		break;
	case OpImmUByte:
3758
		rc = decode_imm(ctxt, op, 1, false);
3759 3760
		break;
	case OpMem:
3761
		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3762 3763 3764 3765
	mem_common:
		*op = ctxt->memop;
		ctxt->memopp = op;
		if ((ctxt->d & BitOp) && op == &ctxt->dst)
3766 3767 3768
			fetch_bit_operand(ctxt);
		op->orig_val = op->val;
		break;
3769 3770 3771
	case OpMem64:
		ctxt->memop.bytes = 8;
		goto mem_common;
3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792
	case OpAcc:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
	case OpDI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
		op->addr.mem.seg = VCPU_SREG_ES;
		op->val = 0;
		break;
	case OpDX:
		op->type = OP_REG;
		op->bytes = 2;
		op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
		fetch_register_operand(op);
		break;
3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806
	case OpCL:
		op->bytes = 1;
		op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
		break;
	case OpImmByte:
		rc = decode_imm(ctxt, op, 1, true);
		break;
	case OpOne:
		op->bytes = 1;
		op->val = 1;
		break;
	case OpImm:
		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
		break;
3807 3808 3809
	case OpMem8:
		ctxt->memop.bytes = 1;
		goto mem_common;
3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838
	case OpMem16:
		ctxt->memop.bytes = 2;
		goto mem_common;
	case OpMem32:
		ctxt->memop.bytes = 4;
		goto mem_common;
	case OpImmU16:
		rc = decode_imm(ctxt, op, 2, false);
		break;
	case OpImmU:
		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
		break;
	case OpSI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
		op->addr.mem.seg = seg_override(ctxt);
		op->val = 0;
		break;
	case OpImmFAddr:
		op->type = OP_IMM;
		op->addr.mem.ea = ctxt->_eip;
		op->bytes = ctxt->op_bytes + 2;
		insn_fetch_arr(op->valptr, op->bytes, ctxt);
		break;
	case OpMemFAddr:
		ctxt->memop.bytes = ctxt->op_bytes + 2;
		goto mem_common;
3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856
	case OpES:
		op->val = VCPU_SREG_ES;
		break;
	case OpCS:
		op->val = VCPU_SREG_CS;
		break;
	case OpSS:
		op->val = VCPU_SREG_SS;
		break;
	case OpDS:
		op->val = VCPU_SREG_DS;
		break;
	case OpFS:
		op->val = VCPU_SREG_FS;
		break;
	case OpGS:
		op->val = VCPU_SREG_GS;
		break;
3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867
	case OpImplicit:
		/* Special instructions do their own operand decoding. */
	default:
		op->type = OP_NONE; /* Disable writeback. */
		break;
	}

done:
	return rc;
}

3868
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3869 3870 3871
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
3872
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3873
	bool op_prefix = false;
3874
	struct opcode opcode;
3875

3876 3877
	ctxt->memop.type = OP_NONE;
	ctxt->memopp = NULL;
3878 3879 3880
	ctxt->_eip = ctxt->eip;
	ctxt->fetch.start = ctxt->_eip;
	ctxt->fetch.end = ctxt->fetch.start + insn_len;
3881
	if (insn_len > 0)
3882
		memcpy(ctxt->fetch.data, insn, insn_len);
3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
3900
		return EMULATION_FAILED;
3901 3902
	}

3903 3904
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
3905 3906 3907

	/* Legacy prefixes. */
	for (;;) {
3908
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
3909
		case 0x66:	/* operand-size override */
3910
			op_prefix = true;
3911
			/* switch between 2/4 bytes */
3912
			ctxt->op_bytes = def_op_bytes ^ 6;
3913 3914 3915 3916
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
3917
				ctxt->ad_bytes = def_ad_bytes ^ 12;
3918 3919
			else
				/* switch between 2/4 bytes */
3920
				ctxt->ad_bytes = def_ad_bytes ^ 6;
3921 3922 3923 3924 3925
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
3926
			set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3927 3928 3929
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
3930
			set_seg_override(ctxt, ctxt->b & 7);
3931 3932 3933 3934
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
3935
			ctxt->rex_prefix = ctxt->b;
3936 3937
			continue;
		case 0xf0:	/* LOCK */
3938
			ctxt->lock_prefix = 1;
3939 3940 3941
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
3942
			ctxt->rep_prefix = ctxt->b;
3943 3944 3945 3946 3947 3948 3949
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

3950
		ctxt->rex_prefix = 0;
3951 3952 3953 3954 3955
	}

done_prefixes:

	/* REX prefix. */
3956 3957
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
3958 3959

	/* Opcode byte(s). */
3960
	opcode = opcode_table[ctxt->b];
3961
	/* Two-byte opcode? */
3962 3963
	if (ctxt->b == 0x0f) {
		ctxt->twobyte = 1;
3964
		ctxt->b = insn_fetch(u8, ctxt);
3965
		opcode = twobyte_table[ctxt->b];
3966
	}
3967
	ctxt->d = opcode.flags;
3968

3969 3970 3971
	if (ctxt->d & ModRM)
		ctxt->modrm = insn_fetch(u8, ctxt);

3972 3973
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
3974
		case Group:
3975
			goffset = (ctxt->modrm >> 3) & 7;
3976 3977 3978
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
3979 3980
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
3981 3982 3983 3984 3985
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
3986
			goffset = ctxt->modrm & 7;
3987
			opcode = opcode.u.group[goffset];
3988 3989
			break;
		case Prefix:
3990
			if (ctxt->rep_prefix && op_prefix)
3991
				return EMULATION_FAILED;
3992
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3993 3994 3995 3996 3997 3998 3999 4000
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
		default:
4001
			return EMULATION_FAILED;
4002
		}
4003

4004
		ctxt->d &= ~(u64)GroupMask;
4005
		ctxt->d |= opcode.flags;
4006 4007
	}

4008 4009 4010
	ctxt->execute = opcode.u.execute;
	ctxt->check_perm = opcode.check_perm;
	ctxt->intercept = opcode.intercept;
4011 4012

	/* Unrecognised? */
4013
	if (ctxt->d == 0 || (ctxt->d & Undefined))
4014
		return EMULATION_FAILED;
4015

4016
	if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4017
		return EMULATION_FAILED;
4018

4019 4020
	if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
		ctxt->op_bytes = 8;
4021

4022
	if (ctxt->d & Op3264) {
4023
		if (mode == X86EMUL_MODE_PROT64)
4024
			ctxt->op_bytes = 8;
4025
		else
4026
			ctxt->op_bytes = 4;
4027 4028
	}

4029 4030
	if (ctxt->d & Sse)
		ctxt->op_bytes = 16;
A
Avi Kivity 已提交
4031 4032
	else if (ctxt->d & Mmx)
		ctxt->op_bytes = 8;
A
Avi Kivity 已提交
4033

4034
	/* ModRM and SIB bytes. */
4035
	if (ctxt->d & ModRM) {
4036
		rc = decode_modrm(ctxt, &ctxt->memop);
4037 4038 4039
		if (!ctxt->has_seg_override)
			set_seg_override(ctxt, ctxt->modrm_seg);
	} else if (ctxt->d & MemAbs)
4040
		rc = decode_abs(ctxt, &ctxt->memop);
4041 4042 4043
	if (rc != X86EMUL_CONTINUE)
		goto done;

4044 4045
	if (!ctxt->has_seg_override)
		set_seg_override(ctxt, VCPU_SREG_DS);
4046

4047
	ctxt->memop.addr.mem.seg = seg_override(ctxt);
4048

4049 4050
	if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4051 4052 4053 4054 4055

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
4056
	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4057 4058 4059
	if (rc != X86EMUL_CONTINUE)
		goto done;

4060 4061 4062 4063
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
4064
	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4065 4066 4067
	if (rc != X86EMUL_CONTINUE)
		goto done;

4068
	/* Decode and fetch the destination operand: register or memory. */
4069
	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4070 4071

done:
4072 4073
	if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
		ctxt->memopp->addr.mem.ea += ctxt->_eip;
4074

4075
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4076 4077
}

4078 4079 4080 4081 4082
bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
{
	return ctxt->d & PageTable;
}

4083 4084 4085 4086 4087 4088 4089 4090 4091
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
4092 4093 4094
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
4095
		 ((ctxt->eflags & EFLG_ZF) == 0))
4096
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
4097 4098 4099 4100 4101 4102
		    ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
		return true;

	return false;
}

A
Avi Kivity 已提交
4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115
static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
{
	bool fault = false;

	ctxt->ops->get_fpu(ctxt);
	asm volatile("1: fwait \n\t"
		     "2: \n\t"
		     ".pushsection .fixup,\"ax\" \n\t"
		     "3: \n\t"
		     "movb $1, %[fault] \n\t"
		     "jmp 2b \n\t"
		     ".popsection \n\t"
		     _ASM_EXTABLE(1b, 3b)
4116
		     : [fault]"+qm"(fault));
A
Avi Kivity 已提交
4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131
	ctxt->ops->put_fpu(ctxt);

	if (unlikely(fault))
		return emulate_exception(ctxt, MF_VECTOR, 0, false);

	return X86EMUL_CONTINUE;
}

static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
				       struct operand *op)
{
	if (op->type == OP_MM)
		read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
}

4132
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4133
{
4134
	struct x86_emulate_ops *ops = ctxt->ops;
4135
	int rc = X86EMUL_CONTINUE;
4136
	int saved_dst_type = ctxt->dst.type;
4137

4138
	ctxt->mem_read.pos = 0;
4139

4140
	if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4141
		rc = emulate_ud(ctxt);
4142 4143 4144
		goto done;
	}

4145
	/* LOCK prefix is allowed only with some instructions */
4146
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4147
		rc = emulate_ud(ctxt);
4148 4149 4150
		goto done;
	}

4151
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4152
		rc = emulate_ud(ctxt);
4153 4154 4155
		goto done;
	}

A
Avi Kivity 已提交
4156 4157
	if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
	    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
A
Avi Kivity 已提交
4158 4159 4160 4161
		rc = emulate_ud(ctxt);
		goto done;
	}

A
Avi Kivity 已提交
4162
	if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
A
Avi Kivity 已提交
4163 4164 4165 4166
		rc = emulate_nm(ctxt);
		goto done;
	}

A
Avi Kivity 已提交
4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180
	if (ctxt->d & Mmx) {
		rc = flush_pending_x87_faults(ctxt);
		if (rc != X86EMUL_CONTINUE)
			goto done;
		/*
		 * Now that we know the fpu is exception safe, we can fetch
		 * operands from it.
		 */
		fetch_possible_mmx_operand(ctxt, &ctxt->src);
		fetch_possible_mmx_operand(ctxt, &ctxt->src2);
		if (!(ctxt->d & Mov))
			fetch_possible_mmx_operand(ctxt, &ctxt->dst);
	}

4181 4182
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
4183
					      X86_ICPT_PRE_EXCEPT);
4184 4185 4186 4187
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4188
	/* Privileged instruction can be executed only in CPL=0 */
4189
	if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4190
		rc = emulate_gp(ctxt, 0);
4191 4192 4193
		goto done;
	}

4194
	/* Instruction can only be executed in protected mode */
4195
	if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
4196 4197 4198 4199
		rc = emulate_ud(ctxt);
		goto done;
	}

4200
	/* Do instruction specific permission checks */
4201 4202
	if (ctxt->check_perm) {
		rc = ctxt->check_perm(ctxt);
4203 4204 4205 4206
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4207 4208
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
4209
					      X86_ICPT_POST_EXCEPT);
4210 4211 4212 4213
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4214
	if (ctxt->rep_prefix && (ctxt->d & String)) {
4215
		/* All REP prefixes have the same first termination condition */
4216 4217
		if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
			ctxt->eip = ctxt->_eip;
4218 4219 4220 4221
			goto done;
		}
	}

4222 4223 4224
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
4225
		if (rc != X86EMUL_CONTINUE)
4226
			goto done;
4227
		ctxt->src.orig_val64 = ctxt->src.val64;
4228 4229
	}

4230 4231 4232
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
4233 4234 4235 4236
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4237
	if ((ctxt->d & DstMask) == ImplicitOps)
4238 4239 4240
		goto special_insn;


4241
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4242
		/* optimisation - avoid slow emulated read if Mov */
4243 4244
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
4245 4246
		if (rc != X86EMUL_CONTINUE)
			goto done;
4247
	}
4248
	ctxt->dst.orig_val = ctxt->dst.val;
4249

4250 4251
special_insn:

4252 4253
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
4254
					      X86_ICPT_POST_MEMACCESS);
4255 4256 4257 4258
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4259 4260
	if (ctxt->execute) {
		rc = ctxt->execute(ctxt);
4261 4262 4263 4264 4265
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

4266
	if (ctxt->twobyte)
A
Avi Kivity 已提交
4267 4268
		goto twobyte_insn;

4269
	switch (ctxt->b) {
4270
	case 0x40 ... 0x47: /* inc r16/r32 */
4271
		emulate_1op(ctxt, "inc");
4272 4273
		break;
	case 0x48 ... 0x4f: /* dec r16/r32 */
4274
		emulate_1op(ctxt, "dec");
4275
		break;
A
Avi Kivity 已提交
4276
	case 0x63:		/* movsxd */
4277
		if (ctxt->mode != X86EMUL_MODE_PROT64)
A
Avi Kivity 已提交
4278
			goto cannot_emulate;
4279
		ctxt->dst.val = (s32) ctxt->src.val;
A
Avi Kivity 已提交
4280
		break;
4281
	case 0x70 ... 0x7f: /* jcc (short) */
4282 4283
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
4284
		break;
N
Nitin A Kamble 已提交
4285
	case 0x8d: /* lea r16/r32, m */
4286
		ctxt->dst.val = ctxt->src.addr.mem.ea;
N
Nitin A Kamble 已提交
4287
		break;
4288
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
4289
		if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
4290
			break;
4291 4292
		rc = em_xchg(ctxt);
		break;
4293
	case 0x98: /* cbw/cwde/cdqe */
4294 4295 4296 4297
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4298 4299
		}
		break;
4300
	case 0xc0 ... 0xc1:
4301
		rc = em_grp2(ctxt);
4302
		break;
4303
	case 0xcc:		/* int3 */
4304 4305
		rc = emulate_int(ctxt, 3);
		break;
4306
	case 0xcd:		/* int n */
4307
		rc = emulate_int(ctxt, ctxt->src.val);
4308 4309
		break;
	case 0xce:		/* into */
4310 4311
		if (ctxt->eflags & EFLG_OF)
			rc = emulate_int(ctxt, 4);
4312
		break;
4313
	case 0xd0 ... 0xd1:	/* Grp2 */
4314
		rc = em_grp2(ctxt);
4315 4316
		break;
	case 0xd2 ... 0xd3:	/* Grp2 */
4317
		ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4318
		rc = em_grp2(ctxt);
4319
		break;
4320
	case 0xe9: /* jmp rel */
4321
	case 0xeb: /* jmp rel short */
4322 4323
		jmp_rel(ctxt, ctxt->src.val);
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
4324
		break;
4325
	case 0xf4:              /* hlt */
4326
		ctxt->ops->halt(ctxt);
4327
		break;
4328 4329 4330 4331 4332 4333 4334
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
		ctxt->eflags ^= EFLG_CF;
		break;
	case 0xf8: /* clc */
		ctxt->eflags &= ~EFLG_CF;
		break;
4335 4336 4337
	case 0xf9: /* stc */
		ctxt->eflags |= EFLG_CF;
		break;
4338 4339 4340 4341 4342 4343
	case 0xfc: /* cld */
		ctxt->eflags &= ~EFLG_DF;
		break;
	case 0xfd: /* std */
		ctxt->eflags |= EFLG_DF;
		break;
4344 4345
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4346
	}
4347

4348 4349 4350
	if (rc != X86EMUL_CONTINUE)
		goto done;

4351
writeback:
4352
	rc = writeback(ctxt);
4353
	if (rc != X86EMUL_CONTINUE)
4354 4355
		goto done;

4356 4357 4358 4359
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
4360
	ctxt->dst.type = saved_dst_type;
4361

4362 4363 4364
	if ((ctxt->d & SrcMask) == SrcSI)
		string_addr_inc(ctxt, seg_override(ctxt),
				VCPU_REGS_RSI, &ctxt->src);
4365

4366
	if ((ctxt->d & DstMask) == DstDI)
4367
		string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4368
				&ctxt->dst);
4369

4370 4371 4372
	if (ctxt->rep_prefix && (ctxt->d & String)) {
		struct read_cache *r = &ctxt->io_read;
		register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4373

4374 4375 4376 4377 4378
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
4379
			if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4380 4381 4382 4383 4384 4385
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
4386
				ctxt->mem_read.end = 0;
4387 4388 4389
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
4390
		}
4391
	}
4392

4393
	ctxt->eip = ctxt->_eip;
4394 4395

done:
4396 4397
	if (rc == X86EMUL_PROPAGATE_FAULT)
		ctxt->have_exception = true;
4398 4399 4400
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

4401
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
4402 4403

twobyte_insn:
4404
	switch (ctxt->b) {
4405
	case 0x09:		/* wbinvd */
4406
		(ctxt->ops->wbinvd)(ctxt);
4407 4408
		break;
	case 0x08:		/* invd */
4409 4410 4411 4412
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
		break;
	case 0x20: /* mov cr, reg */
4413
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4414
		break;
A
Avi Kivity 已提交
4415
	case 0x21: /* mov from dr to reg */
4416
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
A
Avi Kivity 已提交
4417 4418
		break;
	case 0x40 ... 0x4f:	/* cmov */
4419 4420 4421
		ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
		if (!test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
4422
		break;
4423
	case 0x80 ... 0x8f: /* jnz rel, etc*/
4424 4425
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
4426
		break;
4427
	case 0x90 ... 0x9f:     /* setcc r/m8 */
4428
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4429
		break;
4430 4431
	case 0xa4: /* shld imm8, r, r/m */
	case 0xa5: /* shld cl, r, r/m */
4432
		emulate_2op_cl(ctxt, "shld");
4433 4434 4435
		break;
	case 0xac: /* shrd imm8, r, r/m */
	case 0xad: /* shrd cl, r, r/m */
4436
		emulate_2op_cl(ctxt, "shrd");
4437
		break;
4438 4439
	case 0xae:              /* clflush */
		break;
A
Avi Kivity 已提交
4440
	case 0xb6 ... 0xb7:	/* movzx */
4441 4442 4443
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
						       : (u16) ctxt->src.val;
A
Avi Kivity 已提交
4444 4445
		break;
	case 0xbe ... 0xbf:	/* movsx */
4446 4447 4448
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
							(s16) ctxt->src.val;
A
Avi Kivity 已提交
4449
		break;
4450
	case 0xc0 ... 0xc1:	/* xadd */
4451
		emulate_2op_SrcV(ctxt, "add");
4452
		/* Write back the register source. */
4453 4454
		ctxt->src.val = ctxt->dst.orig_val;
		write_register_operand(&ctxt->src);
4455
		break;
4456
	case 0xc3:		/* movnti */
4457 4458 4459
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
							(u64) ctxt->src.val;
4460
		break;
4461 4462
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4463
	}
4464 4465 4466 4467

	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
4468 4469 4470
	goto writeback;

cannot_emulate:
4471
	return EMULATION_FAILED;
A
Avi Kivity 已提交
4472
}