emulate.c 111.7 KB
Newer Older
A
Avi Kivity 已提交
1
/******************************************************************************
2
 * emulate.c
A
Avi Kivity 已提交
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
A
Avi Kivity 已提交
10 11
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
A
Avi Kivity 已提交
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
A
Avi Kivity 已提交
27

28
#include "x86.h"
29
#include "tss.h"
30

A
Avi Kivity 已提交
31 32 33 34 35 36 37 38 39 40
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
41
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
42
/* Destination operand type. */
43 44 45 46 47 48
#define ImplicitOps (1<<1)	/* Implicit in opcode. No generic decode. */
#define DstReg      (2<<1)	/* Register operand. */
#define DstMem      (3<<1)	/* Memory operand. */
#define DstAcc      (4<<1)	/* Destination Accumulator */
#define DstDI       (5<<1)	/* Destination is in ES:(E)DI */
#define DstMem64    (6<<1)	/* 64bit memory operand */
49
#define DstImmUByte (7<<1)	/* 8-bit unsigned immediate operand */
50
#define DstMask     (7<<1)
A
Avi Kivity 已提交
51
/* Source operand type. */
52 53 54 55 56 57 58
#define SrcNone     (0<<4)	/* No source operand. */
#define SrcReg      (1<<4)	/* Register operand. */
#define SrcMem      (2<<4)	/* Memory operand. */
#define SrcMem16    (3<<4)	/* Memory operand (16-bit). */
#define SrcMem32    (4<<4)	/* Memory operand (32-bit). */
#define SrcImm      (5<<4)	/* Immediate operand. */
#define SrcImmByte  (6<<4)	/* 8-bit sign-extended immediate operand. */
59
#define SrcOne      (7<<4)	/* Implied '1' */
60
#define SrcImmUByte (8<<4)      /* 8-bit unsigned immediate operand. */
61
#define SrcImmU     (9<<4)      /* Immediate operand, unsigned */
62
#define SrcSI       (0xa<<4)	/* Source is in the DS:RSI */
63 64
#define SrcImmFAddr (0xb<<4)	/* Source is immediate far address */
#define SrcMemFAddr (0xc<<4)	/* Source is far address in memory */
65
#define SrcAcc      (0xd<<4)	/* Source Accumulator */
66
#define SrcImmU16   (0xe<<4)    /* Immediate operand, unsigned, 16 bits */
67
#define SrcMask     (0xf<<4)
A
Avi Kivity 已提交
68
/* Generic ModRM decode. */
69
#define ModRM       (1<<8)
A
Avi Kivity 已提交
70
/* Destination is only written; never read. */
71 72 73
#define Mov         (1<<9)
#define BitOp       (1<<10)
#define MemAbs      (1<<11)      /* Memory operand is absolute displacement */
74 75
#define String      (1<<12)     /* String instruction (rep capable) */
#define Stack       (1<<13)     /* Stack instruction (push/pop) */
76 77
#define Group       (1<<14)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (1<<15)     /* Alternate decoding of mod == 3 */
78
#define Prefix      (1<<16)     /* Instruction varies with 66/f2/f3 prefix */
A
Avi Kivity 已提交
79
#define Sse         (1<<17)     /* SSE Vector instruction */
80
#define RMExt       (1<<18)     /* Opcode extension in ModRM r/m if mod == 3 */
81
/* Misc flags */
82
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
83
#define VendorSpecific (1<<22) /* Vendor specific instruction */
84
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
85
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
86
#define Undefined   (1<<25) /* No Such Instruction */
87
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
88
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
89
#define No64	    (1<<28)
90 91 92 93 94
/* Source 2 operand type */
#define Src2None    (0<<29)
#define Src2CL      (1<<29)
#define Src2ImmByte (2<<29)
#define Src2One     (3<<29)
95
#define Src2Imm     (4<<29)
96
#define Src2Mask    (7<<29)
A
Avi Kivity 已提交
97

98 99 100 101 102 103 104 105
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
106

107 108
struct opcode {
	u32 flags;
109
	u8 intercept;
110
	union {
111
		int (*execute)(struct x86_emulate_ctxt *ctxt);
112 113
		struct opcode *group;
		struct group_dual *gdual;
114
		struct gprefix *gprefix;
115
	} u;
116
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
117 118 119 120 121
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
122 123
};

124 125 126 127 128 129 130
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

A
Avi Kivity 已提交
131
/* EFLAGS bit definitions. */
132 133 134 135
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
136 137
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
138 139
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
A
Avi Kivity 已提交
140 141
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
142
#define EFLG_IF (1<<9)
143
#define EFLG_TF (1<<8)
A
Avi Kivity 已提交
144 145 146 147 148 149
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

150 151 152
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

A
Avi Kivity 已提交
153 154 155 156 157 158 159
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

160
#if defined(CONFIG_X86_64)
A
Avi Kivity 已提交
161 162 163 164 165 166 167 168 169 170 171 172 173 174
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
A
Avi Kivity 已提交
190 191 192 193 194 195 196 197 198

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

199 200 201 202 203 204
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

205
#define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
206 207 208 209 210
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
211
			: "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
212 213
			  "=&r" (_tmp)					\
			: _y ((_src).val), "i" (EFLAGS_MASK));		\
214
	} while (0)
215 216


A
Avi Kivity 已提交
217 218
/* Raw emulation: instruction has two explicit operands. */
#define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
219 220 221 222 223
	do {								\
		unsigned long _tmp;					\
									\
		switch ((_dst).bytes) {					\
		case 2:							\
224
			____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
225 226
			break;						\
		case 4:							\
227
			____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
228 229
			break;						\
		case 8:							\
230
			ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
231 232
			break;						\
		}							\
A
Avi Kivity 已提交
233 234 235 236
	} while (0)

#define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
	do {								     \
237
		unsigned long _tmp;					     \
M
Mike Day 已提交
238
		switch ((_dst).bytes) {				             \
A
Avi Kivity 已提交
239
		case 1:							     \
240
			____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
A
Avi Kivity 已提交
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
			break;						     \
		default:						     \
			__emulate_2op_nobyte(_op, _src, _dst, _eflags,	     \
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
#define emulate_2op_SrcB(_op, _src, _dst, _eflags)                      \
	__emulate_2op(_op, _src, _dst, _eflags,				\
		      "b", "c", "b", "c", "b", "c", "b", "c")

/* Source operand is byte, word, long or quad sized. */
#define emulate_2op_SrcV(_op, _src, _dst, _eflags)                      \
	__emulate_2op(_op, _src, _dst, _eflags,				\
		      "b", "q", "w", "r", _LO32, "r", "", "r")

/* Source operand is word, long or quad sized. */
#define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags)               \
	__emulate_2op_nobyte(_op, _src, _dst, _eflags,			\
			     "w", "r", _LO32, "r", "", "r")

264
/* Instruction has three operands and one operand is stored in ECX register */
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
#define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type)	\
	do {								\
		unsigned long _tmp;					\
		_type _clv  = (_cl).val;				\
		_type _srcv = (_src).val;				\
		_type _dstv = (_dst).val;				\
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
			: "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp)	\
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
		(_cl).val  = (unsigned long) _clv;			\
		(_src).val = (unsigned long) _srcv;			\
		(_dst).val = (unsigned long) _dstv;			\
283 284
	} while (0)

285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
#define emulate_2op_cl(_op, _cl, _src, _dst, _eflags)			\
	do {								\
		switch ((_dst).bytes) {					\
		case 2:							\
			__emulate_2op_cl(_op, _cl, _src, _dst, _eflags,	\
					 "w", unsigned short);         	\
			break;						\
		case 4:							\
			__emulate_2op_cl(_op, _cl, _src, _dst, _eflags,	\
					 "l", unsigned int);           	\
			break;						\
		case 8:							\
			ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
					      "q", unsigned long));	\
			break;						\
		}							\
301 302
	} while (0)

303
#define __emulate_1op(_op, _dst, _eflags, _suffix)			\
A
Avi Kivity 已提交
304 305 306
	do {								\
		unsigned long _tmp;					\
									\
307 308 309 310 311 312 313 314 315 316 317 318
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
			: "=m" (_eflags), "+m" ((_dst).val),		\
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
#define emulate_1op(_op, _dst, _eflags)                                    \
	do {								\
M
Mike Day 已提交
319
		switch ((_dst).bytes) {				        \
320 321 322 323
		case 1:	__emulate_1op(_op, _dst, _eflags, "b"); break;	\
		case 2:	__emulate_1op(_op, _dst, _eflags, "w"); break;	\
		case 4:	__emulate_1op(_op, _dst, _eflags, "l"); break;	\
		case 8:	ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
A
Avi Kivity 已提交
324 325 326
		}							\
	} while (0)

327 328 329 330 331 332 333 334 335 336 337 338 339 340
#define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix)		\
	do {								\
		unsigned long _tmp;					\
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "1")			\
			_op _suffix " %5; "				\
			_POST_EFLAGS("0", "4", "1")			\
			: "=m" (_eflags), "=&r" (_tmp),			\
			  "+a" (_rax), "+d" (_rdx)			\
			: "i" (EFLAGS_MASK), "m" ((_src).val),		\
			  "a" (_rax), "d" (_rdx));			\
	} while (0)

341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
#define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
	do {								\
		unsigned long _tmp;					\
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
			: "=m" (_eflags), "=&r" (_tmp),			\
			  "+a" (_rax), "+d" (_rdx), "+qm"(_ex)		\
			: "i" (EFLAGS_MASK), "m" ((_src).val),		\
			  "a" (_rax), "d" (_rdx));			\
	} while (0)

362
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
#define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags)		\
	do {								\
		switch((_src).bytes) {					\
		case 1:							\
			__emulate_1op_rax_rdx(_op, _src, _rax, _rdx,	\
					      _eflags, "b");		\
			break;						\
		case 2:							\
			__emulate_1op_rax_rdx(_op, _src, _rax, _rdx,	\
					      _eflags, "w");		\
			break;						\
		case 4:							\
			__emulate_1op_rax_rdx(_op, _src, _rax, _rdx,	\
					      _eflags, "l");		\
			break;						\
		case 8:							\
			ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
						   _eflags, "q"));	\
			break;						\
382 383 384
		}							\
	} while (0)

385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
#define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex)	\
	do {								\
		switch((_src).bytes) {					\
		case 1:							\
			__emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx,	\
						 _eflags, "b", _ex);	\
			break;						\
		case 2:							\
			__emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
						 _eflags, "w", _ex);	\
			break;						\
		case 4:							\
			__emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
						 _eflags, "l", _ex);	\
			break;						\
		case 8: ON64(						\
			__emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
						 _eflags, "q", _ex));	\
			break;						\
		}							\
	} while (0)

A
Avi Kivity 已提交
407 408 409
/* Fetch next part of the instruction being emulated. */
#define insn_fetch(_type, _size, _eip)                                  \
({	unsigned long _x;						\
410
	rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size));		\
411
	if (rc != X86EMUL_CONTINUE)					\
A
Avi Kivity 已提交
412 413 414 415 416
		goto done;						\
	(_eip) += (_size);						\
	(_type)_x;							\
})

417
#define insn_fetch_arr(_arr, _size, _eip)				\
418 419 420 421 422 423
({	rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size));		\
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_eip) += (_size);						\
})

424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
		.rep_prefix = ctxt->decode.rep_prefix,
		.modrm_mod  = ctxt->decode.modrm_mod,
		.modrm_reg  = ctxt->decode.modrm_reg,
		.modrm_rm   = ctxt->decode.modrm_rm,
		.src_val    = ctxt->decode.src.val64,
		.src_bytes  = ctxt->decode.src.bytes,
		.dst_bytes  = ctxt->decode.dst.bytes,
		.ad_bytes   = ctxt->decode.ad_bytes,
		.next_rip   = ctxt->eip,
	};

441
	return ctxt->ops->intercept(ctxt, &info, stage);
442 443
}

444 445 446 447 448
static inline unsigned long ad_mask(struct decode_cache *c)
{
	return (1UL << (c->ad_bytes << 3)) - 1;
}

A
Avi Kivity 已提交
449
/* Access/update address held in a register, based on addressing mode. */
450 451 452 453 454 455 456 457 458 459
static inline unsigned long
address_mask(struct decode_cache *c, unsigned long reg)
{
	if (c->ad_bytes == sizeof(unsigned long))
		return reg;
	else
		return reg & ad_mask(c);
}

static inline unsigned long
460
register_address(struct decode_cache *c, unsigned long reg)
461
{
462
	return address_mask(c, reg);
463 464
}

465 466 467 468 469 470 471 472
static inline void
register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
{
	if (c->ad_bytes == sizeof(unsigned long))
		*reg += inc;
	else
		*reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
}
A
Avi Kivity 已提交
473

474 475 476 477
static inline void jmp_rel(struct decode_cache *c, int rel)
{
	register_address_increment(c, &c->eip, rel);
}
478

479 480 481 482 483 484 485
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

486 487 488 489 490 491
static void set_seg_override(struct decode_cache *c, int seg)
{
	c->has_seg_override = true;
	c->seg_override = seg;
}

492 493
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
			      struct x86_emulate_ops *ops, int seg)
494 495 496 497
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

498
	return ops->get_cached_segment_base(ctxt, seg);
499 500
}

501 502 503
static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
			     struct x86_emulate_ops *ops,
			     struct decode_cache *c)
504 505 506 507
{
	if (!c->has_seg_override)
		return 0;

508
	return c->seg_override;
509 510
}

511 512
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
513
{
514 515 516
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
517
	return X86EMUL_PROPAGATE_FAULT;
518 519
}

520 521 522 523 524
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

525
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
526
{
527
	return emulate_exception(ctxt, GP_VECTOR, err, true);
528 529
}

530 531 532 533 534
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

535
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
536
{
537
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
538 539
}

540
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
541
{
542
	return emulate_exception(ctxt, TS_VECTOR, err, true);
543 544
}

545 546
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
547
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
548 549
}

A
Avi Kivity 已提交
550 551 552 553 554
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

555
static int __linearize(struct x86_emulate_ctxt *ctxt,
556
		     struct segmented_address addr,
557
		     unsigned size, bool write, bool fetch,
558 559 560
		     ulong *linear)
{
	struct decode_cache *c = &ctxt->decode;
561 562
	struct desc_struct desc;
	bool usable;
563
	ulong la;
564 565
	u32 lim;
	unsigned cpl, rpl;
566 567

	la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
568 569 570 571 572 573 574 575
	switch (ctxt->mode) {
	case X86EMUL_MODE_REAL:
		break;
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
576 577
		usable = ctxt->ops->get_cached_descriptor(ctxt, &desc, NULL,
							  addr.seg);
578 579 580 581 582 583
		if (!usable)
			goto bad;
		/* code segment or read-only data segment */
		if (((desc.type & 8) || !(desc.type & 2)) && write)
			goto bad;
		/* unreadable code segment */
584
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
585 586 587 588 589 590 591 592 593 594 595 596 597 598
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
			/* exapand-down segment */
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
599
		cpl = ctxt->ops->cpl(ctxt);
600
		rpl = ctxt->ops->get_segment_selector(ctxt, addr.seg) & 3;
601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616
		cpl = max(cpl, rpl);
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
617
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
618 619 620
		la &= (u32)-1;
	*linear = la;
	return X86EMUL_CONTINUE;
621 622 623 624 625
bad:
	if (addr.seg == VCPU_SREG_SS)
		return emulate_ss(ctxt, addr.seg);
	else
		return emulate_gp(ctxt, addr.seg);
626 627
}

628 629 630 631 632 633 634 635 636
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


637 638 639 640 641
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
642 643 644
	int rc;
	ulong linear;

645
	rc = linearize(ctxt, addr, size, false, &linear);
646 647
	if (rc != X86EMUL_CONTINUE)
		return rc;
648
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
649 650
}

651 652
static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
			      struct x86_emulate_ops *ops,
653
			      unsigned long eip, u8 *dest)
654 655 656
{
	struct fetch_cache *fc = &ctxt->decode.fetch;
	int rc;
657
	int size, cur_size;
658

659
	if (eip == fc->end) {
660 661
		unsigned long linear;
		struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
662 663
		cur_size = fc->end - fc->start;
		size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
664 665 666
		rc = __linearize(ctxt, addr, size, false, true, &linear);
		if (rc != X86EMUL_CONTINUE)
			return rc;
667 668
		rc = ops->fetch(ctxt, linear, fc->data + cur_size,
				size, &ctxt->exception);
669
		if (rc != X86EMUL_CONTINUE)
670
			return rc;
671
		fc->end += size;
672
	}
673
	*dest = fc->data[eip - fc->start];
674
	return X86EMUL_CONTINUE;
675 676 677 678 679 680
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
			 struct x86_emulate_ops *ops,
			 unsigned long eip, void *dest, unsigned size)
{
681
	int rc;
682

683
	/* x86 instructions are limited to 15 bytes. */
684
	if (eip + size - ctxt->eip > 15)
685
		return X86EMUL_UNHANDLEABLE;
686 687
	while (size--) {
		rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
688
		if (rc != X86EMUL_CONTINUE)
689 690
			return rc;
	}
691
	return X86EMUL_CONTINUE;
692 693
}

694 695 696 697 698 699 700
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
static void *decode_register(u8 modrm_reg, unsigned long *regs,
			     int highbyte_regs)
A
Avi Kivity 已提交
701 702 703 704 705 706 707 708 709 710 711
{
	void *p;

	p = &regs[modrm_reg];
	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
		p = (unsigned char *)&regs[modrm_reg & 3] + 1;
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
			   struct x86_emulate_ops *ops,
712
			   struct segmented_address addr,
A
Avi Kivity 已提交
713 714 715 716 717 718 719
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
720
	rc = segmented_read_std(ctxt, addr, size, 2);
721
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
722
		return rc;
723
	addr.ea += 2;
724
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
725 726 727
	return rc;
}

728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762
static int test_cc(unsigned int condition, unsigned int flags)
{
	int rc = 0;

	switch ((condition & 15) >> 1) {
	case 0: /* o */
		rc |= (flags & EFLG_OF);
		break;
	case 1: /* b/c/nae */
		rc |= (flags & EFLG_CF);
		break;
	case 2: /* z/e */
		rc |= (flags & EFLG_ZF);
		break;
	case 3: /* be/na */
		rc |= (flags & (EFLG_CF|EFLG_ZF));
		break;
	case 4: /* s */
		rc |= (flags & EFLG_SF);
		break;
	case 5: /* p/pe */
		rc |= (flags & EFLG_PF);
		break;
	case 7: /* le/ng */
		rc |= (flags & EFLG_ZF);
		/* fall through */
	case 6: /* l/nge */
		rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
		break;
	}

	/* Odd condition identifiers (lsb == 1) have inverted sense. */
	return (!!rc ^ (condition & 1));
}

763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

A
Avi Kivity 已提交
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
				    struct operand *op,
838 839 840
				    struct decode_cache *c,
				    int inhibit_bytereg)
{
841
	unsigned reg = c->modrm_reg;
842
	int highbyte_regs = c->rex_prefix == 0;
843 844 845

	if (!(c->d & ModRM))
		reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
846 847 848 849 850 851 852 853 854

	if (c->d & Sse) {
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}

855 856
	op->type = OP_REG;
	if ((c->d & ByteOp) && !inhibit_bytereg) {
857
		op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
858 859
		op->bytes = 1;
	} else {
860
		op->addr.reg = decode_register(reg, c->regs, 0);
861 862
		op->bytes = c->op_bytes;
	}
863
	fetch_register_operand(op);
864 865 866
	op->orig_val = op->val;
}

867
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
868 869
			struct x86_emulate_ops *ops,
			struct operand *op)
870 871 872
{
	struct decode_cache *c = &ctxt->decode;
	u8 sib;
873
	int index_reg = 0, base_reg = 0, scale;
874
	int rc = X86EMUL_CONTINUE;
875
	ulong modrm_ea = 0;
876 877 878 879 880 881 882 883 884 885 886

	if (c->rex_prefix) {
		c->modrm_reg = (c->rex_prefix & 4) << 1;	/* REX.R */
		index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
		c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
	}

	c->modrm = insn_fetch(u8, 1, c->eip);
	c->modrm_mod |= (c->modrm & 0xc0) >> 6;
	c->modrm_reg |= (c->modrm & 0x38) >> 3;
	c->modrm_rm |= (c->modrm & 0x07);
887
	c->modrm_seg = VCPU_SREG_DS;
888 889

	if (c->modrm_mod == 3) {
890 891 892
		op->type = OP_REG;
		op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
		op->addr.reg = decode_register(c->modrm_rm,
893
					       c->regs, c->d & ByteOp);
A
Avi Kivity 已提交
894 895 896 897 898 899 900
		if (c->d & Sse) {
			op->type = OP_XMM;
			op->bytes = 16;
			op->addr.xmm = c->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
			return rc;
		}
901
		fetch_register_operand(op);
902 903 904
		return rc;
	}

905 906
	op->type = OP_MEM;

907 908 909 910 911 912 913 914 915 916
	if (c->ad_bytes == 2) {
		unsigned bx = c->regs[VCPU_REGS_RBX];
		unsigned bp = c->regs[VCPU_REGS_RBP];
		unsigned si = c->regs[VCPU_REGS_RSI];
		unsigned di = c->regs[VCPU_REGS_RDI];

		/* 16-bit ModR/M decode. */
		switch (c->modrm_mod) {
		case 0:
			if (c->modrm_rm == 6)
917
				modrm_ea += insn_fetch(u16, 2, c->eip);
918 919
			break;
		case 1:
920
			modrm_ea += insn_fetch(s8, 1, c->eip);
921 922
			break;
		case 2:
923
			modrm_ea += insn_fetch(u16, 2, c->eip);
924 925 926 927
			break;
		}
		switch (c->modrm_rm) {
		case 0:
928
			modrm_ea += bx + si;
929 930
			break;
		case 1:
931
			modrm_ea += bx + di;
932 933
			break;
		case 2:
934
			modrm_ea += bp + si;
935 936
			break;
		case 3:
937
			modrm_ea += bp + di;
938 939
			break;
		case 4:
940
			modrm_ea += si;
941 942
			break;
		case 5:
943
			modrm_ea += di;
944 945 946
			break;
		case 6:
			if (c->modrm_mod != 0)
947
				modrm_ea += bp;
948 949
			break;
		case 7:
950
			modrm_ea += bx;
951 952 953 954
			break;
		}
		if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
		    (c->modrm_rm == 6 && c->modrm_mod != 0))
955
			c->modrm_seg = VCPU_SREG_SS;
956
		modrm_ea = (u16)modrm_ea;
957 958
	} else {
		/* 32/64-bit ModR/M decode. */
959
		if ((c->modrm_rm & 7) == 4) {
960 961 962 963 964
			sib = insn_fetch(u8, 1, c->eip);
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

965
			if ((base_reg & 7) == 5 && c->modrm_mod == 0)
966
				modrm_ea += insn_fetch(s32, 4, c->eip);
967
			else
968
				modrm_ea += c->regs[base_reg];
969
			if (index_reg != 4)
970
				modrm_ea += c->regs[index_reg] << scale;
971 972
		} else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
			if (ctxt->mode == X86EMUL_MODE_PROT64)
973
				c->rip_relative = 1;
974
		} else
975
			modrm_ea += c->regs[c->modrm_rm];
976 977 978
		switch (c->modrm_mod) {
		case 0:
			if (c->modrm_rm == 5)
979
				modrm_ea += insn_fetch(s32, 4, c->eip);
980 981
			break;
		case 1:
982
			modrm_ea += insn_fetch(s8, 1, c->eip);
983 984
			break;
		case 2:
985
			modrm_ea += insn_fetch(s32, 4, c->eip);
986 987 988
			break;
		}
	}
989
	op->addr.mem.ea = modrm_ea;
990 991 992 993 994
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
995 996
		      struct x86_emulate_ops *ops,
		      struct operand *op)
997 998
{
	struct decode_cache *c = &ctxt->decode;
999
	int rc = X86EMUL_CONTINUE;
1000

1001
	op->type = OP_MEM;
1002 1003
	switch (c->ad_bytes) {
	case 2:
1004
		op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1005 1006
		break;
	case 4:
1007
		op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1008 1009
		break;
	case 8:
1010
		op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1011 1012 1013 1014 1015 1016
		break;
	}
done:
	return rc;
}

1017 1018
static void fetch_bit_operand(struct decode_cache *c)
{
1019
	long sv = 0, mask;
1020

1021
	if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1022 1023 1024 1025 1026 1027 1028
		mask = ~(c->dst.bytes * 8 - 1);

		if (c->src.bytes == 2)
			sv = (s16)c->src.val & (s16)mask;
		else if (c->src.bytes == 4)
			sv = (s32)c->src.val & (s32)mask;

1029
		c->dst.addr.mem.ea += (sv >> 3);
1030
	}
1031 1032 1033

	/* only subword offset */
	c->src.val &= (c->dst.bytes << 3) - 1;
1034 1035
}

1036 1037 1038
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 struct x86_emulate_ops *ops,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1039
{
1040 1041
	int rc;
	struct read_cache *mc = &ctxt->decode.mem_read;
A
Avi Kivity 已提交
1042

1043 1044 1045 1046 1047
	while (size) {
		int n = min(size, 8u);
		size -= n;
		if (mc->pos < mc->end)
			goto read_cached;
1048

1049 1050
		rc = ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
					&ctxt->exception);
1051 1052 1053
		if (rc != X86EMUL_CONTINUE)
			return rc;
		mc->end += n;
A
Avi Kivity 已提交
1054

1055 1056 1057 1058 1059
	read_cached:
		memcpy(dest, mc->data + mc->pos, n);
		mc->pos += n;
		dest += n;
		addr += n;
A
Avi Kivity 已提交
1060
	}
1061 1062
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1063

1064 1065 1066 1067 1068
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1069 1070 1071
	int rc;
	ulong linear;

1072
	rc = linearize(ctxt, addr, size, false, &linear);
1073 1074 1075
	if (rc != X86EMUL_CONTINUE)
		return rc;
	return read_emulated(ctxt, ctxt->ops, linear, data, size);
1076 1077 1078 1079 1080 1081 1082
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1083 1084 1085
	int rc;
	ulong linear;

1086
	rc = linearize(ctxt, addr, size, true, &linear);
1087 1088
	if (rc != X86EMUL_CONTINUE)
		return rc;
1089 1090
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1091 1092 1093 1094 1095 1096 1097
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1098 1099 1100
	int rc;
	ulong linear;

1101
	rc = linearize(ctxt, addr, size, true, &linear);
1102 1103
	if (rc != X86EMUL_CONTINUE)
		return rc;
1104 1105
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1106 1107
}

1108 1109 1110 1111 1112 1113
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   struct x86_emulate_ops *ops,
			   unsigned int size, unsigned short port,
			   void *dest)
{
	struct read_cache *rc = &ctxt->decode.io_read;
1114

1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
	if (rc->pos == rc->end) { /* refill pio read ahead */
		struct decode_cache *c = &ctxt->decode;
		unsigned int in_page, n;
		unsigned int count = c->rep_prefix ?
			address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
		in_page = (ctxt->eflags & EFLG_DF) ?
			offset_in_page(c->regs[VCPU_REGS_RDI]) :
			PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
		n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
			count);
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1128
		if (!ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1129 1130
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1131 1132
	}

1133 1134 1135 1136
	memcpy(dest, rc->data + rc->pos, size);
	rc->pos += size;
	return 1;
}
A
Avi Kivity 已提交
1137

1138 1139 1140 1141 1142 1143 1144
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     struct x86_emulate_ops *ops,
				     u16 selector, struct desc_ptr *dt)
{
	if (selector & 1 << 2) {
		struct desc_struct desc;
		memset (dt, 0, sizeof *dt);
1145 1146
		if (!ops->get_cached_descriptor(ctxt, &desc, NULL,
						VCPU_SREG_LDTR))
1147
			return;
1148

1149 1150 1151
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
		dt->address = get_desc_base(&desc);
	} else
1152
		ops->get_gdt(ctxt, dt);
1153
}
1154

1155 1156 1157 1158 1159 1160 1161 1162 1163
/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   struct x86_emulate_ops *ops,
				   u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	int ret;
	ulong addr;
1164

1165
	get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1166

1167 1168
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1169
	addr = dt.address + index * 8;
1170
	ret = ops->read_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1171

1172 1173
       return ret;
}
1174

1175 1176 1177 1178 1179 1180 1181 1182 1183
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    struct x86_emulate_ops *ops,
				    u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
	int ret;
A
Avi Kivity 已提交
1184

1185
	get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1186

1187 1188
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
A
Avi Kivity 已提交
1189

1190
	addr = dt.address + index * 8;
1191
	ret = ops->write_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1192

1193 1194
	return ret;
}
1195

1196
/* Does not support long mode */
1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   struct x86_emulate_ops *ops,
				   u16 selector, int seg)
{
	struct desc_struct seg_desc;
	u8 dpl, rpl, cpl;
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
	int ret;
1207

1208
	memset(&seg_desc, 0, sizeof seg_desc);
1209

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
	if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
	    || ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
		goto load;
	}

	/* NULL selector is not valid for TR, CS and SS */
	if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
	    && null_selector)
		goto exception;

	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

	if (null_selector) /* for NULL selector skip all following checks */
		goto load;

	ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
	err_vec = GP_VECTOR;

	/* can't load system descriptor into segment selecor */
	if (seg <= VCPU_SREG_GS && !seg_desc.s)
		goto exception;

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	rpl = selector & 3;
	dpl = seg_desc.dpl;
1251
	cpl = ops->cpl(ctxt);
1252 1253 1254 1255 1256 1257 1258 1259 1260

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1261
		break;
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1277
		break;
1278 1279 1280 1281 1282 1283 1284 1285 1286
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1287
		/*
1288 1289 1290
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1291
		 */
1292 1293 1294 1295
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1296
		break;
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
		seg_desc.type |= 1;
		ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
		if (ret != X86EMUL_CONTINUE)
			return ret;
	}
load:
1307 1308
	ops->set_segment_selector(ctxt, selector, seg);
	ops->set_cached_descriptor(ctxt, &seg_desc, 0, seg);
1309 1310 1311 1312 1313 1314
	return X86EMUL_CONTINUE;
exception:
	emulate_exception(ctxt, err_vec, err_code, true);
	return X86EMUL_PROPAGATE_FAULT;
}

1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
static void write_register_operand(struct operand *op)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (op->bytes) {
	case 1:
		*(u8 *)op->addr.reg = (u8)op->val;
		break;
	case 2:
		*(u16 *)op->addr.reg = (u16)op->val;
		break;
	case 4:
		*op->addr.reg = (u32)op->val;
		break;	/* 64b: zero-extend */
	case 8:
		*op->addr.reg = op->val;
		break;
	}
}

1334 1335 1336 1337 1338 1339 1340 1341
static inline int writeback(struct x86_emulate_ctxt *ctxt,
			    struct x86_emulate_ops *ops)
{
	int rc;
	struct decode_cache *c = &ctxt->decode;

	switch (c->dst.type) {
	case OP_REG:
1342
		write_register_operand(&c->dst);
A
Avi Kivity 已提交
1343
		break;
1344 1345
	case OP_MEM:
		if (c->lock_prefix)
1346 1347 1348 1349 1350
			rc = segmented_cmpxchg(ctxt,
					       c->dst.addr.mem,
					       &c->dst.orig_val,
					       &c->dst.val,
					       c->dst.bytes);
1351
		else
1352 1353 1354 1355
			rc = segmented_write(ctxt,
					     c->dst.addr.mem,
					     &c->dst.val,
					     c->dst.bytes);
1356 1357
		if (rc != X86EMUL_CONTINUE)
			return rc;
1358
		break;
A
Avi Kivity 已提交
1359 1360 1361
	case OP_XMM:
		write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
		break;
1362 1363
	case OP_NONE:
		/* no writeback */
1364
		break;
1365
	default:
1366
		break;
A
Avi Kivity 已提交
1367
	}
1368 1369
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1370

1371
static int em_push(struct x86_emulate_ctxt *ctxt)
1372 1373
{
	struct decode_cache *c = &ctxt->decode;
1374
	struct segmented_address addr;
1375

1376
	register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1377 1378 1379 1380 1381 1382
	addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
	addr.seg = VCPU_SREG_SS;

	/* Disable writeback. */
	c->dst.type = OP_NONE;
	return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1383
}
1384

1385 1386 1387 1388 1389 1390
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       struct x86_emulate_ops *ops,
		       void *dest, int len)
{
	struct decode_cache *c = &ctxt->decode;
	int rc;
1391
	struct segmented_address addr;
1392

1393 1394
	addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
	addr.seg = VCPU_SREG_SS;
1395
	rc = segmented_read(ctxt, addr, dest, len);
1396 1397 1398 1399 1400
	if (rc != X86EMUL_CONTINUE)
		return rc;

	register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
	return rc;
1401 1402
}

1403 1404 1405
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
		       struct x86_emulate_ops *ops,
		       void *dest, int len)
1406 1407
{
	int rc;
1408 1409
	unsigned long val, change_mask;
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1410
	int cpl = ops->cpl(ctxt);
1411

1412 1413 1414
	rc = emulate_pop(ctxt, ops, &val, len);
	if (rc != X86EMUL_CONTINUE)
		return rc;
1415

1416 1417
	change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
		| EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1418

1419 1420 1421 1422 1423 1424 1425 1426 1427 1428
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
			change_mask |= EFLG_IOPL;
		if (cpl <= iopl)
			change_mask |= EFLG_IF;
		break;
	case X86EMUL_MODE_VM86:
1429 1430
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1431 1432 1433 1434 1435
		change_mask |= EFLG_IF;
		break;
	default: /* real mode */
		change_mask |= (EFLG_IOPL | EFLG_IF);
		break;
1436
	}
1437 1438 1439 1440 1441

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1442 1443
}

1444 1445
static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
			     struct x86_emulate_ops *ops, int seg)
1446
{
1447
	struct decode_cache *c = &ctxt->decode;
1448

1449
	c->src.val = ops->get_segment_selector(ctxt, seg);
1450

1451
	return em_push(ctxt);
1452 1453
}

1454 1455
static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
			     struct x86_emulate_ops *ops, int seg)
1456
{
1457 1458 1459
	struct decode_cache *c = &ctxt->decode;
	unsigned long selector;
	int rc;
1460

1461 1462 1463 1464 1465 1466
	rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
	return rc;
1467 1468
}

1469
static int emulate_pusha(struct x86_emulate_ctxt *ctxt)
1470
{
1471 1472 1473 1474
	struct decode_cache *c = &ctxt->decode;
	unsigned long old_esp = c->regs[VCPU_REGS_RSP];
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1475

1476 1477 1478
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
		(c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1479

1480
		rc = em_push(ctxt);
1481 1482
		if (rc != X86EMUL_CONTINUE)
			return rc;
1483

1484
		++reg;
1485 1486
	}

1487
	return rc;
1488 1489
}

1490 1491
static int emulate_popa(struct x86_emulate_ctxt *ctxt,
			struct x86_emulate_ops *ops)
1492
{
1493 1494 1495
	struct decode_cache *c = &ctxt->decode;
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
1496

1497 1498 1499 1500 1501 1502
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
			register_address_increment(c, &c->regs[VCPU_REGS_RSP],
							c->op_bytes);
			--reg;
		}
1503

1504 1505 1506 1507
		rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
		if (rc != X86EMUL_CONTINUE)
			break;
		--reg;
1508
	}
1509
	return rc;
1510 1511
}

1512 1513 1514 1515
int emulate_int_real(struct x86_emulate_ctxt *ctxt,
			       struct x86_emulate_ops *ops, int irq)
{
	struct decode_cache *c = &ctxt->decode;
1516
	int rc;
1517 1518 1519 1520 1521 1522 1523
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
	c->src.val = ctxt->eflags;
1524
	rc = em_push(ctxt);
1525 1526
	if (rc != X86EMUL_CONTINUE)
		return rc;
1527 1528 1529

	ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);

1530
	c->src.val = ops->get_segment_selector(ctxt, VCPU_SREG_CS);
1531
	rc = em_push(ctxt);
1532 1533
	if (rc != X86EMUL_CONTINUE)
		return rc;
1534 1535

	c->src.val = c->eip;
1536
	rc = em_push(ctxt);
1537 1538 1539
	if (rc != X86EMUL_CONTINUE)
		return rc;

1540
	ops->get_idt(ctxt, &dt);
1541 1542 1543 1544

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

1545
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1546 1547 1548
	if (rc != X86EMUL_CONTINUE)
		return rc;

1549
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577
	if (rc != X86EMUL_CONTINUE)
		return rc;

	rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	c->eip = eip;

	return rc;
}

static int emulate_int(struct x86_emulate_ctxt *ctxt,
		       struct x86_emulate_ops *ops, int irq)
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
		return emulate_int_real(ctxt, ops, irq);
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

1578 1579
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
			     struct x86_emulate_ops *ops)
1580
{
1581 1582 1583 1584 1585 1586 1587 1588 1589
	struct decode_cache *c = &ctxt->decode;
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
	unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
			     EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
			     EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
	unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1590

1591
	/* TODO: Add stack limit check */
1592

1593
	rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1594

1595 1596
	if (rc != X86EMUL_CONTINUE)
		return rc;
1597

1598 1599
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
1600

1601
	rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1602

1603 1604
	if (rc != X86EMUL_CONTINUE)
		return rc;
1605

1606
	rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1607

1608 1609
	if (rc != X86EMUL_CONTINUE)
		return rc;
1610

1611
	rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1612

1613 1614
	if (rc != X86EMUL_CONTINUE)
		return rc;
1615

1616
	c->eip = temp_eip;
1617 1618


1619 1620 1621 1622 1623
	if (c->op_bytes == 4)
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
	else if (c->op_bytes == 2) {
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
1624
	}
1625 1626 1627 1628 1629

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
	ctxt->eflags |= EFLG_RESERVED_ONE_MASK;

	return rc;
1630 1631
}

1632 1633
static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
				    struct x86_emulate_ops* ops)
1634
{
1635 1636 1637 1638 1639 1640 1641
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
		return emulate_iret_real(ctxt, ops);
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
1642
	default:
1643 1644
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
1645 1646 1647
	}
}

1648
static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1649
				struct x86_emulate_ops *ops)
1650 1651 1652
{
	struct decode_cache *c = &ctxt->decode;

1653
	return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1654 1655
}

1656
static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1657
{
1658
	struct decode_cache *c = &ctxt->decode;
1659 1660
	switch (c->modrm_reg) {
	case 0:	/* rol */
1661
		emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1662 1663
		break;
	case 1:	/* ror */
1664
		emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1665 1666
		break;
	case 2:	/* rcl */
1667
		emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1668 1669
		break;
	case 3:	/* rcr */
1670
		emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1671 1672 1673
		break;
	case 4:	/* sal/shl */
	case 6:	/* sal/shl */
1674
		emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1675 1676
		break;
	case 5:	/* shr */
1677
		emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1678 1679
		break;
	case 7:	/* sar */
1680
		emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1681 1682 1683 1684 1685
		break;
	}
}

static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1686
			       struct x86_emulate_ops *ops)
1687 1688
{
	struct decode_cache *c = &ctxt->decode;
1689 1690
	unsigned long *rax = &c->regs[VCPU_REGS_RAX];
	unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1691
	u8 de = 0;
1692 1693 1694

	switch (c->modrm_reg) {
	case 0 ... 1:	/* test */
1695
		emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1696 1697 1698 1699 1700
		break;
	case 2:	/* not */
		c->dst.val = ~c->dst.val;
		break;
	case 3:	/* neg */
1701
		emulate_1op("neg", c->dst, ctxt->eflags);
1702
		break;
1703 1704 1705 1706 1707 1708 1709
	case 4: /* mul */
		emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
		break;
	case 5: /* imul */
		emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
		break;
	case 6: /* div */
1710 1711
		emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
				       ctxt->eflags, de);
1712 1713
		break;
	case 7: /* idiv */
1714 1715
		emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
				       ctxt->eflags, de);
1716
		break;
1717
	default:
1718
		return X86EMUL_UNHANDLEABLE;
1719
	}
1720 1721
	if (de)
		return emulate_de(ctxt);
1722
	return X86EMUL_CONTINUE;
1723 1724
}

1725
static int emulate_grp45(struct x86_emulate_ctxt *ctxt)
1726 1727
{
	struct decode_cache *c = &ctxt->decode;
1728
	int rc = X86EMUL_CONTINUE;
1729 1730 1731

	switch (c->modrm_reg) {
	case 0:	/* inc */
1732
		emulate_1op("inc", c->dst, ctxt->eflags);
1733 1734
		break;
	case 1:	/* dec */
1735
		emulate_1op("dec", c->dst, ctxt->eflags);
1736
		break;
1737 1738 1739 1740 1741
	case 2: /* call near abs */ {
		long int old_eip;
		old_eip = c->eip;
		c->eip = c->src.val;
		c->src.val = old_eip;
1742
		rc = em_push(ctxt);
1743 1744
		break;
	}
1745
	case 4: /* jmp abs */
1746
		c->eip = c->src.val;
1747 1748
		break;
	case 6:	/* push */
1749
		rc = em_push(ctxt);
1750 1751
		break;
	}
1752
	return rc;
1753 1754 1755
}

static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1756
			       struct x86_emulate_ops *ops)
1757 1758
{
	struct decode_cache *c = &ctxt->decode;
1759
	u64 old = c->dst.orig_val64;
1760 1761 1762 1763 1764

	if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
	    ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
		c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
		c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1765
		ctxt->eflags &= ~EFLG_ZF;
1766
	} else {
1767 1768
		c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
			(u32) c->regs[VCPU_REGS_RBX];
1769

1770
		ctxt->eflags |= EFLG_ZF;
1771
	}
1772
	return X86EMUL_CONTINUE;
1773 1774
}

1775 1776 1777 1778 1779 1780 1781 1782
static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
			   struct x86_emulate_ops *ops)
{
	struct decode_cache *c = &ctxt->decode;
	int rc;
	unsigned long cs;

	rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1783
	if (rc != X86EMUL_CONTINUE)
1784 1785 1786 1787
		return rc;
	if (c->op_bytes == 4)
		c->eip = (u32)c->eip;
	rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1788
	if (rc != X86EMUL_CONTINUE)
1789
		return rc;
1790
	rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1791 1792 1793
	return rc;
}

1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810
static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
			   struct x86_emulate_ops *ops, int seg)
{
	struct decode_cache *c = &ctxt->decode;
	unsigned short sel;
	int rc;

	memcpy(&sel, c->src.valptr + c->op_bytes, 2);

	rc = load_segment_descriptor(ctxt, ops, sel, seg);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	c->dst.val = c->src.val;
	return rc;
}

1811 1812
static inline void
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1813 1814
			struct x86_emulate_ops *ops, struct desc_struct *cs,
			struct desc_struct *ss)
1815
{
1816
	memset(cs, 0, sizeof(struct desc_struct));
1817
	ops->get_cached_descriptor(ctxt, cs, NULL, VCPU_SREG_CS);
1818
	memset(ss, 0, sizeof(struct desc_struct));
1819 1820

	cs->l = 0;		/* will be adjusted later */
1821
	set_desc_base(cs, 0);	/* flat segment */
1822
	cs->g = 1;		/* 4kb granularity */
1823
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
1824 1825 1826
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
1827 1828
	cs->p = 1;
	cs->d = 1;
1829

1830 1831
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
1832 1833 1834
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
1835
	ss->d = 1;		/* 32bit stack segment */
1836
	ss->dpl = 0;
1837
	ss->p = 1;
1838 1839 1840
}

static int
1841
emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1842 1843
{
	struct decode_cache *c = &ctxt->decode;
1844
	struct desc_struct cs, ss;
1845
	u64 msr_data;
1846
	u16 cs_sel, ss_sel;
1847
	u64 efer = 0;
1848 1849

	/* syscall is not available in real mode */
1850
	if (ctxt->mode == X86EMUL_MODE_REAL ||
1851 1852
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
1853

1854
	ops->get_msr(ctxt, MSR_EFER, &efer);
1855
	setup_syscalls_segments(ctxt, ops, &cs, &ss);
1856

1857
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
1858
	msr_data >>= 32;
1859 1860
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
1861

1862
	if (efer & EFER_LMA) {
1863
		cs.d = 0;
1864 1865
		cs.l = 1;
	}
1866 1867 1868 1869
	ops->set_cached_descriptor(ctxt, &cs, 0, VCPU_SREG_CS);
	ops->set_segment_selector(ctxt, cs_sel, VCPU_SREG_CS);
	ops->set_cached_descriptor(ctxt, &ss, 0, VCPU_SREG_SS);
	ops->set_segment_selector(ctxt, ss_sel, VCPU_SREG_SS);
1870 1871

	c->regs[VCPU_REGS_RCX] = c->eip;
1872
	if (efer & EFER_LMA) {
1873 1874 1875
#ifdef CONFIG_X86_64
		c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;

1876
		ops->get_msr(ctxt,
1877 1878
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
1879 1880
		c->eip = msr_data;

1881
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1882 1883 1884 1885
		ctxt->eflags &= ~(msr_data | EFLG_RF);
#endif
	} else {
		/* legacy mode */
1886
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
1887 1888 1889 1890 1891
		c->eip = (u32)msr_data;

		ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
	}

1892
	return X86EMUL_CONTINUE;
1893 1894
}

1895
static int
1896
emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1897 1898
{
	struct decode_cache *c = &ctxt->decode;
1899
	struct desc_struct cs, ss;
1900
	u64 msr_data;
1901
	u16 cs_sel, ss_sel;
1902
	u64 efer = 0;
1903

1904
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1905
	/* inject #GP if in real mode */
1906 1907
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
1908 1909 1910 1911

	/* XXX sysenter/sysexit have not been tested in 64bit mode.
	* Therefore, we inject an #UD.
	*/
1912 1913
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return emulate_ud(ctxt);
1914

1915
	setup_syscalls_segments(ctxt, ops, &cs, &ss);
1916

1917
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1918 1919
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT32:
1920 1921
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
1922 1923
		break;
	case X86EMUL_MODE_PROT64:
1924 1925
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
1926 1927 1928 1929
		break;
	}

	ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1930 1931 1932 1933
	cs_sel = (u16)msr_data;
	cs_sel &= ~SELECTOR_RPL_MASK;
	ss_sel = cs_sel + 8;
	ss_sel &= ~SELECTOR_RPL_MASK;
1934
	if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1935
		cs.d = 0;
1936 1937 1938
		cs.l = 1;
	}

1939 1940 1941 1942
	ops->set_cached_descriptor(ctxt, &cs, 0, VCPU_SREG_CS);
	ops->set_segment_selector(ctxt, cs_sel, VCPU_SREG_CS);
	ops->set_cached_descriptor(ctxt, &ss, 0, VCPU_SREG_SS);
	ops->set_segment_selector(ctxt, ss_sel, VCPU_SREG_SS);
1943

1944
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1945 1946
	c->eip = msr_data;

1947
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1948 1949
	c->regs[VCPU_REGS_RSP] = msr_data;

1950
	return X86EMUL_CONTINUE;
1951 1952
}

1953
static int
1954
emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1955 1956
{
	struct decode_cache *c = &ctxt->decode;
1957
	struct desc_struct cs, ss;
1958 1959
	u64 msr_data;
	int usermode;
1960
	u16 cs_sel, ss_sel;
1961

1962 1963
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
1964 1965
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
1966

1967
	setup_syscalls_segments(ctxt, ops, &cs, &ss);
1968 1969 1970 1971 1972 1973 1974 1975

	if ((c->rex_prefix & 0x8) != 0x0)
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

	cs.dpl = 3;
	ss.dpl = 3;
1976
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1977 1978
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
1979
		cs_sel = (u16)(msr_data + 16);
1980 1981
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
1982
		ss_sel = (u16)(msr_data + 24);
1983 1984
		break;
	case X86EMUL_MODE_PROT64:
1985
		cs_sel = (u16)(msr_data + 32);
1986 1987
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
1988 1989
		ss_sel = cs_sel + 8;
		cs.d = 0;
1990 1991 1992
		cs.l = 1;
		break;
	}
1993 1994
	cs_sel |= SELECTOR_RPL_MASK;
	ss_sel |= SELECTOR_RPL_MASK;
1995

1996 1997 1998 1999
	ops->set_cached_descriptor(ctxt, &cs, 0, VCPU_SREG_CS);
	ops->set_segment_selector(ctxt, cs_sel, VCPU_SREG_CS);
	ops->set_cached_descriptor(ctxt, &ss, 0, VCPU_SREG_SS);
	ops->set_segment_selector(ctxt, ss_sel, VCPU_SREG_SS);
2000

2001 2002
	c->eip = c->regs[VCPU_REGS_RDX];
	c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2003

2004
	return X86EMUL_CONTINUE;
2005 2006
}

2007 2008
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
			      struct x86_emulate_ops *ops)
2009 2010 2011 2012 2013 2014 2015
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2016
	return ops->cpl(ctxt) > iopl;
2017 2018 2019 2020 2021 2022
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    struct x86_emulate_ops *ops,
					    u16 port, u16 len)
{
2023
	struct desc_struct tr_seg;
2024
	u32 base3;
2025
	int r;
2026
	u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
2027
	unsigned mask = (1 << len) - 1;
2028
	unsigned long base;
2029

2030
	ops->get_cached_descriptor(ctxt, &tr_seg, &base3, VCPU_SREG_TR);
2031
	if (!tr_seg.p)
2032
		return false;
2033
	if (desc_limit_scaled(&tr_seg) < 103)
2034
		return false;
2035 2036 2037 2038
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2039
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2040 2041
	if (r != X86EMUL_CONTINUE)
		return false;
2042
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2043
		return false;
2044
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 struct x86_emulate_ops *ops,
				 u16 port, u16 len)
{
2056 2057 2058
	if (ctxt->perm_ok)
		return true;

2059
	if (emulator_bad_iopl(ctxt, ops))
2060 2061
		if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
			return false;
2062 2063 2064

	ctxt->perm_ok = true;

2065 2066 2067
	return true;
}

2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct x86_emulate_ops *ops,
				struct tss_segment_16 *tss)
{
	struct decode_cache *c = &ctxt->decode;

	tss->ip = c->eip;
	tss->flag = ctxt->eflags;
	tss->ax = c->regs[VCPU_REGS_RAX];
	tss->cx = c->regs[VCPU_REGS_RCX];
	tss->dx = c->regs[VCPU_REGS_RDX];
	tss->bx = c->regs[VCPU_REGS_RBX];
	tss->sp = c->regs[VCPU_REGS_RSP];
	tss->bp = c->regs[VCPU_REGS_RBP];
	tss->si = c->regs[VCPU_REGS_RSI];
	tss->di = c->regs[VCPU_REGS_RDI];

2085 2086 2087 2088 2089
	tss->es = ops->get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = ops->get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = ops->get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = ops->get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = ops->get_segment_selector(ctxt, VCPU_SREG_LDTR);
2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct x86_emulate_ops *ops,
				 struct tss_segment_16 *tss)
{
	struct decode_cache *c = &ctxt->decode;
	int ret;

	c->eip = tss->ip;
	ctxt->eflags = tss->flag | 2;
	c->regs[VCPU_REGS_RAX] = tss->ax;
	c->regs[VCPU_REGS_RCX] = tss->cx;
	c->regs[VCPU_REGS_RDX] = tss->dx;
	c->regs[VCPU_REGS_RBX] = tss->bx;
	c->regs[VCPU_REGS_RSP] = tss->sp;
	c->regs[VCPU_REGS_RBP] = tss->bp;
	c->regs[VCPU_REGS_RSI] = tss->si;
	c->regs[VCPU_REGS_RDI] = tss->di;

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2114 2115 2116 2117 2118
	ops->set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	ops->set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	ops->set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	ops->set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	ops->set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
	ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  struct x86_emulate_ops *ops,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
	struct tss_segment_16 tss_seg;
	int ret;
2150
	u32 new_tss_base = get_desc_base(new_desc);
2151

2152
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2153
			    &ctxt->exception);
2154
	if (ret != X86EMUL_CONTINUE)
2155 2156 2157 2158 2159
		/* FIXME: need to provide precise fault address */
		return ret;

	save_state_to_tss16(ctxt, ops, &tss_seg);

2160
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2161
			     &ctxt->exception);
2162
	if (ret != X86EMUL_CONTINUE)
2163 2164 2165
		/* FIXME: need to provide precise fault address */
		return ret;

2166
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2167
			    &ctxt->exception);
2168
	if (ret != X86EMUL_CONTINUE)
2169 2170 2171 2172 2173 2174
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2175
		ret = ops->write_std(ctxt, new_tss_base,
2176 2177
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2178
				     &ctxt->exception);
2179
		if (ret != X86EMUL_CONTINUE)
2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192
			/* FIXME: need to provide precise fault address */
			return ret;
	}

	return load_state_from_tss16(ctxt, ops, &tss_seg);
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct x86_emulate_ops *ops,
				struct tss_segment_32 *tss)
{
	struct decode_cache *c = &ctxt->decode;

2193
	tss->cr3 = ops->get_cr(ctxt, 3);
2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204
	tss->eip = c->eip;
	tss->eflags = ctxt->eflags;
	tss->eax = c->regs[VCPU_REGS_RAX];
	tss->ecx = c->regs[VCPU_REGS_RCX];
	tss->edx = c->regs[VCPU_REGS_RDX];
	tss->ebx = c->regs[VCPU_REGS_RBX];
	tss->esp = c->regs[VCPU_REGS_RSP];
	tss->ebp = c->regs[VCPU_REGS_RBP];
	tss->esi = c->regs[VCPU_REGS_RSI];
	tss->edi = c->regs[VCPU_REGS_RDI];

2205 2206 2207 2208 2209 2210 2211
	tss->es = ops->get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = ops->get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = ops->get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = ops->get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = ops->get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = ops->get_segment_selector(ctxt, VCPU_SREG_GS);
	tss->ldt_selector = ops->get_segment_selector(ctxt, VCPU_SREG_LDTR);
2212 2213 2214 2215 2216 2217 2218 2219 2220
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct x86_emulate_ops *ops,
				 struct tss_segment_32 *tss)
{
	struct decode_cache *c = &ctxt->decode;
	int ret;

2221
	if (ops->set_cr(ctxt, 3, tss->cr3))
2222
		return emulate_gp(ctxt, 0);
2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237
	c->eip = tss->eip;
	ctxt->eflags = tss->eflags | 2;
	c->regs[VCPU_REGS_RAX] = tss->eax;
	c->regs[VCPU_REGS_RCX] = tss->ecx;
	c->regs[VCPU_REGS_RDX] = tss->edx;
	c->regs[VCPU_REGS_RBX] = tss->ebx;
	c->regs[VCPU_REGS_RSP] = tss->esp;
	c->regs[VCPU_REGS_RBP] = tss->ebp;
	c->regs[VCPU_REGS_RSI] = tss->esi;
	c->regs[VCPU_REGS_RDI] = tss->edi;

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2238 2239 2240 2241 2242 2243 2244
	ops->set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	ops->set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	ops->set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	ops->set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	ops->set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	ops->set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	ops->set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
	ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  struct x86_emulate_ops *ops,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
	struct tss_segment_32 tss_seg;
	int ret;
2282
	u32 new_tss_base = get_desc_base(new_desc);
2283

2284
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2285
			    &ctxt->exception);
2286
	if (ret != X86EMUL_CONTINUE)
2287 2288 2289 2290 2291
		/* FIXME: need to provide precise fault address */
		return ret;

	save_state_to_tss32(ctxt, ops, &tss_seg);

2292
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2293
			     &ctxt->exception);
2294
	if (ret != X86EMUL_CONTINUE)
2295 2296 2297
		/* FIXME: need to provide precise fault address */
		return ret;

2298
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2299
			    &ctxt->exception);
2300
	if (ret != X86EMUL_CONTINUE)
2301 2302 2303 2304 2305 2306
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2307
		ret = ops->write_std(ctxt, new_tss_base,
2308 2309
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2310
				     &ctxt->exception);
2311
		if (ret != X86EMUL_CONTINUE)
2312 2313 2314 2315 2316 2317 2318 2319
			/* FIXME: need to provide precise fault address */
			return ret;
	}

	return load_state_from_tss32(ctxt, ops, &tss_seg);
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2320 2321 2322
				   struct x86_emulate_ops *ops,
				   u16 tss_selector, int reason,
				   bool has_error_code, u32 error_code)
2323 2324 2325
{
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
2326
	u16 old_tss_sel = ops->get_segment_selector(ctxt, VCPU_SREG_TR);
2327
	ulong old_tss_base =
2328
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2329
	u32 desc_limit;
2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343

	/* FIXME: old_tss_base == ~0 ? */

	ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
	if (ret != X86EMUL_CONTINUE)
		return ret;
	ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

	if (reason != TASK_SWITCH_IRET) {
		if ((tss_selector & 3) > next_tss_desc.dpl ||
2344
		    ops->cpl(ctxt) > next_tss_desc.dpl)
2345
			return emulate_gp(ctxt, 0);
2346 2347
	}

2348 2349 2350 2351
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
2352
		emulate_ts(ctxt, tss_selector & 0xfffc);
2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375
		return X86EMUL_PROPAGATE_FAULT;
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
		write_segment_descriptor(ctxt, ops, old_tss_sel,
					 &curr_tss_desc);
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
	   note that old_tss_sel is not used afetr this point */
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
		ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
				     old_tss_base, &next_tss_desc);
	else
		ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
				     old_tss_base, &next_tss_desc);
2376 2377
	if (ret != X86EMUL_CONTINUE)
		return ret;
2378 2379 2380 2381 2382 2383 2384 2385 2386 2387

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
		write_segment_descriptor(ctxt, ops, tss_selector,
					 &next_tss_desc);
	}

2388
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
2389 2390
	ops->set_cached_descriptor(ctxt, &next_tss_desc, 0, VCPU_SREG_TR);
	ops->set_segment_selector(ctxt, tss_selector, VCPU_SREG_TR);
2391

2392 2393 2394 2395 2396 2397
	if (has_error_code) {
		struct decode_cache *c = &ctxt->decode;

		c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		c->lock_prefix = 0;
		c->src.val = (unsigned long) error_code;
2398
		ret = em_push(ctxt);
2399 2400
	}

2401 2402 2403 2404
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2405 2406
			 u16 tss_selector, int reason,
			 bool has_error_code, u32 error_code)
2407
{
2408
	struct x86_emulate_ops *ops = ctxt->ops;
2409 2410 2411 2412
	struct decode_cache *c = &ctxt->decode;
	int rc;

	c->eip = ctxt->eip;
2413
	c->dst.type = OP_NONE;
2414

2415 2416
	rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
				     has_error_code, error_code);
2417

2418 2419
	if (rc == X86EMUL_CONTINUE)
		ctxt->eip = c->eip;
2420

2421
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2422 2423
}

2424
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2425
			    int reg, struct operand *op)
2426 2427 2428 2429
{
	struct decode_cache *c = &ctxt->decode;
	int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;

2430
	register_address_increment(c, &c->regs[reg], df * op->bytes);
2431 2432
	op->addr.mem.ea = register_address(c, c->regs[reg]);
	op->addr.mem.seg = seg;
2433 2434
}

2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
	al = c->dst.val;

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

	c->dst.val = al;
	/* Set PF, ZF, SF */
	c->src.type = OP_IMM;
	c->src.val = 0;
	c->src.bytes = 1;
	emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

2474 2475 2476 2477 2478 2479 2480
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	u16 sel, old_cs;
	ulong old_eip;
	int rc;

2481
	old_cs = ctxt->ops->get_segment_selector(ctxt, VCPU_SREG_CS);
2482 2483 2484 2485 2486 2487 2488 2489 2490 2491
	old_eip = c->eip;

	memcpy(&sel, c->src.valptr + c->op_bytes, 2);
	if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
		return X86EMUL_CONTINUE;

	c->eip = 0;
	memcpy(&c->eip, c->src.valptr, c->op_bytes);

	c->src.val = old_cs;
2492
	rc = em_push(ctxt);
2493 2494 2495 2496
	if (rc != X86EMUL_CONTINUE)
		return rc;

	c->src.val = old_eip;
2497
	return em_push(ctxt);
2498 2499
}

2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	int rc;

	c->dst.type = OP_REG;
	c->dst.addr.reg = &c->eip;
	c->dst.bytes = c->op_bytes;
	rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
	return X86EMUL_CONTINUE;
}

2515
static int em_imul(struct x86_emulate_ctxt *ctxt)
2516 2517 2518 2519 2520 2521 2522
{
	struct decode_cache *c = &ctxt->decode;

	emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
	return X86EMUL_CONTINUE;
}

2523 2524 2525 2526 2527 2528 2529 2530
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;

	c->dst.val = c->src2.val;
	return em_imul(ctxt);
}

2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;

	c->dst.type = OP_REG;
	c->dst.bytes = c->src.bytes;
	c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
	c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);

	return X86EMUL_CONTINUE;
}

2543 2544 2545 2546 2547
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	u64 tsc = 0;

2548
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2549 2550 2551 2552 2553
	c->regs[VCPU_REGS_RAX] = (u32)tsc;
	c->regs[VCPU_REGS_RDX] = tsc >> 32;
	return X86EMUL_CONTINUE;
}

2554 2555 2556 2557 2558 2559 2560
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	c->dst.val = c->src.val;
	return X86EMUL_CONTINUE;
}

2561 2562 2563 2564 2565 2566 2567
static int em_movdqu(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
	return X86EMUL_CONTINUE;
}

2568 2569 2570
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
2571 2572 2573
	int rc;
	ulong linear;

2574
	rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2575
	if (rc == X86EMUL_CONTINUE)
2576
		ctxt->ops->invlpg(ctxt, linear);
2577 2578 2579 2580 2581
	/* Disable writeback. */
	c->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

2582 2583 2584 2585 2586 2587 2588 2589 2590 2591
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680
static int em_vmcall(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	int rc;

	if (c->modrm_mod != 3 || c->modrm_rm != 1)
		return X86EMUL_UNHANDLEABLE;

	rc = ctxt->ops->fix_hypercall(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
	c->eip = ctxt->eip;
	/* Disable writeback. */
	c->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	struct desc_ptr desc_ptr;
	int rc;

	rc = read_descriptor(ctxt, ctxt->ops, c->src.addr.mem,
			     &desc_ptr.size, &desc_ptr.address,
			     c->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_gdt(ctxt, &desc_ptr);
	/* Disable writeback. */
	c->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_svm(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	int rc;

	switch (c->modrm_rm) {
	case 1:
		rc = ctxt->ops->fix_hypercall(ctxt);
		break;
	default:
		return X86EMUL_UNHANDLEABLE;
	}
	/* Disable writeback. */
	c->dst.type = OP_NONE;
	return rc;
}

static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	struct desc_ptr desc_ptr;
	int rc;

	rc = read_descriptor(ctxt, ctxt->ops, c->src.addr.mem,
			     &desc_ptr.size,
			     &desc_ptr.address,
			     c->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_idt(ctxt, &desc_ptr);
	/* Disable writeback. */
	c->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;

	c->dst.bytes = 2;
	c->dst.val = ctxt->ops->get_cr(ctxt, 0);
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
			  | (c->src.val & 0x0f));
	c->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;

	if (!valid_cr(c->modrm_reg))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	u64 new_val = c->src.val64;
	int cr = c->modrm_reg;
2708
	u64 efer = 0;
2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
2726
		u64 cr4;
2727 2728 2729 2730
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

2731 2732
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2733 2734 2735 2736 2737 2738 2739 2740 2741 2742

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

2743 2744
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
2745
			rsvd = CR3_L_MODE_RESERVED_BITS;
2746
		else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2747
			rsvd = CR3_PAE_RESERVED_BITS;
2748
		else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2749 2750 2751 2752 2753 2754 2755 2756
			rsvd = CR3_NONPAE_RESERVED_BITS;

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
2757
		u64 cr4;
2758

2759 2760
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

2772 2773 2774 2775
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

2776
	ctxt->ops->get_dr(ctxt, 7, &dr7);
2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	int dr = c->modrm_reg;
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

2791
	cr4 = ctxt->ops->get_cr(ctxt, 4);
2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

	if (check_dr7_gd(ctxt))
		return emulate_db(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;
	u64 new_val = c->src.val64;
	int dr = c->modrm_reg;

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

2813 2814 2815 2816
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
	u64 efer;

2817
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2818 2819 2820 2821 2822 2823 2824 2825 2826

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
2827
	u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2828 2829

	/* Valid physical address? */
2830
	if (rax & 0xffff000000000000ULL)
2831 2832 2833 2834 2835
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

2836 2837
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
2838
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2839

2840
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2841 2842 2843 2844 2845
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

2846 2847
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
2848
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2849
	u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
2850

2851
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2852 2853 2854 2855 2856 2857
	    (rcx > 3))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;

	c->dst.bytes = min(c->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;

	c->src.bytes = min(c->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

2880
#define D(_y) { .flags = (_y) }
2881
#define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2882 2883
#define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
		      .check_perm = (_p) }
2884
#define N    D(0)
2885
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2886 2887 2888
#define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
#define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2889 2890
#define II(_f, _e, _i) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2891 2892 2893
#define IIP(_f, _e, _i, _p) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
	  .check_perm = (_p) }
2894
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2895

2896
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
2897
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2898 2899
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)

2900 2901 2902 2903
#define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM),			\
		D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock),		\
		D2bv(((_f) & ~Lock) | DstAcc | SrcImm)

2904 2905 2906 2907 2908 2909
static struct opcode group7_rm1[] = {
	DI(SrcNone | ModRM | Priv, monitor),
	DI(SrcNone | ModRM | Priv, mwait),
	N, N, N, N, N, N,
};

2910 2911
static struct opcode group7_rm3[] = {
	DIP(SrcNone | ModRM | Prot | Priv, vmrun,   check_svme_pa),
2912
	DI(SrcNone | ModRM | Prot | VendorSpecific, vmmcall),
2913 2914 2915 2916 2917 2918 2919
	DIP(SrcNone | ModRM | Prot | Priv, vmload,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, vmsave,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, stgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, clgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, skinit,  check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
};
2920

2921 2922 2923 2924 2925
static struct opcode group7_rm7[] = {
	N,
	DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
	N, N, N, N, N, N,
};
2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936
static struct opcode group1[] = {
	X7(D(Lock)), N
};

static struct opcode group1A[] = {
	D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
};

static struct opcode group3[] = {
	D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
	D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2937
	X4(D(SrcMem | ModRM)),
2938 2939 2940 2941 2942 2943 2944 2945 2946
};

static struct opcode group4[] = {
	D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
	N, N, N, N, N, N,
};

static struct opcode group5[] = {
	D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2947 2948
	D(SrcMem | ModRM | Stack),
	I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2949 2950 2951 2952
	D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
	D(SrcMem | ModRM | Stack), N,
};

2953 2954 2955 2956 2957 2958 2959 2960
static struct opcode group6[] = {
	DI(ModRM | Prot,        sldt),
	DI(ModRM | Prot,        str),
	DI(ModRM | Prot | Priv, lldt),
	DI(ModRM | Prot | Priv, ltr),
	N, N, N, N,
};

2961
static struct group_dual group7 = { {
2962 2963 2964
	DI(ModRM | Mov | DstMem | Priv, sgdt),
	DI(ModRM | Mov | DstMem | Priv, sidt),
	DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2965 2966 2967
	DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
	DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
	DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2968
}, {
2969
	D(SrcNone | ModRM | Priv | VendorSpecific), EXT(0, group7_rm1),
2970
	N, EXT(0, group7_rm3),
2971
	DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2972
	DI(SrcMem16 | ModRM | Mov | Priv, lmsw), EXT(0, group7_rm7),
2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986
} };

static struct opcode group8[] = {
	N, N, N, N,
	D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
	D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
};

static struct group_dual group9 = { {
	N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
}, {
	N, N, N, N, N, N, N, N,
} };

2987 2988 2989 2990
static struct opcode group11[] = {
	I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
};

2991 2992 2993 2994
static struct gprefix pfx_0f_6f_0f_7f = {
	N, N, N, I(Sse, em_movdqu),
};

2995 2996
static struct opcode opcode_table[256] = {
	/* 0x00 - 0x07 */
2997
	D6ALU(Lock),
2998 2999
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	/* 0x08 - 0x0F */
3000
	D6ALU(Lock),
3001 3002
	D(ImplicitOps | Stack | No64), N,
	/* 0x10 - 0x17 */
3003
	D6ALU(Lock),
3004 3005
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	/* 0x18 - 0x1F */
3006
	D6ALU(Lock),
3007 3008
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	/* 0x20 - 0x27 */
3009
	D6ALU(Lock), N, N,
3010
	/* 0x28 - 0x2F */
3011
	D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
3012
	/* 0x30 - 0x37 */
3013
	D6ALU(Lock), N, N,
3014
	/* 0x38 - 0x3F */
3015
	D6ALU(0), N, N,
3016 3017 3018
	/* 0x40 - 0x4F */
	X16(D(DstReg)),
	/* 0x50 - 0x57 */
3019
	X8(I(SrcReg | Stack, em_push)),
3020 3021 3022 3023 3024 3025 3026
	/* 0x58 - 0x5F */
	X8(D(DstReg | Stack)),
	/* 0x60 - 0x67 */
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
	N, N, N, N,
	/* 0x68 - 0x6F */
3027 3028
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3029 3030
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3031 3032
	D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
	D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
3033 3034 3035 3036 3037 3038 3039
	/* 0x70 - 0x7F */
	X16(D(SrcImmByte)),
	/* 0x80 - 0x87 */
	G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
	G(DstMem | SrcImm | ModRM | Group, group1),
	G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
	G(DstMem | SrcImmByte | ModRM | Group, group1),
3040
	D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
3041
	/* 0x88 - 0x8F */
3042 3043
	I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3044
	D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3045 3046
	D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
	/* 0x90 - 0x97 */
3047
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3048
	/* 0x98 - 0x9F */
3049
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3050
	I(SrcImmFAddr | No64, em_call_far), N,
3051
	DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
3052
	/* 0xA0 - 0xA7 */
3053 3054 3055 3056
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
	I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
	I2bv(SrcSI | DstDI | Mov | String, em_mov),
	D2bv(SrcSI | DstDI | String),
3057
	/* 0xA8 - 0xAF */
3058
	D2bv(DstAcc | SrcImm),
3059 3060
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3061
	D2bv(SrcAcc | DstDI | String),
3062
	/* 0xB0 - 0xB7 */
3063
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3064
	/* 0xB8 - 0xBF */
3065
	X8(I(DstReg | SrcImm | Mov, em_mov)),
3066
	/* 0xC0 - 0xC7 */
3067
	D2bv(DstMem | SrcImmByte | ModRM),
3068 3069
	I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
	D(ImplicitOps | Stack),
3070
	D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3071
	G(ByteOp, group11), G(0, group11),
3072 3073
	/* 0xC8 - 0xCF */
	N, N, N, D(ImplicitOps | Stack),
3074 3075
	D(ImplicitOps), DI(SrcImmByte, intn),
	D(ImplicitOps | No64), DI(ImplicitOps, iret),
3076
	/* 0xD0 - 0xD7 */
3077
	D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3078 3079 3080 3081
	N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
3082
	X4(D(SrcImmByte)),
3083 3084
	D2bvIP(SrcImmUByte | DstAcc, in,  check_perm_in),
	D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3085 3086 3087
	/* 0xE8 - 0xEF */
	D(SrcImm | Stack), D(SrcImm | ImplicitOps),
	D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
3088 3089
	D2bvIP(SrcNone | DstAcc,     in,  check_perm_in),
	D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
3090
	/* 0xF0 - 0xF7 */
3091
	N, DI(ImplicitOps, icebp), N, N,
3092 3093
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
3094
	/* 0xF8 - 0xFF */
3095
	D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3096 3097 3098 3099 3100
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

static struct opcode twobyte_table[256] = {
	/* 0x00 - 0x0F */
3101
	G(0, group6), GD(0, &group7), N, N,
3102
	N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3103
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3104 3105 3106 3107
	N, D(ImplicitOps | ModRM), N, N,
	/* 0x10 - 0x1F */
	N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
	/* 0x20 - 0x2F */
3108
	DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3109
	DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3110
	DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3111
	DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3112 3113 3114
	N, N, N, N,
	N, N, N, N, N, N, N, N,
	/* 0x30 - 0x3F */
3115 3116 3117 3118
	DI(ImplicitOps | Priv, wrmsr),
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
	DI(ImplicitOps | Priv, rdmsr),
	DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3119 3120
	D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
	N, N,
3121 3122 3123 3124 3125 3126
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
	X16(D(DstReg | SrcMem | ModRM | Mov)),
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
3127 3128 3129 3130
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3131
	/* 0x70 - 0x7F */
3132 3133 3134 3135
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3136 3137 3138
	/* 0x80 - 0x8F */
	X16(D(SrcImm)),
	/* 0x90 - 0x9F */
3139
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3140 3141
	/* 0xA0 - 0xA7 */
	D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3142
	DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3143 3144 3145 3146
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM), N, N,
	/* 0xA8 - 0xAF */
	D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3147
	DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3148 3149
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM),
3150
	D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3151
	/* 0xB0 - 0xB7 */
3152
	D2bv(DstMem | SrcReg | ModRM | Lock),
3153 3154 3155
	D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
	D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3156 3157
	/* 0xB8 - 0xBF */
	N, N,
3158
	G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3159 3160
	D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3161
	/* 0xC0 - 0xCF */
3162
	D2bv(DstMem | SrcReg | ModRM | Lock),
3163
	N, D(DstMem | SrcReg | ModRM | Mov),
3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178
	N, N, N, GD(0, &group9),
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

#undef D
#undef N
#undef G
#undef GD
#undef I
3179
#undef GP
3180
#undef EXT
3181

3182
#undef D2bv
3183
#undef D2bvIP
3184
#undef I2bv
3185
#undef D6ALU
3186

3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205
static unsigned imm_size(struct decode_cache *c)
{
	unsigned size;

	size = (c->d & ByteOp) ? 1 : c->op_bytes;
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	struct decode_cache *c = &ctxt->decode;
	struct x86_emulate_ops *ops = ctxt->ops;
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
3206
	op->addr.mem.ea = c->eip;
3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
		op->val = insn_fetch(s8, 1, c->eip);
		break;
	case 2:
		op->val = insn_fetch(s16, 2, c->eip);
		break;
	case 4:
		op->val = insn_fetch(s32, 4, c->eip);
		break;
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

3236
int
3237
x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3238 3239 3240 3241 3242
{
	struct x86_emulate_ops *ops = ctxt->ops;
	struct decode_cache *c = &ctxt->decode;
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
3243 3244
	int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
	bool op_prefix = false;
3245
	struct opcode opcode, *g_mod012, *g_mod3;
3246
	struct operand memop = { .type = OP_NONE };
3247 3248

	c->eip = ctxt->eip;
3249 3250 3251 3252
	c->fetch.start = c->eip;
	c->fetch.end = c->fetch.start + insn_len;
	if (insn_len > 0)
		memcpy(c->fetch.data, insn, insn_len);
3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
		return -1;
	}

	c->op_bytes = def_op_bytes;
	c->ad_bytes = def_ad_bytes;

	/* Legacy prefixes. */
	for (;;) {
		switch (c->b = insn_fetch(u8, 1, c->eip)) {
		case 0x66:	/* operand-size override */
3280
			op_prefix = true;
3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311
			/* switch between 2/4 bytes */
			c->op_bytes = def_op_bytes ^ 6;
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
				c->ad_bytes = def_ad_bytes ^ 12;
			else
				/* switch between 2/4 bytes */
				c->ad_bytes = def_ad_bytes ^ 6;
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
			set_seg_override(c, (c->b >> 3) & 3);
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
			set_seg_override(c, c->b & 7);
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
			c->rex_prefix = c->b;
			continue;
		case 0xf0:	/* LOCK */
			c->lock_prefix = 1;
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
3312
			c->rep_prefix = c->b;
3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

		c->rex_prefix = 0;
	}

done_prefixes:

	/* REX prefix. */
3326 3327
	if (c->rex_prefix & 8)
		c->op_bytes = 8;	/* REX.W */
3328 3329 3330

	/* Opcode byte(s). */
	opcode = opcode_table[c->b];
3331 3332 3333 3334 3335
	/* Two-byte opcode? */
	if (c->b == 0x0f) {
		c->twobyte = 1;
		c->b = insn_fetch(u8, 1, c->eip);
		opcode = twobyte_table[c->b];
3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357
	}
	c->d = opcode.flags;

	if (c->d & Group) {
		dual = c->d & GroupDual;
		c->modrm = insn_fetch(u8, 1, c->eip);
		--c->eip;

		if (c->d & GroupDual) {
			g_mod012 = opcode.u.gdual->mod012;
			g_mod3 = opcode.u.gdual->mod3;
		} else
			g_mod012 = g_mod3 = opcode.u.group;

		c->d &= ~(Group | GroupDual);

		goffset = (c->modrm >> 3) & 7;

		if ((c->modrm >> 6) == 3)
			opcode = g_mod3[goffset];
		else
			opcode = g_mod012[goffset];
3358 3359 3360 3361 3362 3363

		if (opcode.flags & RMExt) {
			goffset = c->modrm & 7;
			opcode = opcode.u.group[goffset];
		}

3364 3365 3366
		c->d |= opcode.flags;
	}

3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379
	if (c->d & Prefix) {
		if (c->rep_prefix && op_prefix)
			return X86EMUL_UNHANDLEABLE;
		simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
		switch (simd_prefix) {
		case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
		case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
		case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
		case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
		}
		c->d |= opcode.flags;
	}

3380
	c->execute = opcode.u.execute;
3381
	c->check_perm = opcode.check_perm;
3382
	c->intercept = opcode.intercept;
3383 3384

	/* Unrecognised? */
A
Avi Kivity 已提交
3385
	if (c->d == 0 || (c->d & Undefined))
3386 3387
		return -1;

3388 3389 3390
	if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
		return -1;

3391 3392 3393
	if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
		c->op_bytes = 8;

3394 3395 3396 3397 3398 3399 3400
	if (c->d & Op3264) {
		if (mode == X86EMUL_MODE_PROT64)
			c->op_bytes = 8;
		else
			c->op_bytes = 4;
	}

A
Avi Kivity 已提交
3401 3402 3403
	if (c->d & Sse)
		c->op_bytes = 16;

3404
	/* ModRM and SIB bytes. */
3405
	if (c->d & ModRM) {
3406
		rc = decode_modrm(ctxt, ops, &memop);
3407 3408 3409
		if (!c->has_seg_override)
			set_seg_override(c, c->modrm_seg);
	} else if (c->d & MemAbs)
3410
		rc = decode_abs(ctxt, ops, &memop);
3411 3412 3413 3414 3415 3416
	if (rc != X86EMUL_CONTINUE)
		goto done;

	if (!c->has_seg_override)
		set_seg_override(c, VCPU_SREG_DS);

3417
	memop.addr.mem.seg = seg_override(ctxt, ops, c);
3418

3419
	if (memop.type == OP_MEM && c->ad_bytes != 8)
3420
		memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3421

3422
	if (memop.type == OP_MEM && c->rip_relative)
3423
		memop.addr.mem.ea += c->eip;
3424 3425 3426 3427 3428 3429 3430 3431 3432

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
	switch (c->d & SrcMask) {
	case SrcNone:
		break;
	case SrcReg:
A
Avi Kivity 已提交
3433
		decode_register_operand(ctxt, &c->src, c, 0);
3434 3435
		break;
	case SrcMem16:
3436
		memop.bytes = 2;
3437 3438
		goto srcmem_common;
	case SrcMem32:
3439
		memop.bytes = 4;
3440 3441
		goto srcmem_common;
	case SrcMem:
3442
		memop.bytes = (c->d & ByteOp) ? 1 :
3443 3444
							   c->op_bytes;
	srcmem_common:
3445
		c->src = memop;
3446
		break;
3447
	case SrcImmU16:
3448 3449
		rc = decode_imm(ctxt, &c->src, 2, false);
		break;
3450
	case SrcImm:
3451 3452
		rc = decode_imm(ctxt, &c->src, imm_size(c), true);
		break;
3453
	case SrcImmU:
3454
		rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3455 3456
		break;
	case SrcImmByte:
3457 3458
		rc = decode_imm(ctxt, &c->src, 1, true);
		break;
3459
	case SrcImmUByte:
3460
		rc = decode_imm(ctxt, &c->src, 1, false);
3461 3462 3463 3464
		break;
	case SrcAcc:
		c->src.type = OP_REG;
		c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3465
		c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3466
		fetch_register_operand(&c->src);
3467 3468 3469 3470 3471 3472 3473 3474
		break;
	case SrcOne:
		c->src.bytes = 1;
		c->src.val = 1;
		break;
	case SrcSI:
		c->src.type = OP_MEM;
		c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3475 3476 3477
		c->src.addr.mem.ea =
			register_address(c, c->regs[VCPU_REGS_RSI]);
		c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3478 3479 3480 3481
		c->src.val = 0;
		break;
	case SrcImmFAddr:
		c->src.type = OP_IMM;
3482
		c->src.addr.mem.ea = c->eip;
3483 3484 3485 3486
		c->src.bytes = c->op_bytes + 2;
		insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
		break;
	case SrcMemFAddr:
3487 3488
		memop.bytes = c->op_bytes + 2;
		goto srcmem_common;
3489 3490 3491
		break;
	}

3492 3493 3494
	if (rc != X86EMUL_CONTINUE)
		goto done;

3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
	switch (c->d & Src2Mask) {
	case Src2None:
		break;
	case Src2CL:
		c->src2.bytes = 1;
		c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
		break;
	case Src2ImmByte:
3507
		rc = decode_imm(ctxt, &c->src2, 1, true);
3508 3509 3510 3511 3512
		break;
	case Src2One:
		c->src2.bytes = 1;
		c->src2.val = 1;
		break;
3513 3514 3515
	case Src2Imm:
		rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
		break;
3516 3517
	}

3518 3519 3520
	if (rc != X86EMUL_CONTINUE)
		goto done;

3521 3522 3523
	/* Decode and fetch the destination operand: register or memory. */
	switch (c->d & DstMask) {
	case DstReg:
A
Avi Kivity 已提交
3524
		decode_register_operand(ctxt, &c->dst, c,
3525 3526
			 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
		break;
3527 3528
	case DstImmUByte:
		c->dst.type = OP_IMM;
3529
		c->dst.addr.mem.ea = c->eip;
3530 3531 3532
		c->dst.bytes = 1;
		c->dst.val = insn_fetch(u8, 1, c->eip);
		break;
3533 3534
	case DstMem:
	case DstMem64:
3535
		c->dst = memop;
3536 3537 3538 3539
		if ((c->d & DstMask) == DstMem64)
			c->dst.bytes = 8;
		else
			c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3540 3541
		if (c->d & BitOp)
			fetch_bit_operand(c);
3542
		c->dst.orig_val = c->dst.val;
3543 3544 3545 3546
		break;
	case DstAcc:
		c->dst.type = OP_REG;
		c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3547
		c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3548
		fetch_register_operand(&c->dst);
3549 3550 3551 3552 3553
		c->dst.orig_val = c->dst.val;
		break;
	case DstDI:
		c->dst.type = OP_MEM;
		c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3554 3555 3556
		c->dst.addr.mem.ea =
			register_address(c, c->regs[VCPU_REGS_RDI]);
		c->dst.addr.mem.seg = VCPU_SREG_ES;
3557 3558
		c->dst.val = 0;
		break;
3559 3560 3561 3562 3563
	case ImplicitOps:
		/* Special instructions do their own operand decoding. */
	default:
		c->dst.type = OP_NONE; /* Disable writeback. */
		return 0;
3564 3565 3566
	}

done:
3567
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3568 3569
}

3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	struct decode_cache *c = &ctxt->decode;

	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
	if (((c->b == 0xa6) || (c->b == 0xa7) ||
	     (c->b == 0xae) || (c->b == 0xaf))
	    && (((c->rep_prefix == REPE_PREFIX) &&
		 ((ctxt->eflags & EFLG_ZF) == 0))
		|| ((c->rep_prefix == REPNE_PREFIX) &&
		    ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
		return true;

	return false;
}

3592
int
3593
x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3594
{
3595
	struct x86_emulate_ops *ops = ctxt->ops;
3596 3597
	u64 msr_data;
	struct decode_cache *c = &ctxt->decode;
3598
	int rc = X86EMUL_CONTINUE;
3599
	int saved_dst_type = c->dst.type;
3600
	int irq; /* Used for int 3, int, and into */
3601

3602
	ctxt->decode.mem_read.pos = 0;
3603

3604
	if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3605
		rc = emulate_ud(ctxt);
3606 3607 3608
		goto done;
	}

3609
	/* LOCK prefix is allowed only with some instructions */
3610
	if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3611
		rc = emulate_ud(ctxt);
3612 3613 3614
		goto done;
	}

3615
	if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3616
		rc = emulate_ud(ctxt);
3617 3618 3619
		goto done;
	}

A
Avi Kivity 已提交
3620
	if ((c->d & Sse)
3621 3622
	    && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
		|| !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
A
Avi Kivity 已提交
3623 3624 3625 3626
		rc = emulate_ud(ctxt);
		goto done;
	}

3627
	if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
A
Avi Kivity 已提交
3628 3629 3630 3631
		rc = emulate_nm(ctxt);
		goto done;
	}

3632
	if (unlikely(ctxt->guest_mode) && c->intercept) {
3633 3634
		rc = emulator_check_intercept(ctxt, c->intercept,
					      X86_ICPT_PRE_EXCEPT);
3635 3636 3637 3638
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3639
	/* Privileged instruction can be executed only in CPL=0 */
3640
	if ((c->d & Priv) && ops->cpl(ctxt)) {
3641
		rc = emulate_gp(ctxt, 0);
3642 3643 3644
		goto done;
	}

3645 3646 3647 3648 3649 3650
	/* Instruction can only be executed in protected mode */
	if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
		rc = emulate_ud(ctxt);
		goto done;
	}

3651 3652 3653 3654 3655 3656 3657
	/* Do instruction specific permission checks */
	if (c->check_perm) {
		rc = c->check_perm(ctxt);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3658
	if (unlikely(ctxt->guest_mode) && c->intercept) {
3659 3660
		rc = emulator_check_intercept(ctxt, c->intercept,
					      X86_ICPT_POST_EXCEPT);
3661 3662 3663 3664
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3665 3666
	if (c->rep_prefix && (c->d & String)) {
		/* All REP prefixes have the same first termination condition */
3667
		if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3668
			ctxt->eip = c->eip;
3669 3670 3671 3672
			goto done;
		}
	}

3673
	if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3674 3675
		rc = segmented_read(ctxt, c->src.addr.mem,
				    c->src.valptr, c->src.bytes);
3676
		if (rc != X86EMUL_CONTINUE)
3677
			goto done;
3678
		c->src.orig_val64 = c->src.val64;
3679 3680
	}

3681
	if (c->src2.type == OP_MEM) {
3682 3683
		rc = segmented_read(ctxt, c->src2.addr.mem,
				    &c->src2.val, c->src2.bytes);
3684 3685 3686 3687
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3688 3689 3690 3691
	if ((c->d & DstMask) == ImplicitOps)
		goto special_insn;


3692 3693
	if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
		/* optimisation - avoid slow emulated read if Mov */
3694
		rc = segmented_read(ctxt, c->dst.addr.mem,
3695
				   &c->dst.val, c->dst.bytes);
3696 3697
		if (rc != X86EMUL_CONTINUE)
			goto done;
3698
	}
3699
	c->dst.orig_val = c->dst.val;
3700

3701 3702
special_insn:

3703
	if (unlikely(ctxt->guest_mode) && c->intercept) {
3704 3705
		rc = emulator_check_intercept(ctxt, c->intercept,
					      X86_ICPT_POST_MEMACCESS);
3706 3707 3708 3709
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3710 3711 3712 3713 3714 3715 3716
	if (c->execute) {
		rc = c->execute(ctxt);
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

3717
	if (c->twobyte)
A
Avi Kivity 已提交
3718 3719
		goto twobyte_insn;

3720
	switch (c->b) {
A
Avi Kivity 已提交
3721 3722
	case 0x00 ... 0x05:
	      add:		/* add */
3723
		emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
3724
		break;
3725
	case 0x06:		/* push es */
3726
		rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3727 3728 3729 3730
		break;
	case 0x07:		/* pop es */
		rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
		break;
A
Avi Kivity 已提交
3731 3732
	case 0x08 ... 0x0d:
	      or:		/* or */
3733
		emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
3734
		break;
3735
	case 0x0e:		/* push cs */
3736
		rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3737
		break;
A
Avi Kivity 已提交
3738 3739
	case 0x10 ... 0x15:
	      adc:		/* adc */
3740
		emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
3741
		break;
3742
	case 0x16:		/* push ss */
3743
		rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3744 3745 3746 3747
		break;
	case 0x17:		/* pop ss */
		rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
		break;
A
Avi Kivity 已提交
3748 3749
	case 0x18 ... 0x1d:
	      sbb:		/* sbb */
3750
		emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
3751
		break;
3752
	case 0x1e:		/* push ds */
3753
		rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3754 3755 3756 3757
		break;
	case 0x1f:		/* pop ds */
		rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
		break;
3758
	case 0x20 ... 0x25:
A
Avi Kivity 已提交
3759
	      and:		/* and */
3760
		emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
3761 3762 3763
		break;
	case 0x28 ... 0x2d:
	      sub:		/* sub */
3764
		emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
3765 3766 3767
		break;
	case 0x30 ... 0x35:
	      xor:		/* xor */
3768
		emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
3769 3770 3771
		break;
	case 0x38 ... 0x3d:
	      cmp:		/* cmp */
3772
		c->dst.type = OP_NONE; /* Disable writeback. */
3773
		emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
3774
		break;
3775 3776 3777 3778 3779 3780 3781 3782
	case 0x40 ... 0x47: /* inc r16/r32 */
		emulate_1op("inc", c->dst, ctxt->eflags);
		break;
	case 0x48 ... 0x4f: /* dec r16/r32 */
		emulate_1op("dec", c->dst, ctxt->eflags);
		break;
	case 0x58 ... 0x5f: /* pop reg */
	pop_instruction:
3783
		rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3784
		break;
3785
	case 0x60:	/* pusha */
3786
		rc = emulate_pusha(ctxt);
3787 3788 3789 3790
		break;
	case 0x61:	/* popa */
		rc = emulate_popa(ctxt, ops);
		break;
A
Avi Kivity 已提交
3791
	case 0x63:		/* movsxd */
3792
		if (ctxt->mode != X86EMUL_MODE_PROT64)
A
Avi Kivity 已提交
3793
			goto cannot_emulate;
3794
		c->dst.val = (s32) c->src.val;
A
Avi Kivity 已提交
3795
		break;
3796 3797
	case 0x6c:		/* insb */
	case 0x6d:		/* insw/insd */
3798 3799
		c->src.val = c->regs[VCPU_REGS_RDX];
		goto do_io_in;
3800 3801
	case 0x6e:		/* outsb */
	case 0x6f:		/* outsw/outsd */
3802 3803
		c->dst.val = c->regs[VCPU_REGS_RDX];
		goto do_io_out;
3804
		break;
3805
	case 0x70 ... 0x7f: /* jcc (short) */
3806
		if (test_cc(c->b, ctxt->eflags))
3807
			jmp_rel(c, c->src.val);
3808
		break;
A
Avi Kivity 已提交
3809
	case 0x80 ... 0x83:	/* Grp1 */
3810
		switch (c->modrm_reg) {
A
Avi Kivity 已提交
3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829
		case 0:
			goto add;
		case 1:
			goto or;
		case 2:
			goto adc;
		case 3:
			goto sbb;
		case 4:
			goto and;
		case 5:
			goto sub;
		case 6:
			goto xor;
		case 7:
			goto cmp;
		}
		break;
	case 0x84 ... 0x85:
3830
	test:
3831
		emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
3832 3833
		break;
	case 0x86 ... 0x87:	/* xchg */
3834
	xchg:
A
Avi Kivity 已提交
3835
		/* Write back the register source. */
3836 3837
		c->src.val = c->dst.val;
		write_register_operand(&c->src);
A
Avi Kivity 已提交
3838 3839 3840 3841
		/*
		 * Write back the memory destination with implicit LOCK
		 * prefix.
		 */
3842
		c->dst.val = c->src.orig_val;
3843
		c->lock_prefix = 1;
A
Avi Kivity 已提交
3844
		break;
3845 3846
	case 0x8c:  /* mov r/m, sreg */
		if (c->modrm_reg > VCPU_SREG_GS) {
3847
			rc = emulate_ud(ctxt);
3848
			goto done;
3849
		}
3850
		c->dst.val = ops->get_segment_selector(ctxt, c->modrm_reg);
3851
		break;
N
Nitin A Kamble 已提交
3852
	case 0x8d: /* lea r16/r32, m */
3853
		c->dst.val = c->src.addr.mem.ea;
N
Nitin A Kamble 已提交
3854
		break;
3855 3856 3857 3858
	case 0x8e: { /* mov seg, r/m16 */
		uint16_t sel;

		sel = c->src.val;
3859

3860 3861
		if (c->modrm_reg == VCPU_SREG_CS ||
		    c->modrm_reg > VCPU_SREG_GS) {
3862
			rc = emulate_ud(ctxt);
3863 3864 3865
			goto done;
		}

3866
		if (c->modrm_reg == VCPU_SREG_SS)
3867
			ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3868

3869
		rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3870 3871 3872 3873

		c->dst.type = OP_NONE;  /* Disable writeback. */
		break;
	}
A
Avi Kivity 已提交
3874
	case 0x8f:		/* pop (sole member of Grp1a) */
3875
		rc = emulate_grp1a(ctxt, ops);
A
Avi Kivity 已提交
3876
		break;
3877 3878
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
		if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3879
			break;
3880
		goto xchg;
3881 3882 3883 3884 3885 3886 3887
	case 0x98: /* cbw/cwde/cdqe */
		switch (c->op_bytes) {
		case 2: c->dst.val = (s8)c->dst.val; break;
		case 4: c->dst.val = (s16)c->dst.val; break;
		case 8: c->dst.val = (s32)c->dst.val; break;
		}
		break;
N
Nitin A Kamble 已提交
3888
	case 0x9c: /* pushf */
3889
		c->src.val =  (unsigned long) ctxt->eflags;
3890
		rc = em_push(ctxt);
3891
		break;
N
Nitin A Kamble 已提交
3892
	case 0x9d: /* popf */
A
Avi Kivity 已提交
3893
		c->dst.type = OP_REG;
3894
		c->dst.addr.reg = &ctxt->eflags;
A
Avi Kivity 已提交
3895
		c->dst.bytes = c->op_bytes;
3896 3897
		rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
		break;
A
Avi Kivity 已提交
3898
	case 0xa6 ... 0xa7:	/* cmps */
3899
		goto cmp;
3900 3901
	case 0xa8 ... 0xa9:	/* test ax, imm */
		goto test;
A
Avi Kivity 已提交
3902
	case 0xae ... 0xaf:	/* scas */
3903
		goto cmp;
3904 3905 3906
	case 0xc0 ... 0xc1:
		emulate_grp2(ctxt);
		break;
3907
	case 0xc3: /* ret */
A
Avi Kivity 已提交
3908
		c->dst.type = OP_REG;
3909
		c->dst.addr.reg = &c->eip;
A
Avi Kivity 已提交
3910
		c->dst.bytes = c->op_bytes;
3911
		goto pop_instruction;
3912 3913 3914 3915 3916 3917
	case 0xc4:		/* les */
		rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
		break;
	case 0xc5:		/* lds */
		rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
		break;
3918 3919
	case 0xcb:		/* ret far */
		rc = emulate_ret_far(ctxt, ops);
3920
		break;
3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934
	case 0xcc:		/* int3 */
		irq = 3;
		goto do_interrupt;
	case 0xcd:		/* int n */
		irq = c->src.val;
	do_interrupt:
		rc = emulate_int(ctxt, ops, irq);
		break;
	case 0xce:		/* into */
		if (ctxt->eflags & EFLG_OF) {
			irq = 4;
			goto do_interrupt;
		}
		break;
3935 3936
	case 0xcf:		/* iret */
		rc = emulate_iret(ctxt, ops);
3937
		break;
3938 3939 3940 3941 3942 3943 3944
	case 0xd0 ... 0xd1:	/* Grp2 */
		emulate_grp2(ctxt);
		break;
	case 0xd2 ... 0xd3:	/* Grp2 */
		c->src.val = c->regs[VCPU_REGS_RCX];
		emulate_grp2(ctxt);
		break;
3945 3946 3947 3948 3949 3950
	case 0xe0 ... 0xe2:	/* loop/loopz/loopnz */
		register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
		if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
		    (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
			jmp_rel(c, c->src.val);
		break;
3951 3952 3953 3954
	case 0xe3:	/* jcxz/jecxz/jrcxz */
		if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
			jmp_rel(c, c->src.val);
		break;
3955 3956
	case 0xe4: 	/* inb */
	case 0xe5: 	/* in */
3957
		goto do_io_in;
3958 3959
	case 0xe6: /* outb */
	case 0xe7: /* out */
3960
		goto do_io_out;
3961
	case 0xe8: /* call (near) */ {
3962
		long int rel = c->src.val;
3963
		c->src.val = (unsigned long) c->eip;
3964
		jmp_rel(c, rel);
3965
		rc = em_push(ctxt);
3966
		break;
3967 3968
	}
	case 0xe9: /* jmp rel */
3969
		goto jmp;
3970 3971
	case 0xea: { /* jmp far */
		unsigned short sel;
3972
	jump_far:
3973 3974 3975
		memcpy(&sel, c->src.valptr + c->op_bytes, 2);

		if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3976
			goto done;
3977

3978 3979
		c->eip = 0;
		memcpy(&c->eip, c->src.valptr, c->op_bytes);
3980
		break;
3981
	}
3982 3983
	case 0xeb:
	      jmp:		/* jmp rel short */
3984
		jmp_rel(c, c->src.val);
3985
		c->dst.type = OP_NONE; /* Disable writeback. */
3986
		break;
3987 3988
	case 0xec: /* in al,dx */
	case 0xed: /* in (e/r)ax,dx */
3989 3990
		c->src.val = c->regs[VCPU_REGS_RDX];
	do_io_in:
3991 3992
		if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
				     &c->dst.val))
3993 3994
			goto done; /* IO is needed */
		break;
3995 3996
	case 0xee: /* out dx,al */
	case 0xef: /* out dx,(e/r)ax */
3997
		c->dst.val = c->regs[VCPU_REGS_RDX];
3998
	do_io_out:
3999 4000
		ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
				      &c->src.val, 1);
4001
		c->dst.type = OP_NONE;	/* Disable writeback. */
4002
		break;
4003
	case 0xf4:              /* hlt */
4004
		ctxt->ops->halt(ctxt);
4005
		break;
4006 4007 4008 4009
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
		ctxt->eflags ^= EFLG_CF;
		break;
4010
	case 0xf6 ... 0xf7:	/* Grp3 */
4011
		rc = emulate_grp3(ctxt, ops);
4012
		break;
4013 4014 4015
	case 0xf8: /* clc */
		ctxt->eflags &= ~EFLG_CF;
		break;
4016 4017 4018
	case 0xf9: /* stc */
		ctxt->eflags |= EFLG_CF;
		break;
4019
	case 0xfa: /* cli */
4020
		if (emulator_bad_iopl(ctxt, ops)) {
4021
			rc = emulate_gp(ctxt, 0);
4022
			goto done;
4023
		} else
4024
			ctxt->eflags &= ~X86_EFLAGS_IF;
4025 4026
		break;
	case 0xfb: /* sti */
4027
		if (emulator_bad_iopl(ctxt, ops)) {
4028
			rc = emulate_gp(ctxt, 0);
4029 4030
			goto done;
		} else {
4031
			ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4032 4033
			ctxt->eflags |= X86_EFLAGS_IF;
		}
4034
		break;
4035 4036 4037 4038 4039 4040
	case 0xfc: /* cld */
		ctxt->eflags &= ~EFLG_DF;
		break;
	case 0xfd: /* std */
		ctxt->eflags |= EFLG_DF;
		break;
4041 4042
	case 0xfe: /* Grp4 */
	grp45:
4043
		rc = emulate_grp45(ctxt);
4044
		break;
4045 4046 4047 4048
	case 0xff: /* Grp5 */
		if (c->modrm_reg == 5)
			goto jump_far;
		goto grp45;
4049 4050
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4051
	}
4052

4053 4054 4055
	if (rc != X86EMUL_CONTINUE)
		goto done;

4056 4057
writeback:
	rc = writeback(ctxt, ops);
4058
	if (rc != X86EMUL_CONTINUE)
4059 4060
		goto done;

4061 4062 4063 4064 4065 4066
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
	c->dst.type = saved_dst_type;

4067
	if ((c->d & SrcMask) == SrcSI)
4068
		string_addr_inc(ctxt, seg_override(ctxt, ops, c),
4069
				VCPU_REGS_RSI, &c->src);
4070 4071

	if ((c->d & DstMask) == DstDI)
4072
		string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4073
				&c->dst);
4074

4075
	if (c->rep_prefix && (c->d & String)) {
4076
		struct read_cache *r = &ctxt->decode.io_read;
4077
		register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4078

4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
			if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
				ctxt->decode.mem_read.end = 0;
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
4095
		}
4096
	}
4097 4098

	ctxt->eip = c->eip;
4099 4100

done:
4101 4102
	if (rc == X86EMUL_PROPAGATE_FAULT)
		ctxt->have_exception = true;
4103 4104 4105
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

4106
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
4107 4108

twobyte_insn:
4109
	switch (c->b) {
A
Avi Kivity 已提交
4110
	case 0x01: /* lgdt, lidt, lmsw */
4111
		switch (c->modrm_reg) {
4112
		case 0: /* vmcall */
4113
			rc = em_vmcall(ctxt);
4114
			break;
A
Avi Kivity 已提交
4115
		case 2: /* lgdt */
4116
			rc = em_lgdt(ctxt);
A
Avi Kivity 已提交
4117
			break;
4118
		case 3: /* lidt/vmmcall */
4119 4120 4121 4122
			if (c->modrm_mod == 3)
				return em_svm(ctxt);
			else
				return em_lidt(ctxt);
A
Avi Kivity 已提交
4123 4124
			break;
		case 4: /* smsw */
4125
			rc = em_smsw(ctxt);
A
Avi Kivity 已提交
4126 4127
			break;
		case 6: /* lmsw */
4128
			rc = em_lmsw(ctxt);
A
Avi Kivity 已提交
4129 4130
			break;
		case 7: /* invlpg*/
4131
			rc = em_invlpg(ctxt);
A
Avi Kivity 已提交
4132 4133 4134 4135 4136
			break;
		default:
			goto cannot_emulate;
		}
		break;
4137
	case 0x05: 		/* syscall */
4138
		rc = emulate_syscall(ctxt, ops);
4139
		break;
4140
	case 0x06:
4141
		rc = em_clts(ctxt);
4142 4143
		break;
	case 0x09:		/* wbinvd */
4144
		(ctxt->ops->wbinvd)(ctxt);
4145 4146
		break;
	case 0x08:		/* invd */
4147 4148 4149 4150
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
		break;
	case 0x20: /* mov cr, reg */
4151
		c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4152
		break;
A
Avi Kivity 已提交
4153
	case 0x21: /* mov from dr to reg */
4154
		ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
A
Avi Kivity 已提交
4155
		break;
4156
	case 0x22: /* mov reg, cr */
4157
		if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4158
			emulate_gp(ctxt, 0);
4159
			rc = X86EMUL_PROPAGATE_FAULT;
4160 4161
			goto done;
		}
4162 4163
		c->dst.type = OP_NONE;
		break;
A
Avi Kivity 已提交
4164
	case 0x23: /* mov from reg to dr */
4165
		if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4166
				((ctxt->mode == X86EMUL_MODE_PROT64) ?
4167
				 ~0ULL : ~0U)) < 0) {
4168
			/* #UD condition is already handled by the code above */
4169
			emulate_gp(ctxt, 0);
4170
			rc = X86EMUL_PROPAGATE_FAULT;
4171 4172 4173
			goto done;
		}

4174
		c->dst.type = OP_NONE;	/* no writeback */
A
Avi Kivity 已提交
4175
		break;
4176 4177 4178 4179
	case 0x30:
		/* wrmsr */
		msr_data = (u32)c->regs[VCPU_REGS_RAX]
			| ((u64)c->regs[VCPU_REGS_RDX] << 32);
4180
		if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4181
			emulate_gp(ctxt, 0);
4182
			rc = X86EMUL_PROPAGATE_FAULT;
4183
			goto done;
4184 4185 4186 4187 4188
		}
		rc = X86EMUL_CONTINUE;
		break;
	case 0x32:
		/* rdmsr */
4189
		if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4190
			emulate_gp(ctxt, 0);
4191
			rc = X86EMUL_PROPAGATE_FAULT;
4192
			goto done;
4193 4194 4195 4196 4197 4198
		} else {
			c->regs[VCPU_REGS_RAX] = (u32)msr_data;
			c->regs[VCPU_REGS_RDX] = msr_data >> 32;
		}
		rc = X86EMUL_CONTINUE;
		break;
4199
	case 0x34:		/* sysenter */
4200
		rc = emulate_sysenter(ctxt, ops);
4201 4202
		break;
	case 0x35:		/* sysexit */
4203
		rc = emulate_sysexit(ctxt, ops);
4204
		break;
A
Avi Kivity 已提交
4205
	case 0x40 ... 0x4f:	/* cmov */
4206
		c->dst.val = c->dst.orig_val = c->src.val;
4207 4208
		if (!test_cc(c->b, ctxt->eflags))
			c->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
4209
		break;
4210
	case 0x80 ... 0x8f: /* jnz rel, etc*/
4211
		if (test_cc(c->b, ctxt->eflags))
4212
			jmp_rel(c, c->src.val);
4213
		break;
4214 4215 4216
	case 0x90 ... 0x9f:     /* setcc r/m8 */
		c->dst.val = test_cc(c->b, ctxt->eflags);
		break;
4217
	case 0xa0:	  /* push fs */
4218
		rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4219 4220 4221 4222
		break;
	case 0xa1:	 /* pop fs */
		rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
		break;
4223 4224
	case 0xa3:
	      bt:		/* bt */
Q
Qing He 已提交
4225
		c->dst.type = OP_NONE;
4226 4227
		/* only subword offset */
		c->src.val &= (c->dst.bytes << 3) - 1;
4228
		emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4229
		break;
4230 4231 4232 4233
	case 0xa4: /* shld imm8, r, r/m */
	case 0xa5: /* shld cl, r, r/m */
		emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
		break;
4234
	case 0xa8:	/* push gs */
4235
		rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4236 4237 4238 4239
		break;
	case 0xa9:	/* pop gs */
		rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
		break;
4240 4241
	case 0xab:
	      bts:		/* bts */
4242
		emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4243
		break;
4244 4245 4246 4247
	case 0xac: /* shrd imm8, r, r/m */
	case 0xad: /* shrd cl, r, r/m */
		emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
		break;
4248 4249
	case 0xae:              /* clflush */
		break;
A
Avi Kivity 已提交
4250 4251 4252 4253 4254
	case 0xb0 ... 0xb1:	/* cmpxchg */
		/*
		 * Save real source value, then compare EAX against
		 * destination.
		 */
4255 4256
		c->src.orig_val = c->src.val;
		c->src.val = c->regs[VCPU_REGS_RAX];
4257 4258
		emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
		if (ctxt->eflags & EFLG_ZF) {
A
Avi Kivity 已提交
4259
			/* Success: write back to memory. */
4260
			c->dst.val = c->src.orig_val;
A
Avi Kivity 已提交
4261 4262
		} else {
			/* Failure: write the value we saw to EAX. */
4263
			c->dst.type = OP_REG;
4264
			c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
A
Avi Kivity 已提交
4265 4266
		}
		break;
4267 4268 4269
	case 0xb2:		/* lss */
		rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
		break;
A
Avi Kivity 已提交
4270 4271
	case 0xb3:
	      btr:		/* btr */
4272
		emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
A
Avi Kivity 已提交
4273
		break;
4274 4275 4276 4277 4278 4279
	case 0xb4:		/* lfs */
		rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
		break;
	case 0xb5:		/* lgs */
		rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
		break;
A
Avi Kivity 已提交
4280
	case 0xb6 ... 0xb7:	/* movzx */
4281 4282 4283
		c->dst.bytes = c->op_bytes;
		c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
						       : (u16) c->src.val;
A
Avi Kivity 已提交
4284 4285
		break;
	case 0xba:		/* Grp8 */
4286
		switch (c->modrm_reg & 3) {
A
Avi Kivity 已提交
4287 4288 4289 4290 4291 4292 4293 4294 4295 4296
		case 0:
			goto bt;
		case 1:
			goto bts;
		case 2:
			goto btr;
		case 3:
			goto btc;
		}
		break;
4297 4298
	case 0xbb:
	      btc:		/* btc */
4299
		emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4300
		break;
4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324
	case 0xbc: {		/* bsf */
		u8 zf;
		__asm__ ("bsf %2, %0; setz %1"
			 : "=r"(c->dst.val), "=q"(zf)
			 : "r"(c->src.val));
		ctxt->eflags &= ~X86_EFLAGS_ZF;
		if (zf) {
			ctxt->eflags |= X86_EFLAGS_ZF;
			c->dst.type = OP_NONE;	/* Disable writeback. */
		}
		break;
	}
	case 0xbd: {		/* bsr */
		u8 zf;
		__asm__ ("bsr %2, %0; setz %1"
			 : "=r"(c->dst.val), "=q"(zf)
			 : "r"(c->src.val));
		ctxt->eflags &= ~X86_EFLAGS_ZF;
		if (zf) {
			ctxt->eflags |= X86_EFLAGS_ZF;
			c->dst.type = OP_NONE;	/* Disable writeback. */
		}
		break;
	}
A
Avi Kivity 已提交
4325
	case 0xbe ... 0xbf:	/* movsx */
4326 4327 4328
		c->dst.bytes = c->op_bytes;
		c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
							(s16) c->src.val;
A
Avi Kivity 已提交
4329
		break;
4330 4331 4332 4333 4334 4335
	case 0xc0 ... 0xc1:	/* xadd */
		emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
		/* Write back the register source. */
		c->src.val = c->dst.orig_val;
		write_register_operand(&c->src);
		break;
4336
	case 0xc3:		/* movnti */
4337 4338 4339
		c->dst.bytes = c->op_bytes;
		c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
							(u64) c->src.val;
4340
		break;
A
Avi Kivity 已提交
4341
	case 0xc7:		/* Grp9 (cmpxchg8b) */
4342
		rc = emulate_grp9(ctxt, ops);
4343
		break;
4344 4345
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4346
	}
4347 4348 4349 4350

	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
4351 4352 4353
	goto writeback;

cannot_emulate:
4354
	return EMULATION_FAILED;
A
Avi Kivity 已提交
4355
}