emulate.c 107.4 KB
Newer Older
A
Avi Kivity 已提交
1
/******************************************************************************
2
 * emulate.c
A
Avi Kivity 已提交
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
A
Avi Kivity 已提交
10 11
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
A
Avi Kivity 已提交
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
A
Avi Kivity 已提交
27

28
#include "x86.h"
29
#include "tss.h"
30

A
Avi Kivity 已提交
31 32 33 34 35 36 37 38 39 40
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
41
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
42
/* Destination operand type. */
43 44 45 46 47 48
#define ImplicitOps (1<<1)	/* Implicit in opcode. No generic decode. */
#define DstReg      (2<<1)	/* Register operand. */
#define DstMem      (3<<1)	/* Memory operand. */
#define DstAcc      (4<<1)	/* Destination Accumulator */
#define DstDI       (5<<1)	/* Destination is in ES:(E)DI */
#define DstMem64    (6<<1)	/* 64bit memory operand */
49
#define DstImmUByte (7<<1)	/* 8-bit unsigned immediate operand */
50 51
#define DstDX       (8<<1)	/* Destination is in DX register */
#define DstMask     (0xf<<1)
A
Avi Kivity 已提交
52
/* Source operand type. */
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
#define SrcNone     (0<<5)	/* No source operand. */
#define SrcReg      (1<<5)	/* Register operand. */
#define SrcMem      (2<<5)	/* Memory operand. */
#define SrcMem16    (3<<5)	/* Memory operand (16-bit). */
#define SrcMem32    (4<<5)	/* Memory operand (32-bit). */
#define SrcImm      (5<<5)	/* Immediate operand. */
#define SrcImmByte  (6<<5)	/* 8-bit sign-extended immediate operand. */
#define SrcOne      (7<<5)	/* Implied '1' */
#define SrcImmUByte (8<<5)      /* 8-bit unsigned immediate operand. */
#define SrcImmU     (9<<5)      /* Immediate operand, unsigned */
#define SrcSI       (0xa<<5)	/* Source is in the DS:RSI */
#define SrcImmFAddr (0xb<<5)	/* Source is immediate far address */
#define SrcMemFAddr (0xc<<5)	/* Source is far address in memory */
#define SrcAcc      (0xd<<5)	/* Source Accumulator */
#define SrcImmU16   (0xe<<5)    /* Immediate operand, unsigned, 16 bits */
#define SrcDX       (0xf<<5)	/* Source is in DX register */
#define SrcMask     (0xf<<5)
A
Avi Kivity 已提交
70
/* Generic ModRM decode. */
71
#define ModRM       (1<<9)
A
Avi Kivity 已提交
72
/* Destination is only written; never read. */
73 74 75 76 77 78 79 80 81 82 83
#define Mov         (1<<10)
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
#define Sse         (1<<18)     /* SSE Vector instruction */
84
/* Misc flags */
85
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
86
#define VendorSpecific (1<<22) /* Vendor specific instruction */
87
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
88
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
89
#define Undefined   (1<<25) /* No Such Instruction */
90
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
91
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
92
#define No64	    (1<<28)
93 94 95 96 97
/* Source 2 operand type */
#define Src2None    (0<<29)
#define Src2CL      (1<<29)
#define Src2ImmByte (2<<29)
#define Src2One     (3<<29)
98
#define Src2Imm     (4<<29)
99
#define Src2Mask    (7<<29)
A
Avi Kivity 已提交
100

101 102 103 104 105 106 107 108
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
109

110 111
struct opcode {
	u32 flags;
112
	u8 intercept;
113
	union {
114
		int (*execute)(struct x86_emulate_ctxt *ctxt);
115 116
		struct opcode *group;
		struct group_dual *gdual;
117
		struct gprefix *gprefix;
118
	} u;
119
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
120 121 122 123 124
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
125 126
};

127 128 129 130 131 132 133
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

A
Avi Kivity 已提交
134
/* EFLAGS bit definitions. */
135 136 137 138
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
139 140
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
141 142
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
A
Avi Kivity 已提交
143 144
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
145
#define EFLG_IF (1<<9)
146
#define EFLG_TF (1<<8)
A
Avi Kivity 已提交
147 148 149 150 151 152
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

153 154 155
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

A
Avi Kivity 已提交
156 157 158 159 160 161 162
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

163
#if defined(CONFIG_X86_64)
A
Avi Kivity 已提交
164 165 166 167 168 169 170 171 172 173 174 175 176 177
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
A
Avi Kivity 已提交
193 194 195 196 197 198 199 200 201

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

202 203 204 205 206 207
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

208
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
209 210 211 212 213
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
214 215
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
216
			  "=&r" (_tmp)					\
217
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
218
	} while (0)
219 220


A
Avi Kivity 已提交
221
/* Raw emulation: instruction has two explicit operands. */
222
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
223 224 225
	do {								\
		unsigned long _tmp;					\
									\
226
		switch ((ctxt)->dst.bytes) {				\
227
		case 2:							\
228
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
229 230
			break;						\
		case 4:							\
231
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
232 233
			break;						\
		case 8:							\
234
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
235 236
			break;						\
		}							\
A
Avi Kivity 已提交
237 238
	} while (0)

239
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
A
Avi Kivity 已提交
240
	do {								     \
241
		unsigned long _tmp;					     \
242
		switch ((ctxt)->dst.bytes) {				     \
A
Avi Kivity 已提交
243
		case 1:							     \
244
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
A
Avi Kivity 已提交
245 246
			break;						     \
		default:						     \
247
			__emulate_2op_nobyte(ctxt, _op,			     \
A
Avi Kivity 已提交
248 249 250 251 252 253
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
254 255
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
A
Avi Kivity 已提交
256 257

/* Source operand is byte, word, long or quad sized. */
258 259
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
260 261

/* Source operand is word, long or quad sized. */
262 263
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
264

265
/* Instruction has three operands and one operand is stored in ECX register */
266
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
267 268
	do {								\
		unsigned long _tmp;					\
269 270 271
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
272 273 274 275 276
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
277
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
278 279 280
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
281 282 283
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
284 285
	} while (0)

286
#define emulate_2op_cl(ctxt, _op)					\
287
	do {								\
288
		switch ((ctxt)->dst.bytes) {				\
289
		case 2:							\
290
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
291 292
			break;						\
		case 4:							\
293
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
294 295
			break;						\
		case 8:							\
296
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
297 298
			break;						\
		}							\
299 300
	} while (0)

301
#define __emulate_1op(ctxt, _op, _suffix)				\
A
Avi Kivity 已提交
302 303 304
	do {								\
		unsigned long _tmp;					\
									\
305 306 307 308
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
309
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
310 311 312 313 314
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
315
#define emulate_1op(ctxt, _op)						\
316
	do {								\
317 318 319 320 321
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
A
Avi Kivity 已提交
322 323 324
		}							\
	} while (0)

325
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
326 327
	do {								\
		unsigned long _tmp;					\
328 329
		ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX];		\
		ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX];		\
330 331 332 333 334 335 336 337 338 339 340 341
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
342 343 344 345
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val),	\
			  "a" (*rax), "d" (*rdx));			\
346 347
	} while (0)

348
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
349
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
350
	do {								\
351
		switch((ctxt)->src.bytes) {				\
352
		case 1:							\
353
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
354 355
			break;						\
		case 2:							\
356
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
357 358
			break;						\
		case 4:							\
359
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
360 361
			break;						\
		case 8: ON64(						\
362
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
363 364 365 366
			break;						\
		}							\
	} while (0)

367 368 369 370 371 372
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
373 374 375 376 377 378 379 380
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
381 382 383
		.next_rip   = ctxt->eip,
	};

384
	return ctxt->ops->intercept(ctxt, &info, stage);
385 386
}

387
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
388
{
389
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
390 391
}

A
Avi Kivity 已提交
392
/* Access/update address held in a register, based on addressing mode. */
393
static inline unsigned long
394
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
395
{
396
	if (ctxt->ad_bytes == sizeof(unsigned long))
397 398
		return reg;
	else
399
		return reg & ad_mask(ctxt);
400 401 402
}

static inline unsigned long
403
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
404
{
405
	return address_mask(ctxt, reg);
406 407
}

408
static inline void
409
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
410
{
411
	if (ctxt->ad_bytes == sizeof(unsigned long))
412 413
		*reg += inc;
	else
414
		*reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
415
}
A
Avi Kivity 已提交
416

417
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
418
{
419
	register_address_increment(ctxt, &ctxt->_eip, rel);
420
}
421

422 423 424 425 426 427 428
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

429
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
430
{
431 432
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
433 434
}

435
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
436 437 438 439
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

440
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
441 442
}

443
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
444
{
445
	if (!ctxt->has_seg_override)
446 447
		return 0;

448
	return ctxt->seg_override;
449 450
}

451 452
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
453
{
454 455 456
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
457
	return X86EMUL_PROPAGATE_FAULT;
458 459
}

460 461 462 463 464
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

465
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
466
{
467
	return emulate_exception(ctxt, GP_VECTOR, err, true);
468 469
}

470 471 472 473 474
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

475
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
476
{
477
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
478 479
}

480
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
481
{
482
	return emulate_exception(ctxt, TS_VECTOR, err, true);
483 484
}

485 486
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
487
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
488 489
}

A
Avi Kivity 已提交
490 491 492 493 494
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

515
static int __linearize(struct x86_emulate_ctxt *ctxt,
516
		     struct segmented_address addr,
517
		     unsigned size, bool write, bool fetch,
518 519
		     ulong *linear)
{
520 521
	struct desc_struct desc;
	bool usable;
522
	ulong la;
523
	u32 lim;
524
	u16 sel;
525
	unsigned cpl, rpl;
526

527
	la = seg_base(ctxt, addr.seg) + addr.ea;
528 529 530 531 532 533 534 535
	switch (ctxt->mode) {
	case X86EMUL_MODE_REAL:
		break;
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
536 537
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
538 539 540 541 542 543
		if (!usable)
			goto bad;
		/* code segment or read-only data segment */
		if (((desc.type & 8) || !(desc.type & 2)) && write)
			goto bad;
		/* unreadable code segment */
544
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
545 546 547 548 549 550 551 552 553 554 555 556 557 558
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
			/* exapand-down segment */
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
559
		cpl = ctxt->ops->cpl(ctxt);
560
		rpl = sel & 3;
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
		cpl = max(cpl, rpl);
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
577
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
578 579 580
		la &= (u32)-1;
	*linear = la;
	return X86EMUL_CONTINUE;
581 582 583 584 585
bad:
	if (addr.seg == VCPU_SREG_SS)
		return emulate_ss(ctxt, addr.seg);
	else
		return emulate_gp(ctxt, addr.seg);
586 587
}

588 589 590 591 592 593 594 595 596
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


597 598 599 600 601
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
602 603 604
	int rc;
	ulong linear;

605
	rc = linearize(ctxt, addr, size, false, &linear);
606 607
	if (rc != X86EMUL_CONTINUE)
		return rc;
608
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
609 610
}

611 612 613 614 615 616 617 618
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
619
{
620
	struct fetch_cache *fc = &ctxt->fetch;
621
	int rc;
622
	int size, cur_size;
623

624
	if (ctxt->_eip == fc->end) {
625
		unsigned long linear;
626 627
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
628
		cur_size = fc->end - fc->start;
629 630
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
631
		rc = __linearize(ctxt, addr, size, false, true, &linear);
632
		if (unlikely(rc != X86EMUL_CONTINUE))
633
			return rc;
634 635
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
636
		if (unlikely(rc != X86EMUL_CONTINUE))
637
			return rc;
638
		fc->end += size;
639
	}
640 641
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
642
	return X86EMUL_CONTINUE;
643 644 645
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
646
			 void *dest, unsigned size)
647
{
648
	int rc;
649

650
	/* x86 instructions are limited to 15 bytes. */
651
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
652
		return X86EMUL_UNHANDLEABLE;
653
	while (size--) {
654
		rc = do_insn_fetch_byte(ctxt, dest++);
655
		if (rc != X86EMUL_CONTINUE)
656 657
			return rc;
	}
658
	return X86EMUL_CONTINUE;
659 660
}

661
/* Fetch next part of the instruction being emulated. */
662
#define insn_fetch(_type, _ctxt)					\
663
({	unsigned long _x;						\
664
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
665 666 667 668 669
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

670 671
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
672 673 674 675
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

676 677 678 679 680 681 682
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
static void *decode_register(u8 modrm_reg, unsigned long *regs,
			     int highbyte_regs)
A
Avi Kivity 已提交
683 684 685 686 687 688 689 690 691 692
{
	void *p;

	p = &regs[modrm_reg];
	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
		p = (unsigned char *)&regs[modrm_reg & 3] + 1;
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
693
			   struct segmented_address addr,
A
Avi Kivity 已提交
694 695 696 697 698 699 700
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
701
	rc = segmented_read_std(ctxt, addr, size, 2);
702
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
703
		return rc;
704
	addr.ea += 2;
705
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
706 707 708
	return rc;
}

709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743
static int test_cc(unsigned int condition, unsigned int flags)
{
	int rc = 0;

	switch ((condition & 15) >> 1) {
	case 0: /* o */
		rc |= (flags & EFLG_OF);
		break;
	case 1: /* b/c/nae */
		rc |= (flags & EFLG_CF);
		break;
	case 2: /* z/e */
		rc |= (flags & EFLG_ZF);
		break;
	case 3: /* be/na */
		rc |= (flags & (EFLG_CF|EFLG_ZF));
		break;
	case 4: /* s */
		rc |= (flags & EFLG_SF);
		break;
	case 5: /* p/pe */
		rc |= (flags & EFLG_PF);
		break;
	case 7: /* le/ng */
		rc |= (flags & EFLG_ZF);
		/* fall through */
	case 6: /* l/nge */
		rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
		break;
	}

	/* Odd condition identifiers (lsb == 1) have inverted sense. */
	return (!!rc ^ (condition & 1));
}

744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

A
Avi Kivity 已提交
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
				    struct operand *op,
819 820
				    int inhibit_bytereg)
{
821 822
	unsigned reg = ctxt->modrm_reg;
	int highbyte_regs = ctxt->rex_prefix == 0;
823

824 825
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
826

827
	if (ctxt->d & Sse) {
A
Avi Kivity 已提交
828 829 830 831 832 833 834
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}

835
	op->type = OP_REG;
836 837
	if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
		op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
838 839
		op->bytes = 1;
	} else {
840 841
		op->addr.reg = decode_register(reg, ctxt->regs, 0);
		op->bytes = ctxt->op_bytes;
842
	}
843
	fetch_register_operand(op);
844 845 846
	op->orig_val = op->val;
}

847
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
848
			struct operand *op)
849 850
{
	u8 sib;
851
	int index_reg = 0, base_reg = 0, scale;
852
	int rc = X86EMUL_CONTINUE;
853
	ulong modrm_ea = 0;
854

855 856 857 858
	if (ctxt->rex_prefix) {
		ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1;	/* REX.R */
		index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
		ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
859 860
	}

861
	ctxt->modrm = insn_fetch(u8, ctxt);
862 863 864 865
	ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
	ctxt->modrm_rm |= (ctxt->modrm & 0x07);
	ctxt->modrm_seg = VCPU_SREG_DS;
866

867
	if (ctxt->modrm_mod == 3) {
868
		op->type = OP_REG;
869 870 871 872
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.reg = decode_register(ctxt->modrm_rm,
					       ctxt->regs, ctxt->d & ByteOp);
		if (ctxt->d & Sse) {
A
Avi Kivity 已提交
873 874
			op->type = OP_XMM;
			op->bytes = 16;
875 876
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
A
Avi Kivity 已提交
877 878
			return rc;
		}
879
		fetch_register_operand(op);
880 881 882
		return rc;
	}

883 884
	op->type = OP_MEM;

885 886 887 888 889
	if (ctxt->ad_bytes == 2) {
		unsigned bx = ctxt->regs[VCPU_REGS_RBX];
		unsigned bp = ctxt->regs[VCPU_REGS_RBP];
		unsigned si = ctxt->regs[VCPU_REGS_RSI];
		unsigned di = ctxt->regs[VCPU_REGS_RDI];
890 891

		/* 16-bit ModR/M decode. */
892
		switch (ctxt->modrm_mod) {
893
		case 0:
894
			if (ctxt->modrm_rm == 6)
895
				modrm_ea += insn_fetch(u16, ctxt);
896 897
			break;
		case 1:
898
			modrm_ea += insn_fetch(s8, ctxt);
899 900
			break;
		case 2:
901
			modrm_ea += insn_fetch(u16, ctxt);
902 903
			break;
		}
904
		switch (ctxt->modrm_rm) {
905
		case 0:
906
			modrm_ea += bx + si;
907 908
			break;
		case 1:
909
			modrm_ea += bx + di;
910 911
			break;
		case 2:
912
			modrm_ea += bp + si;
913 914
			break;
		case 3:
915
			modrm_ea += bp + di;
916 917
			break;
		case 4:
918
			modrm_ea += si;
919 920
			break;
		case 5:
921
			modrm_ea += di;
922 923
			break;
		case 6:
924
			if (ctxt->modrm_mod != 0)
925
				modrm_ea += bp;
926 927
			break;
		case 7:
928
			modrm_ea += bx;
929 930
			break;
		}
931 932 933
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
934
		modrm_ea = (u16)modrm_ea;
935 936
	} else {
		/* 32/64-bit ModR/M decode. */
937
		if ((ctxt->modrm_rm & 7) == 4) {
938
			sib = insn_fetch(u8, ctxt);
939 940 941 942
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

943
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
944
				modrm_ea += insn_fetch(s32, ctxt);
945
			else
946
				modrm_ea += ctxt->regs[base_reg];
947
			if (index_reg != 4)
948 949
				modrm_ea += ctxt->regs[index_reg] << scale;
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
950
			if (ctxt->mode == X86EMUL_MODE_PROT64)
951
				ctxt->rip_relative = 1;
952
		} else
953 954
			modrm_ea += ctxt->regs[ctxt->modrm_rm];
		switch (ctxt->modrm_mod) {
955
		case 0:
956
			if (ctxt->modrm_rm == 5)
957
				modrm_ea += insn_fetch(s32, ctxt);
958 959
			break;
		case 1:
960
			modrm_ea += insn_fetch(s8, ctxt);
961 962
			break;
		case 2:
963
			modrm_ea += insn_fetch(s32, ctxt);
964 965 966
			break;
		}
	}
967
	op->addr.mem.ea = modrm_ea;
968 969 970 971 972
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
973
		      struct operand *op)
974
{
975
	int rc = X86EMUL_CONTINUE;
976

977
	op->type = OP_MEM;
978
	switch (ctxt->ad_bytes) {
979
	case 2:
980
		op->addr.mem.ea = insn_fetch(u16, ctxt);
981 982
		break;
	case 4:
983
		op->addr.mem.ea = insn_fetch(u32, ctxt);
984 985
		break;
	case 8:
986
		op->addr.mem.ea = insn_fetch(u64, ctxt);
987 988 989 990 991 992
		break;
	}
done:
	return rc;
}

993
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
994
{
995
	long sv = 0, mask;
996

997 998
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
		mask = ~(ctxt->dst.bytes * 8 - 1);
999

1000 1001 1002 1003
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1004

1005
		ctxt->dst.addr.mem.ea += (sv >> 3);
1006
	}
1007 1008

	/* only subword offset */
1009
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1010 1011
}

1012 1013
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1014
{
1015
	int rc;
1016
	struct read_cache *mc = &ctxt->mem_read;
A
Avi Kivity 已提交
1017

1018 1019 1020 1021 1022
	while (size) {
		int n = min(size, 8u);
		size -= n;
		if (mc->pos < mc->end)
			goto read_cached;
1023

1024 1025
		rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
					      &ctxt->exception);
1026 1027 1028
		if (rc != X86EMUL_CONTINUE)
			return rc;
		mc->end += n;
A
Avi Kivity 已提交
1029

1030 1031 1032 1033 1034
	read_cached:
		memcpy(dest, mc->data + mc->pos, n);
		mc->pos += n;
		dest += n;
		addr += n;
A
Avi Kivity 已提交
1035
	}
1036 1037
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1038

1039 1040 1041 1042 1043
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1044 1045 1046
	int rc;
	ulong linear;

1047
	rc = linearize(ctxt, addr, size, false, &linear);
1048 1049
	if (rc != X86EMUL_CONTINUE)
		return rc;
1050
	return read_emulated(ctxt, linear, data, size);
1051 1052 1053 1054 1055 1056 1057
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1058 1059 1060
	int rc;
	ulong linear;

1061
	rc = linearize(ctxt, addr, size, true, &linear);
1062 1063
	if (rc != X86EMUL_CONTINUE)
		return rc;
1064 1065
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1066 1067 1068 1069 1070 1071 1072
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1073 1074 1075
	int rc;
	ulong linear;

1076
	rc = linearize(ctxt, addr, size, true, &linear);
1077 1078
	if (rc != X86EMUL_CONTINUE)
		return rc;
1079 1080
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1081 1082
}

1083 1084 1085 1086
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1087
	struct read_cache *rc = &ctxt->io_read;
1088

1089 1090
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1091 1092
		unsigned int count = ctxt->rep_prefix ?
			address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1093
		in_page = (ctxt->eflags & EFLG_DF) ?
1094 1095
			offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
			PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1096 1097 1098 1099 1100
		n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
			count);
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1101
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1102 1103
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1104 1105
	}

1106 1107 1108 1109
	memcpy(dest, rc->data + rc->pos, size);
	rc->pos += size;
	return 1;
}
A
Avi Kivity 已提交
1110

1111 1112 1113
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1114 1115
	struct x86_emulate_ops *ops = ctxt->ops;

1116 1117
	if (selector & 1 << 2) {
		struct desc_struct desc;
1118 1119
		u16 sel;

1120
		memset (dt, 0, sizeof *dt);
1121
		if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1122
			return;
1123

1124 1125 1126
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
		dt->address = get_desc_base(&desc);
	} else
1127
		ops->get_gdt(ctxt, dt);
1128
}
1129

1130 1131 1132 1133 1134 1135 1136
/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1137

1138
	get_descriptor_table_ptr(ctxt, selector, &dt);
1139

1140 1141
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1142

1143 1144 1145
	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
1146
}
1147

1148 1149 1150 1151 1152 1153 1154
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
A
Avi Kivity 已提交
1155

1156
	get_descriptor_table_ptr(ctxt, selector, &dt);
1157

1158 1159
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
A
Avi Kivity 已提交
1160

1161
	addr = dt.address + index * 8;
1162 1163
	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
				    &ctxt->exception);
1164
}
1165

1166
/* Does not support long mode */
1167 1168 1169 1170 1171 1172 1173 1174 1175
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
	struct desc_struct seg_desc;
	u8 dpl, rpl, cpl;
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
	int ret;
1176

1177
	memset(&seg_desc, 0, sizeof seg_desc);
1178

1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
	if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
	    || ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
		goto load;
	}

	/* NULL selector is not valid for TR, CS and SS */
	if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
	    && null_selector)
		goto exception;

	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

	if (null_selector) /* for NULL selector skip all following checks */
		goto load;

1202
	ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
	err_vec = GP_VECTOR;

	/* can't load system descriptor into segment selecor */
	if (seg <= VCPU_SREG_GS && !seg_desc.s)
		goto exception;

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	rpl = selector & 3;
	dpl = seg_desc.dpl;
1220
	cpl = ctxt->ops->cpl(ctxt);
1221 1222 1223 1224 1225 1226 1227 1228 1229

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1230
		break;
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1246
		break;
1247 1248 1249 1250 1251 1252 1253 1254 1255
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1256
		/*
1257 1258 1259
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1260
		 */
1261 1262 1263 1264
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1265
		break;
1266 1267 1268 1269 1270
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
		seg_desc.type |= 1;
1271
		ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1272 1273 1274 1275
		if (ret != X86EMUL_CONTINUE)
			return ret;
	}
load:
1276
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1277 1278 1279 1280 1281 1282
	return X86EMUL_CONTINUE;
exception:
	emulate_exception(ctxt, err_vec, err_code, true);
	return X86EMUL_PROPAGATE_FAULT;
}

1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301
static void write_register_operand(struct operand *op)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (op->bytes) {
	case 1:
		*(u8 *)op->addr.reg = (u8)op->val;
		break;
	case 2:
		*(u16 *)op->addr.reg = (u16)op->val;
		break;
	case 4:
		*op->addr.reg = (u32)op->val;
		break;	/* 64b: zero-extend */
	case 8:
		*op->addr.reg = op->val;
		break;
	}
}

1302
static int writeback(struct x86_emulate_ctxt *ctxt)
1303 1304 1305
{
	int rc;

1306
	switch (ctxt->dst.type) {
1307
	case OP_REG:
1308
		write_register_operand(&ctxt->dst);
A
Avi Kivity 已提交
1309
		break;
1310
	case OP_MEM:
1311
		if (ctxt->lock_prefix)
1312
			rc = segmented_cmpxchg(ctxt,
1313 1314 1315 1316
					       ctxt->dst.addr.mem,
					       &ctxt->dst.orig_val,
					       &ctxt->dst.val,
					       ctxt->dst.bytes);
1317
		else
1318
			rc = segmented_write(ctxt,
1319 1320 1321
					     ctxt->dst.addr.mem,
					     &ctxt->dst.val,
					     ctxt->dst.bytes);
1322 1323
		if (rc != X86EMUL_CONTINUE)
			return rc;
1324
		break;
A
Avi Kivity 已提交
1325
	case OP_XMM:
1326
		write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
A
Avi Kivity 已提交
1327
		break;
1328 1329
	case OP_NONE:
		/* no writeback */
1330
		break;
1331
	default:
1332
		break;
A
Avi Kivity 已提交
1333
	}
1334 1335
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1336

1337
static int em_push(struct x86_emulate_ctxt *ctxt)
1338
{
1339
	struct segmented_address addr;
1340

1341 1342
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1343 1344 1345
	addr.seg = VCPU_SREG_SS;

	/* Disable writeback. */
1346 1347
	ctxt->dst.type = OP_NONE;
	return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1348
}
1349

1350 1351 1352 1353
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1354
	struct segmented_address addr;
1355

1356
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1357
	addr.seg = VCPU_SREG_SS;
1358
	rc = segmented_read(ctxt, addr, dest, len);
1359 1360 1361
	if (rc != X86EMUL_CONTINUE)
		return rc;

1362
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1363
	return rc;
1364 1365
}

1366 1367
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1368
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1369 1370
}

1371
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1372
			void *dest, int len)
1373 1374
{
	int rc;
1375 1376
	unsigned long val, change_mask;
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1377
	int cpl = ctxt->ops->cpl(ctxt);
1378

1379
	rc = emulate_pop(ctxt, &val, len);
1380 1381
	if (rc != X86EMUL_CONTINUE)
		return rc;
1382

1383 1384
	change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
		| EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1385

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
			change_mask |= EFLG_IOPL;
		if (cpl <= iopl)
			change_mask |= EFLG_IF;
		break;
	case X86EMUL_MODE_VM86:
1396 1397
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1398 1399 1400 1401 1402
		change_mask |= EFLG_IF;
		break;
	default: /* real mode */
		change_mask |= (EFLG_IOPL | EFLG_IF);
		break;
1403
	}
1404 1405 1406 1407 1408

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1409 1410
}

1411 1412
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1413 1414 1415 1416
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1417 1418
}

1419
static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1420
{
1421
	ctxt->src.val = get_segment_selector(ctxt, seg);
1422

1423
	return em_push(ctxt);
1424 1425
}

1426
static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1427
{
1428 1429
	unsigned long selector;
	int rc;
1430

1431
	rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1432 1433 1434
	if (rc != X86EMUL_CONTINUE)
		return rc;

1435
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1436
	return rc;
1437 1438
}

1439
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1440
{
1441
	unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1442 1443
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1444

1445 1446
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
1447
		(ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1448

1449
		rc = em_push(ctxt);
1450 1451
		if (rc != X86EMUL_CONTINUE)
			return rc;
1452

1453
		++reg;
1454 1455
	}

1456
	return rc;
1457 1458
}

1459 1460
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
1461
	ctxt->src.val =  (unsigned long)ctxt->eflags;
1462 1463 1464
	return em_push(ctxt);
}

1465
static int em_popa(struct x86_emulate_ctxt *ctxt)
1466
{
1467 1468
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
1469

1470 1471
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
1472 1473
			register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
							ctxt->op_bytes);
1474 1475
			--reg;
		}
1476

1477
		rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1478 1479 1480
		if (rc != X86EMUL_CONTINUE)
			break;
		--reg;
1481
	}
1482
	return rc;
1483 1484
}

1485
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1486
{
1487
	struct x86_emulate_ops *ops = ctxt->ops;
1488
	int rc;
1489 1490 1491 1492 1493 1494
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
1495
	ctxt->src.val = ctxt->eflags;
1496
	rc = em_push(ctxt);
1497 1498
	if (rc != X86EMUL_CONTINUE)
		return rc;
1499 1500 1501

	ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);

1502
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1503
	rc = em_push(ctxt);
1504 1505
	if (rc != X86EMUL_CONTINUE)
		return rc;
1506

1507
	ctxt->src.val = ctxt->_eip;
1508
	rc = em_push(ctxt);
1509 1510 1511
	if (rc != X86EMUL_CONTINUE)
		return rc;

1512
	ops->get_idt(ctxt, &dt);
1513 1514 1515 1516

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

1517
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1518 1519 1520
	if (rc != X86EMUL_CONTINUE)
		return rc;

1521
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1522 1523 1524
	if (rc != X86EMUL_CONTINUE)
		return rc;

1525
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1526 1527 1528
	if (rc != X86EMUL_CONTINUE)
		return rc;

1529
	ctxt->_eip = eip;
1530 1531 1532 1533

	return rc;
}

1534
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1535 1536 1537
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1538
		return emulate_int_real(ctxt, irq);
1539 1540 1541 1542 1543 1544 1545 1546 1547 1548
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

1549
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1550
{
1551 1552 1553 1554 1555 1556 1557 1558
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
	unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
			     EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
			     EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
	unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1559

1560
	/* TODO: Add stack limit check */
1561

1562
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1563

1564 1565
	if (rc != X86EMUL_CONTINUE)
		return rc;
1566

1567 1568
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
1569

1570
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1571

1572 1573
	if (rc != X86EMUL_CONTINUE)
		return rc;
1574

1575
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1576

1577 1578
	if (rc != X86EMUL_CONTINUE)
		return rc;
1579

1580
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1581

1582 1583
	if (rc != X86EMUL_CONTINUE)
		return rc;
1584

1585
	ctxt->_eip = temp_eip;
1586 1587


1588
	if (ctxt->op_bytes == 4)
1589
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1590
	else if (ctxt->op_bytes == 2) {
1591 1592
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
1593
	}
1594 1595 1596 1597 1598

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
	ctxt->eflags |= EFLG_RESERVED_ONE_MASK;

	return rc;
1599 1600
}

1601
static int em_iret(struct x86_emulate_ctxt *ctxt)
1602
{
1603 1604
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1605
		return emulate_iret_real(ctxt);
1606 1607 1608 1609
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
1610
	default:
1611 1612
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
1613 1614 1615
	}
}

1616 1617 1618 1619 1620
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned short sel;

1621
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1622

1623
	rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1624 1625 1626
	if (rc != X86EMUL_CONTINUE)
		return rc;

1627 1628
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1629 1630 1631
	return X86EMUL_CONTINUE;
}

1632
static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1633
{
1634
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1635 1636
}

1637
static int em_grp2(struct x86_emulate_ctxt *ctxt)
1638
{
1639
	switch (ctxt->modrm_reg) {
1640
	case 0:	/* rol */
1641
		emulate_2op_SrcB(ctxt, "rol");
1642 1643
		break;
	case 1:	/* ror */
1644
		emulate_2op_SrcB(ctxt, "ror");
1645 1646
		break;
	case 2:	/* rcl */
1647
		emulate_2op_SrcB(ctxt, "rcl");
1648 1649
		break;
	case 3:	/* rcr */
1650
		emulate_2op_SrcB(ctxt, "rcr");
1651 1652 1653
		break;
	case 4:	/* sal/shl */
	case 6:	/* sal/shl */
1654
		emulate_2op_SrcB(ctxt, "sal");
1655 1656
		break;
	case 5:	/* shr */
1657
		emulate_2op_SrcB(ctxt, "shr");
1658 1659
		break;
	case 7:	/* sar */
1660
		emulate_2op_SrcB(ctxt, "sar");
1661 1662
		break;
	}
1663
	return X86EMUL_CONTINUE;
1664 1665
}

1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694
static int em_not(struct x86_emulate_ctxt *ctxt)
{
	ctxt->dst.val = ~ctxt->dst.val;
	return X86EMUL_CONTINUE;
}

static int em_neg(struct x86_emulate_ctxt *ctxt)
{
	emulate_1op(ctxt, "neg");
	return X86EMUL_CONTINUE;
}

static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "mul", ex);
	return X86EMUL_CONTINUE;
}

static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "imul", ex);
	return X86EMUL_CONTINUE;
}

static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1695
{
1696
	u8 de = 0;
1697

1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708
	emulate_1op_rax_rdx(ctxt, "div", de);
	if (de)
		return emulate_de(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 de = 0;

	emulate_1op_rax_rdx(ctxt, "idiv", de);
1709 1710
	if (de)
		return emulate_de(ctxt);
1711
	return X86EMUL_CONTINUE;
1712 1713
}

1714
static int em_grp45(struct x86_emulate_ctxt *ctxt)
1715
{
1716
	int rc = X86EMUL_CONTINUE;
1717

1718
	switch (ctxt->modrm_reg) {
1719
	case 0:	/* inc */
1720
		emulate_1op(ctxt, "inc");
1721 1722
		break;
	case 1:	/* dec */
1723
		emulate_1op(ctxt, "dec");
1724
		break;
1725 1726
	case 2: /* call near abs */ {
		long int old_eip;
1727 1728 1729
		old_eip = ctxt->_eip;
		ctxt->_eip = ctxt->src.val;
		ctxt->src.val = old_eip;
1730
		rc = em_push(ctxt);
1731 1732
		break;
	}
1733
	case 4: /* jmp abs */
1734
		ctxt->_eip = ctxt->src.val;
1735
		break;
1736 1737 1738
	case 5: /* jmp far */
		rc = em_jmp_far(ctxt);
		break;
1739
	case 6:	/* push */
1740
		rc = em_push(ctxt);
1741 1742
		break;
	}
1743
	return rc;
1744 1745
}

1746
static int em_grp9(struct x86_emulate_ctxt *ctxt)
1747
{
1748
	u64 old = ctxt->dst.orig_val64;
1749

1750 1751 1752 1753
	if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
	    ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
		ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
		ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1754
		ctxt->eflags &= ~EFLG_ZF;
1755
	} else {
1756 1757
		ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
			(u32) ctxt->regs[VCPU_REGS_RBX];
1758

1759
		ctxt->eflags |= EFLG_ZF;
1760
	}
1761
	return X86EMUL_CONTINUE;
1762 1763
}

1764 1765
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
1766 1767 1768
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
1769 1770 1771
	return em_pop(ctxt);
}

1772
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1773 1774 1775 1776
{
	int rc;
	unsigned long cs;

1777
	rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1778
	if (rc != X86EMUL_CONTINUE)
1779
		return rc;
1780 1781 1782
	if (ctxt->op_bytes == 4)
		ctxt->_eip = (u32)ctxt->_eip;
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1783
	if (rc != X86EMUL_CONTINUE)
1784
		return rc;
1785
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1786 1787 1788
	return rc;
}

1789
static int emulate_load_segment(struct x86_emulate_ctxt *ctxt, int seg)
1790 1791 1792 1793
{
	unsigned short sel;
	int rc;

1794
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1795

1796
	rc = load_segment_descriptor(ctxt, sel, seg);
1797 1798 1799
	if (rc != X86EMUL_CONTINUE)
		return rc;

1800
	ctxt->dst.val = ctxt->src.val;
1801 1802 1803
	return rc;
}

1804
static void
1805
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1806
			struct desc_struct *cs, struct desc_struct *ss)
1807
{
1808 1809
	u16 selector;

1810
	memset(cs, 0, sizeof(struct desc_struct));
1811
	ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1812
	memset(ss, 0, sizeof(struct desc_struct));
1813 1814

	cs->l = 0;		/* will be adjusted later */
1815
	set_desc_base(cs, 0);	/* flat segment */
1816
	cs->g = 1;		/* 4kb granularity */
1817
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
1818 1819 1820
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
1821 1822
	cs->p = 1;
	cs->d = 1;
1823

1824 1825
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
1826 1827 1828
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
1829
	ss->d = 1;		/* 32bit stack segment */
1830
	ss->dpl = 0;
1831
	ss->p = 1;
1832 1833
}

1834
static int em_syscall(struct x86_emulate_ctxt *ctxt)
1835
{
1836
	struct x86_emulate_ops *ops = ctxt->ops;
1837
	struct desc_struct cs, ss;
1838
	u64 msr_data;
1839
	u16 cs_sel, ss_sel;
1840
	u64 efer = 0;
1841 1842

	/* syscall is not available in real mode */
1843
	if (ctxt->mode == X86EMUL_MODE_REAL ||
1844 1845
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
1846

1847
	ops->get_msr(ctxt, MSR_EFER, &efer);
1848
	setup_syscalls_segments(ctxt, &cs, &ss);
1849

1850
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
1851
	msr_data >>= 32;
1852 1853
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
1854

1855
	if (efer & EFER_LMA) {
1856
		cs.d = 0;
1857 1858
		cs.l = 1;
	}
1859 1860
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1861

1862
	ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1863
	if (efer & EFER_LMA) {
1864
#ifdef CONFIG_X86_64
1865
		ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1866

1867
		ops->get_msr(ctxt,
1868 1869
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
1870
		ctxt->_eip = msr_data;
1871

1872
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1873 1874 1875 1876
		ctxt->eflags &= ~(msr_data | EFLG_RF);
#endif
	} else {
		/* legacy mode */
1877
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
1878
		ctxt->_eip = (u32)msr_data;
1879 1880 1881 1882

		ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
	}

1883
	return X86EMUL_CONTINUE;
1884 1885
}

1886
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1887
{
1888
	struct x86_emulate_ops *ops = ctxt->ops;
1889
	struct desc_struct cs, ss;
1890
	u64 msr_data;
1891
	u16 cs_sel, ss_sel;
1892
	u64 efer = 0;
1893

1894
	ops->get_msr(ctxt, MSR_EFER, &efer);
1895
	/* inject #GP if in real mode */
1896 1897
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
1898 1899 1900 1901

	/* XXX sysenter/sysexit have not been tested in 64bit mode.
	* Therefore, we inject an #UD.
	*/
1902 1903
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return emulate_ud(ctxt);
1904

1905
	setup_syscalls_segments(ctxt, &cs, &ss);
1906

1907
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1908 1909
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT32:
1910 1911
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
1912 1913
		break;
	case X86EMUL_MODE_PROT64:
1914 1915
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
1916 1917 1918 1919
		break;
	}

	ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1920 1921 1922 1923
	cs_sel = (u16)msr_data;
	cs_sel &= ~SELECTOR_RPL_MASK;
	ss_sel = cs_sel + 8;
	ss_sel &= ~SELECTOR_RPL_MASK;
1924
	if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1925
		cs.d = 0;
1926 1927 1928
		cs.l = 1;
	}

1929 1930
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1931

1932
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1933
	ctxt->_eip = msr_data;
1934

1935
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1936
	ctxt->regs[VCPU_REGS_RSP] = msr_data;
1937

1938
	return X86EMUL_CONTINUE;
1939 1940
}

1941
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1942
{
1943
	struct x86_emulate_ops *ops = ctxt->ops;
1944
	struct desc_struct cs, ss;
1945 1946
	u64 msr_data;
	int usermode;
X
Xiao Guangrong 已提交
1947
	u16 cs_sel = 0, ss_sel = 0;
1948

1949 1950
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
1951 1952
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
1953

1954
	setup_syscalls_segments(ctxt, &cs, &ss);
1955

1956
	if ((ctxt->rex_prefix & 0x8) != 0x0)
1957 1958 1959 1960 1961 1962
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

	cs.dpl = 3;
	ss.dpl = 3;
1963
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1964 1965
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
1966
		cs_sel = (u16)(msr_data + 16);
1967 1968
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
1969
		ss_sel = (u16)(msr_data + 24);
1970 1971
		break;
	case X86EMUL_MODE_PROT64:
1972
		cs_sel = (u16)(msr_data + 32);
1973 1974
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
1975 1976
		ss_sel = cs_sel + 8;
		cs.d = 0;
1977 1978 1979
		cs.l = 1;
		break;
	}
1980 1981
	cs_sel |= SELECTOR_RPL_MASK;
	ss_sel |= SELECTOR_RPL_MASK;
1982

1983 1984
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1985

1986 1987
	ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
	ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
1988

1989
	return X86EMUL_CONTINUE;
1990 1991
}

1992
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
1993 1994 1995 1996 1997 1998 1999
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2000
	return ctxt->ops->cpl(ctxt) > iopl;
2001 2002 2003 2004 2005
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2006
	struct x86_emulate_ops *ops = ctxt->ops;
2007
	struct desc_struct tr_seg;
2008
	u32 base3;
2009
	int r;
2010
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2011
	unsigned mask = (1 << len) - 1;
2012
	unsigned long base;
2013

2014
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2015
	if (!tr_seg.p)
2016
		return false;
2017
	if (desc_limit_scaled(&tr_seg) < 103)
2018
		return false;
2019 2020 2021 2022
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2023
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2024 2025
	if (r != X86EMUL_CONTINUE)
		return false;
2026
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2027
		return false;
2028
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2029 2030 2031 2032 2033 2034 2035 2036 2037 2038
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2039 2040 2041
	if (ctxt->perm_ok)
		return true;

2042 2043
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2044
			return false;
2045 2046 2047

	ctxt->perm_ok = true;

2048 2049 2050
	return true;
}

2051 2052 2053
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
2054
	tss->ip = ctxt->_eip;
2055
	tss->flag = ctxt->eflags;
2056 2057 2058 2059 2060 2061 2062 2063
	tss->ax = ctxt->regs[VCPU_REGS_RAX];
	tss->cx = ctxt->regs[VCPU_REGS_RCX];
	tss->dx = ctxt->regs[VCPU_REGS_RDX];
	tss->bx = ctxt->regs[VCPU_REGS_RBX];
	tss->sp = ctxt->regs[VCPU_REGS_RSP];
	tss->bp = ctxt->regs[VCPU_REGS_RBP];
	tss->si = ctxt->regs[VCPU_REGS_RSI];
	tss->di = ctxt->regs[VCPU_REGS_RDI];
2064

2065 2066 2067 2068 2069
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2070 2071 2072 2073 2074 2075 2076
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;

2077
	ctxt->_eip = tss->ip;
2078
	ctxt->eflags = tss->flag | 2;
2079 2080 2081 2082 2083 2084 2085 2086
	ctxt->regs[VCPU_REGS_RAX] = tss->ax;
	ctxt->regs[VCPU_REGS_RCX] = tss->cx;
	ctxt->regs[VCPU_REGS_RDX] = tss->dx;
	ctxt->regs[VCPU_REGS_RBX] = tss->bx;
	ctxt->regs[VCPU_REGS_RSP] = tss->sp;
	ctxt->regs[VCPU_REGS_RBP] = tss->bp;
	ctxt->regs[VCPU_REGS_RSI] = tss->si;
	ctxt->regs[VCPU_REGS_RDI] = tss->di;
2087 2088 2089 2090 2091

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2092 2093 2094 2095 2096
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2097 2098 2099 2100 2101

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2102
	ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2103 2104
	if (ret != X86EMUL_CONTINUE)
		return ret;
2105
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2106 2107
	if (ret != X86EMUL_CONTINUE)
		return ret;
2108
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2109 2110
	if (ret != X86EMUL_CONTINUE)
		return ret;
2111
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2112 2113
	if (ret != X86EMUL_CONTINUE)
		return ret;
2114
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2125
	struct x86_emulate_ops *ops = ctxt->ops;
2126 2127
	struct tss_segment_16 tss_seg;
	int ret;
2128
	u32 new_tss_base = get_desc_base(new_desc);
2129

2130
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2131
			    &ctxt->exception);
2132
	if (ret != X86EMUL_CONTINUE)
2133 2134 2135
		/* FIXME: need to provide precise fault address */
		return ret;

2136
	save_state_to_tss16(ctxt, &tss_seg);
2137

2138
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2139
			     &ctxt->exception);
2140
	if (ret != X86EMUL_CONTINUE)
2141 2142 2143
		/* FIXME: need to provide precise fault address */
		return ret;

2144
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2145
			    &ctxt->exception);
2146
	if (ret != X86EMUL_CONTINUE)
2147 2148 2149 2150 2151 2152
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2153
		ret = ops->write_std(ctxt, new_tss_base,
2154 2155
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2156
				     &ctxt->exception);
2157
		if (ret != X86EMUL_CONTINUE)
2158 2159 2160 2161
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2162
	return load_state_from_tss16(ctxt, &tss_seg);
2163 2164 2165 2166 2167
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
2168
	tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2169
	tss->eip = ctxt->_eip;
2170
	tss->eflags = ctxt->eflags;
2171 2172 2173 2174 2175 2176 2177 2178
	tss->eax = ctxt->regs[VCPU_REGS_RAX];
	tss->ecx = ctxt->regs[VCPU_REGS_RCX];
	tss->edx = ctxt->regs[VCPU_REGS_RDX];
	tss->ebx = ctxt->regs[VCPU_REGS_RBX];
	tss->esp = ctxt->regs[VCPU_REGS_RSP];
	tss->ebp = ctxt->regs[VCPU_REGS_RBP];
	tss->esi = ctxt->regs[VCPU_REGS_RSI];
	tss->edi = ctxt->regs[VCPU_REGS_RDI];
2179

2180 2181 2182 2183 2184 2185 2186
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
	tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2187 2188 2189 2190 2191 2192 2193
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;

2194
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2195
		return emulate_gp(ctxt, 0);
2196
	ctxt->_eip = tss->eip;
2197
	ctxt->eflags = tss->eflags | 2;
2198 2199 2200 2201 2202 2203 2204 2205
	ctxt->regs[VCPU_REGS_RAX] = tss->eax;
	ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
	ctxt->regs[VCPU_REGS_RDX] = tss->edx;
	ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
	ctxt->regs[VCPU_REGS_RSP] = tss->esp;
	ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
	ctxt->regs[VCPU_REGS_RSI] = tss->esi;
	ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2206 2207 2208 2209 2210

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2211 2212 2213 2214 2215 2216 2217
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2218 2219 2220 2221 2222

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2223
	ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2224 2225
	if (ret != X86EMUL_CONTINUE)
		return ret;
2226
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2227 2228
	if (ret != X86EMUL_CONTINUE)
		return ret;
2229
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2230 2231
	if (ret != X86EMUL_CONTINUE)
		return ret;
2232
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2233 2234
	if (ret != X86EMUL_CONTINUE)
		return ret;
2235
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2236 2237
	if (ret != X86EMUL_CONTINUE)
		return ret;
2238
	ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2239 2240
	if (ret != X86EMUL_CONTINUE)
		return ret;
2241
	ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2242 2243 2244 2245 2246 2247 2248 2249 2250 2251
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2252
	struct x86_emulate_ops *ops = ctxt->ops;
2253 2254
	struct tss_segment_32 tss_seg;
	int ret;
2255
	u32 new_tss_base = get_desc_base(new_desc);
2256

2257
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2258
			    &ctxt->exception);
2259
	if (ret != X86EMUL_CONTINUE)
2260 2261 2262
		/* FIXME: need to provide precise fault address */
		return ret;

2263
	save_state_to_tss32(ctxt, &tss_seg);
2264

2265
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2266
			     &ctxt->exception);
2267
	if (ret != X86EMUL_CONTINUE)
2268 2269 2270
		/* FIXME: need to provide precise fault address */
		return ret;

2271
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2272
			    &ctxt->exception);
2273
	if (ret != X86EMUL_CONTINUE)
2274 2275 2276 2277 2278 2279
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2280
		ret = ops->write_std(ctxt, new_tss_base,
2281 2282
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2283
				     &ctxt->exception);
2284
		if (ret != X86EMUL_CONTINUE)
2285 2286 2287 2288
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2289
	return load_state_from_tss32(ctxt, &tss_seg);
2290 2291 2292
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2293 2294
				   u16 tss_selector, int reason,
				   bool has_error_code, u32 error_code)
2295
{
2296
	struct x86_emulate_ops *ops = ctxt->ops;
2297 2298
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
2299
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2300
	ulong old_tss_base =
2301
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2302
	u32 desc_limit;
2303 2304 2305

	/* FIXME: old_tss_base == ~0 ? */

2306
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2307 2308
	if (ret != X86EMUL_CONTINUE)
		return ret;
2309
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2310 2311 2312 2313 2314 2315 2316
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

	if (reason != TASK_SWITCH_IRET) {
		if ((tss_selector & 3) > next_tss_desc.dpl ||
2317
		    ops->cpl(ctxt) > next_tss_desc.dpl)
2318
			return emulate_gp(ctxt, 0);
2319 2320
	}

2321 2322 2323 2324
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
2325
		emulate_ts(ctxt, tss_selector & 0xfffc);
2326 2327 2328 2329 2330
		return X86EMUL_PROPAGATE_FAULT;
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2331
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
	   note that old_tss_sel is not used afetr this point */
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
2343
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2344 2345
				     old_tss_base, &next_tss_desc);
	else
2346
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2347
				     old_tss_base, &next_tss_desc);
2348 2349
	if (ret != X86EMUL_CONTINUE)
		return ret;
2350 2351 2352 2353 2354 2355

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
2356
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2357 2358
	}

2359
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
2360
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2361

2362
	if (has_error_code) {
2363 2364 2365
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
2366
		ret = em_push(ctxt);
2367 2368
	}

2369 2370 2371 2372
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2373 2374
			 u16 tss_selector, int reason,
			 bool has_error_code, u32 error_code)
2375 2376 2377
{
	int rc;

2378 2379
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
2380

2381
	rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2382
				     has_error_code, error_code);
2383

2384
	if (rc == X86EMUL_CONTINUE)
2385
		ctxt->eip = ctxt->_eip;
2386

2387
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2388 2389
}

2390
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2391
			    int reg, struct operand *op)
2392 2393 2394
{
	int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;

2395 2396
	register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2397
	op->addr.mem.seg = seg;
2398 2399
}

2400 2401 2402 2403 2404 2405
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
2406
	al = ctxt->dst.val;
2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

2424
	ctxt->dst.val = al;
2425
	/* Set PF, ZF, SF */
2426 2427 2428
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
2429
	emulate_2op_SrcV(ctxt, "or");
2430 2431 2432 2433 2434 2435 2436 2437
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

2438 2439 2440 2441 2442 2443
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;

2444
	old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2445
	old_eip = ctxt->_eip;
2446

2447
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2448
	if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2449 2450
		return X86EMUL_CONTINUE;

2451 2452
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2453

2454
	ctxt->src.val = old_cs;
2455
	rc = em_push(ctxt);
2456 2457 2458
	if (rc != X86EMUL_CONTINUE)
		return rc;

2459
	ctxt->src.val = old_eip;
2460
	return em_push(ctxt);
2461 2462
}

2463 2464 2465 2466
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2467 2468 2469 2470
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
	rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2471 2472
	if (rc != X86EMUL_CONTINUE)
		return rc;
2473
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2474 2475 2476
	return X86EMUL_CONTINUE;
}

2477 2478
static int em_add(struct x86_emulate_ctxt *ctxt)
{
2479
	emulate_2op_SrcV(ctxt, "add");
2480 2481 2482 2483 2484
	return X86EMUL_CONTINUE;
}

static int em_or(struct x86_emulate_ctxt *ctxt)
{
2485
	emulate_2op_SrcV(ctxt, "or");
2486 2487 2488 2489 2490
	return X86EMUL_CONTINUE;
}

static int em_adc(struct x86_emulate_ctxt *ctxt)
{
2491
	emulate_2op_SrcV(ctxt, "adc");
2492 2493 2494 2495 2496
	return X86EMUL_CONTINUE;
}

static int em_sbb(struct x86_emulate_ctxt *ctxt)
{
2497
	emulate_2op_SrcV(ctxt, "sbb");
2498 2499 2500 2501 2502
	return X86EMUL_CONTINUE;
}

static int em_and(struct x86_emulate_ctxt *ctxt)
{
2503
	emulate_2op_SrcV(ctxt, "and");
2504 2505 2506 2507 2508
	return X86EMUL_CONTINUE;
}

static int em_sub(struct x86_emulate_ctxt *ctxt)
{
2509
	emulate_2op_SrcV(ctxt, "sub");
2510 2511 2512 2513 2514
	return X86EMUL_CONTINUE;
}

static int em_xor(struct x86_emulate_ctxt *ctxt)
{
2515
	emulate_2op_SrcV(ctxt, "xor");
2516 2517 2518 2519 2520
	return X86EMUL_CONTINUE;
}

static int em_cmp(struct x86_emulate_ctxt *ctxt)
{
2521
	emulate_2op_SrcV(ctxt, "cmp");
2522
	/* Disable writeback. */
2523
	ctxt->dst.type = OP_NONE;
2524 2525 2526
	return X86EMUL_CONTINUE;
}

2527 2528
static int em_test(struct x86_emulate_ctxt *ctxt)
{
2529
	emulate_2op_SrcV(ctxt, "test");
2530 2531
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
2532 2533 2534
	return X86EMUL_CONTINUE;
}

2535 2536 2537
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
2538 2539
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
2540 2541

	/* Write back the memory destination with implicit LOCK prefix. */
2542 2543
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
2544 2545 2546
	return X86EMUL_CONTINUE;
}

2547
static int em_imul(struct x86_emulate_ctxt *ctxt)
2548
{
2549
	emulate_2op_SrcV_nobyte(ctxt, "imul");
2550 2551 2552
	return X86EMUL_CONTINUE;
}

2553 2554
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
2555
	ctxt->dst.val = ctxt->src2.val;
2556 2557 2558
	return em_imul(ctxt);
}

2559 2560
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
2561 2562 2563 2564
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
	ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2565 2566 2567 2568

	return X86EMUL_CONTINUE;
}

2569 2570 2571 2572
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

2573
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2574 2575
	ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
	ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2576 2577 2578
	return X86EMUL_CONTINUE;
}

2579 2580
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
2581
	ctxt->dst.val = ctxt->src.val;
2582 2583 2584
	return X86EMUL_CONTINUE;
}

2585 2586
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
2587
	if (ctxt->modrm_reg > VCPU_SREG_GS)
2588 2589
		return emulate_ud(ctxt);

2590
	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2591 2592 2593 2594 2595
	return X86EMUL_CONTINUE;
}

static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
2596
	u16 sel = ctxt->src.val;
2597

2598
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2599 2600
		return emulate_ud(ctxt);

2601
	if (ctxt->modrm_reg == VCPU_SREG_SS)
2602 2603 2604
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
2605 2606
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2607 2608
}

2609 2610
static int em_movdqu(struct x86_emulate_ctxt *ctxt)
{
2611
	memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2612 2613 2614
	return X86EMUL_CONTINUE;
}

2615 2616
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
2617 2618 2619
	int rc;
	ulong linear;

2620
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2621
	if (rc == X86EMUL_CONTINUE)
2622
		ctxt->ops->invlpg(ctxt, linear);
2623
	/* Disable writeback. */
2624
	ctxt->dst.type = OP_NONE;
2625 2626 2627
	return X86EMUL_CONTINUE;
}

2628 2629 2630 2631 2632 2633 2634 2635 2636 2637
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

2638 2639 2640 2641
static int em_vmcall(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2642
	if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2643 2644 2645 2646 2647 2648 2649
		return X86EMUL_UNHANDLEABLE;

	rc = ctxt->ops->fix_hypercall(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
2650
	ctxt->_eip = ctxt->eip;
2651
	/* Disable writeback. */
2652
	ctxt->dst.type = OP_NONE;
2653 2654 2655 2656 2657 2658 2659 2660
	return X86EMUL_CONTINUE;
}

static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

2661
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2662
			     &desc_ptr.size, &desc_ptr.address,
2663
			     ctxt->op_bytes);
2664 2665 2666 2667
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_gdt(ctxt, &desc_ptr);
	/* Disable writeback. */
2668
	ctxt->dst.type = OP_NONE;
2669 2670 2671
	return X86EMUL_CONTINUE;
}

2672
static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2673 2674 2675
{
	int rc;

2676 2677
	rc = ctxt->ops->fix_hypercall(ctxt);

2678
	/* Disable writeback. */
2679
	ctxt->dst.type = OP_NONE;
2680 2681 2682 2683 2684 2685 2686 2687
	return rc;
}

static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

2688
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2689
			     &desc_ptr.size, &desc_ptr.address,
2690
			     ctxt->op_bytes);
2691 2692 2693 2694
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_idt(ctxt, &desc_ptr);
	/* Disable writeback. */
2695
	ctxt->dst.type = OP_NONE;
2696 2697 2698 2699 2700
	return X86EMUL_CONTINUE;
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
2701 2702
	ctxt->dst.bytes = 2;
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2703 2704 2705 2706 2707 2708
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2709 2710
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
2711 2712 2713
	return X86EMUL_CONTINUE;
}

2714 2715
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
2716 2717 2718 2719
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
	if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
		jmp_rel(ctxt, ctxt->src.val);
2720 2721 2722 2723 2724 2725

	return X86EMUL_CONTINUE;
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
2726 2727
	if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
		jmp_rel(ctxt, ctxt->src.val);
2728 2729 2730 2731

	return X86EMUL_CONTINUE;
}

2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
2765
	if (!valid_cr(ctxt->modrm_reg))
2766 2767 2768 2769 2770 2771 2772
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
2773 2774
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
2775
	u64 efer = 0;
2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
2793
		u64 cr4;
2794 2795 2796 2797
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

2798 2799
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2800 2801 2802 2803 2804 2805 2806 2807 2808 2809

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

2810 2811
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
2812
			rsvd = CR3_L_MODE_RESERVED_BITS;
2813
		else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2814
			rsvd = CR3_PAE_RESERVED_BITS;
2815
		else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2816 2817 2818 2819 2820 2821 2822 2823
			rsvd = CR3_NONPAE_RESERVED_BITS;

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
2824
		u64 cr4;
2825

2826 2827
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

2839 2840 2841 2842
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

2843
	ctxt->ops->get_dr(ctxt, 7, &dr7);
2844 2845 2846 2847 2848 2849 2850

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
2851
	int dr = ctxt->modrm_reg;
2852 2853 2854 2855 2856
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

2857
	cr4 = ctxt->ops->get_cr(ctxt, 4);
2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

	if (check_dr7_gd(ctxt))
		return emulate_db(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
2869 2870
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
2871 2872 2873 2874 2875 2876 2877

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

2878 2879 2880 2881
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
	u64 efer;

2882
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2883 2884 2885 2886 2887 2888 2889 2890 2891

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
2892
	u64 rax = ctxt->regs[VCPU_REGS_RAX];
2893 2894

	/* Valid physical address? */
2895
	if (rax & 0xffff000000000000ULL)
2896 2897 2898 2899 2900
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

2901 2902
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
2903
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2904

2905
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2906 2907 2908 2909 2910
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

2911 2912
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
2913
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2914
	u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2915

2916
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2917 2918 2919 2920 2921 2922
	    (rcx > 3))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

2923 2924
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
2925 2926
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2927 2928 2929 2930 2931 2932 2933
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
2934 2935
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2936 2937 2938 2939 2940
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

2941
#define D(_y) { .flags = (_y) }
2942
#define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2943 2944
#define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
		      .check_perm = (_p) }
2945
#define N    D(0)
2946
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2947
#define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2948
#define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2949
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2950 2951
#define II(_f, _e, _i) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2952 2953 2954
#define IIP(_f, _e, _i, _p) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
	  .check_perm = (_p) }
2955
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2956

2957
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
2958
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2959 2960
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)

2961 2962 2963
#define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2964

2965 2966 2967 2968 2969 2970
static struct opcode group7_rm1[] = {
	DI(SrcNone | ModRM | Priv, monitor),
	DI(SrcNone | ModRM | Priv, mwait),
	N, N, N, N, N, N,
};

2971 2972
static struct opcode group7_rm3[] = {
	DIP(SrcNone | ModRM | Prot | Priv, vmrun,   check_svme_pa),
2973
	II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
2974 2975 2976 2977 2978 2979 2980
	DIP(SrcNone | ModRM | Prot | Priv, vmload,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, vmsave,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, stgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, clgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, skinit,  check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
};
2981

2982 2983 2984 2985 2986
static struct opcode group7_rm7[] = {
	N,
	DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
	N, N, N, N, N, N,
};
2987

2988
static struct opcode group1[] = {
2989 2990 2991 2992 2993 2994 2995 2996
	I(Lock, em_add),
	I(Lock, em_or),
	I(Lock, em_adc),
	I(Lock, em_sbb),
	I(Lock, em_and),
	I(Lock, em_sub),
	I(Lock, em_xor),
	I(0, em_cmp),
2997 2998 2999 3000 3001 3002 3003
};

static struct opcode group1A[] = {
	D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
};

static struct opcode group3[] = {
3004 3005 3006 3007 3008 3009 3010 3011
	I(DstMem | SrcImm | ModRM, em_test),
	I(DstMem | SrcImm | ModRM, em_test),
	I(DstMem | SrcNone | ModRM | Lock, em_not),
	I(DstMem | SrcNone | ModRM | Lock, em_neg),
	I(SrcMem | ModRM, em_mul_ex),
	I(SrcMem | ModRM, em_imul_ex),
	I(SrcMem | ModRM, em_div_ex),
	I(SrcMem | ModRM, em_idiv_ex),
3012 3013 3014 3015 3016 3017 3018 3019 3020
};

static struct opcode group4[] = {
	D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
	N, N, N, N, N, N,
};

static struct opcode group5[] = {
	D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3021 3022
	D(SrcMem | ModRM | Stack),
	I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3023 3024 3025 3026
	D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
	D(SrcMem | ModRM | Stack), N,
};

3027 3028 3029 3030 3031 3032 3033 3034
static struct opcode group6[] = {
	DI(ModRM | Prot,        sldt),
	DI(ModRM | Prot,        str),
	DI(ModRM | Prot | Priv, lldt),
	DI(ModRM | Prot | Priv, ltr),
	N, N, N, N,
};

3035
static struct group_dual group7 = { {
3036 3037
	DI(ModRM | Mov | DstMem | Priv, sgdt),
	DI(ModRM | Mov | DstMem | Priv, sidt),
3038 3039 3040 3041 3042
	II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
	II(ModRM | SrcMem | Priv, em_lidt, lidt),
	II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
	II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
	II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3043
}, {
3044 3045
	I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
	EXT(0, group7_rm1),
3046
	N, EXT(0, group7_rm3),
3047 3048
	II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
	II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062
} };

static struct opcode group8[] = {
	N, N, N, N,
	D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
	D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
};

static struct group_dual group9 = { {
	N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
}, {
	N, N, N, N, N, N, N, N,
} };

3063 3064 3065 3066
static struct opcode group11[] = {
	I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
};

3067 3068 3069 3070
static struct gprefix pfx_0f_6f_0f_7f = {
	N, N, N, I(Sse, em_movdqu),
};

3071 3072
static struct opcode opcode_table[256] = {
	/* 0x00 - 0x07 */
3073
	I6ALU(Lock, em_add),
3074 3075
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	/* 0x08 - 0x0F */
3076
	I6ALU(Lock, em_or),
3077 3078
	D(ImplicitOps | Stack | No64), N,
	/* 0x10 - 0x17 */
3079
	I6ALU(Lock, em_adc),
3080 3081
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	/* 0x18 - 0x1F */
3082
	I6ALU(Lock, em_sbb),
3083 3084
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	/* 0x20 - 0x27 */
3085
	I6ALU(Lock, em_and), N, N,
3086
	/* 0x28 - 0x2F */
3087
	I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3088
	/* 0x30 - 0x37 */
3089
	I6ALU(Lock, em_xor), N, N,
3090
	/* 0x38 - 0x3F */
3091
	I6ALU(0, em_cmp), N, N,
3092 3093 3094
	/* 0x40 - 0x4F */
	X16(D(DstReg)),
	/* 0x50 - 0x57 */
3095
	X8(I(SrcReg | Stack, em_push)),
3096
	/* 0x58 - 0x5F */
3097
	X8(I(DstReg | Stack, em_pop)),
3098
	/* 0x60 - 0x67 */
3099 3100
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
3101 3102 3103
	N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
	N, N, N, N,
	/* 0x68 - 0x6F */
3104 3105
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3106 3107
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3108 3109
	D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
	D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3110 3111 3112 3113 3114 3115 3116
	/* 0x70 - 0x7F */
	X16(D(SrcImmByte)),
	/* 0x80 - 0x87 */
	G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
	G(DstMem | SrcImm | ModRM | Group, group1),
	G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
	G(DstMem | SrcImmByte | ModRM | Group, group1),
3117
	I2bv(DstMem | SrcReg | ModRM, em_test),
3118
	I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3119
	/* 0x88 - 0x8F */
3120 3121
	I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3122 3123 3124 3125
	I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
3126
	/* 0x90 - 0x97 */
3127
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3128
	/* 0x98 - 0x9F */
3129
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3130
	I(SrcImmFAddr | No64, em_call_far), N,
3131 3132
	II(ImplicitOps | Stack, em_pushf, pushf),
	II(ImplicitOps | Stack, em_popf, popf), N, N,
3133
	/* 0xA0 - 0xA7 */
3134 3135 3136
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
	I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
	I2bv(SrcSI | DstDI | Mov | String, em_mov),
3137
	I2bv(SrcSI | DstDI | String, em_cmp),
3138
	/* 0xA8 - 0xAF */
3139
	I2bv(DstAcc | SrcImm, em_test),
3140 3141
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3142
	I2bv(SrcAcc | DstDI | String, em_cmp),
3143
	/* 0xB0 - 0xB7 */
3144
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3145
	/* 0xB8 - 0xBF */
3146
	X8(I(DstReg | SrcImm | Mov, em_mov)),
3147
	/* 0xC0 - 0xC7 */
3148
	D2bv(DstMem | SrcImmByte | ModRM),
3149
	I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3150
	I(ImplicitOps | Stack, em_ret),
3151
	D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3152
	G(ByteOp, group11), G(0, group11),
3153
	/* 0xC8 - 0xCF */
3154
	N, N, N, I(ImplicitOps | Stack, em_ret_far),
3155
	D(ImplicitOps), DI(SrcImmByte, intn),
3156
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3157
	/* 0xD0 - 0xD7 */
3158
	D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3159 3160 3161 3162
	N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
3163 3164
	X3(I(SrcImmByte, em_loop)),
	I(SrcImmByte, em_jcxz),
3165 3166
	D2bvIP(SrcImmUByte | DstAcc, in,  check_perm_in),
	D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3167 3168
	/* 0xE8 - 0xEF */
	D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3169
	I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3170 3171
	D2bvIP(SrcDX | DstAcc, in,  check_perm_in),
	D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3172
	/* 0xF0 - 0xF7 */
3173
	N, DI(ImplicitOps, icebp), N, N,
3174 3175
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
3176
	/* 0xF8 - 0xFF */
3177 3178
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3179 3180 3181 3182 3183
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

static struct opcode twobyte_table[256] = {
	/* 0x00 - 0x0F */
3184
	G(0, group6), GD(0, &group7), N, N,
3185 3186
	N, I(ImplicitOps | VendorSpecific, em_syscall),
	II(ImplicitOps | Priv, em_clts, clts), N,
3187
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3188 3189 3190 3191
	N, D(ImplicitOps | ModRM), N, N,
	/* 0x10 - 0x1F */
	N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
	/* 0x20 - 0x2F */
3192
	DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3193
	DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3194
	DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3195
	DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3196 3197 3198
	N, N, N, N,
	N, N, N, N, N, N, N, N,
	/* 0x30 - 0x3F */
3199 3200 3201 3202
	DI(ImplicitOps | Priv, wrmsr),
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
	DI(ImplicitOps | Priv, rdmsr),
	DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3203 3204
	I(ImplicitOps | VendorSpecific, em_sysenter),
	I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3205
	N, N,
3206 3207 3208 3209 3210 3211
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
	X16(D(DstReg | SrcMem | ModRM | Mov)),
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
3212 3213 3214 3215
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3216
	/* 0x70 - 0x7F */
3217 3218 3219 3220
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3221 3222 3223
	/* 0x80 - 0x8F */
	X16(D(SrcImm)),
	/* 0x90 - 0x9F */
3224
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3225 3226
	/* 0xA0 - 0xA7 */
	D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3227
	DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3228 3229 3230 3231
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM), N, N,
	/* 0xA8 - 0xAF */
	D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3232
	DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3233 3234
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM),
3235
	D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3236
	/* 0xB0 - 0xB7 */
3237
	D2bv(DstMem | SrcReg | ModRM | Lock),
3238 3239 3240
	D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
	D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3241 3242
	/* 0xB8 - 0xBF */
	N, N,
3243
	G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3244 3245
	D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3246
	/* 0xC0 - 0xCF */
3247
	D2bv(DstMem | SrcReg | ModRM | Lock),
3248
	N, D(DstMem | SrcReg | ModRM | Mov),
3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263
	N, N, N, GD(0, &group9),
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

#undef D
#undef N
#undef G
#undef GD
#undef I
3264
#undef GP
3265
#undef EXT
3266

3267
#undef D2bv
3268
#undef D2bvIP
3269
#undef I2bv
3270
#undef I6ALU
3271

3272
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3273 3274 3275
{
	unsigned size;

3276
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
3289
	op->addr.mem.ea = ctxt->_eip;
3290 3291 3292
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
3293
		op->val = insn_fetch(s8, ctxt);
3294 3295
		break;
	case 2:
3296
		op->val = insn_fetch(s16, ctxt);
3297 3298
		break;
	case 4:
3299
		op->val = insn_fetch(s32, ctxt);
3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318
		break;
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

3319
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3320 3321 3322
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
3323
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3324
	bool op_prefix = false;
3325
	struct opcode opcode;
3326
	struct operand memop = { .type = OP_NONE }, *memopp = NULL;
3327

3328 3329 3330
	ctxt->_eip = ctxt->eip;
	ctxt->fetch.start = ctxt->_eip;
	ctxt->fetch.end = ctxt->fetch.start + insn_len;
3331
	if (insn_len > 0)
3332
		memcpy(ctxt->fetch.data, insn, insn_len);
3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
3350
		return EMULATION_FAILED;
3351 3352
	}

3353 3354
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
3355 3356 3357

	/* Legacy prefixes. */
	for (;;) {
3358
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
3359
		case 0x66:	/* operand-size override */
3360
			op_prefix = true;
3361
			/* switch between 2/4 bytes */
3362
			ctxt->op_bytes = def_op_bytes ^ 6;
3363 3364 3365 3366
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
3367
				ctxt->ad_bytes = def_ad_bytes ^ 12;
3368 3369
			else
				/* switch between 2/4 bytes */
3370
				ctxt->ad_bytes = def_ad_bytes ^ 6;
3371 3372 3373 3374 3375
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
3376
			set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3377 3378 3379
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
3380
			set_seg_override(ctxt, ctxt->b & 7);
3381 3382 3383 3384
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
3385
			ctxt->rex_prefix = ctxt->b;
3386 3387
			continue;
		case 0xf0:	/* LOCK */
3388
			ctxt->lock_prefix = 1;
3389 3390 3391
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
3392
			ctxt->rep_prefix = ctxt->b;
3393 3394 3395 3396 3397 3398 3399
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

3400
		ctxt->rex_prefix = 0;
3401 3402 3403 3404 3405
	}

done_prefixes:

	/* REX prefix. */
3406 3407
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
3408 3409

	/* Opcode byte(s). */
3410
	opcode = opcode_table[ctxt->b];
3411
	/* Two-byte opcode? */
3412 3413
	if (ctxt->b == 0x0f) {
		ctxt->twobyte = 1;
3414
		ctxt->b = insn_fetch(u8, ctxt);
3415
		opcode = twobyte_table[ctxt->b];
3416
	}
3417
	ctxt->d = opcode.flags;
3418

3419 3420
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
3421
		case Group:
3422
			ctxt->modrm = insn_fetch(u8, ctxt);
3423 3424
			--ctxt->_eip;
			goffset = (ctxt->modrm >> 3) & 7;
3425 3426 3427
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
3428
			ctxt->modrm = insn_fetch(u8, ctxt);
3429 3430 3431
			--ctxt->_eip;
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
3432 3433 3434 3435 3436
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
3437
			goffset = ctxt->modrm & 7;
3438
			opcode = opcode.u.group[goffset];
3439 3440
			break;
		case Prefix:
3441
			if (ctxt->rep_prefix && op_prefix)
3442
				return EMULATION_FAILED;
3443
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3444 3445 3446 3447 3448 3449 3450 3451
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
		default:
3452
			return EMULATION_FAILED;
3453
		}
3454

3455 3456
		ctxt->d &= ~GroupMask;
		ctxt->d |= opcode.flags;
3457 3458
	}

3459 3460 3461
	ctxt->execute = opcode.u.execute;
	ctxt->check_perm = opcode.check_perm;
	ctxt->intercept = opcode.intercept;
3462 3463

	/* Unrecognised? */
3464
	if (ctxt->d == 0 || (ctxt->d & Undefined))
3465
		return EMULATION_FAILED;
3466

3467
	if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3468
		return EMULATION_FAILED;
3469

3470 3471
	if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
		ctxt->op_bytes = 8;
3472

3473
	if (ctxt->d & Op3264) {
3474
		if (mode == X86EMUL_MODE_PROT64)
3475
			ctxt->op_bytes = 8;
3476
		else
3477
			ctxt->op_bytes = 4;
3478 3479
	}

3480 3481
	if (ctxt->d & Sse)
		ctxt->op_bytes = 16;
A
Avi Kivity 已提交
3482

3483
	/* ModRM and SIB bytes. */
3484
	if (ctxt->d & ModRM) {
3485
		rc = decode_modrm(ctxt, &memop);
3486 3487 3488
		if (!ctxt->has_seg_override)
			set_seg_override(ctxt, ctxt->modrm_seg);
	} else if (ctxt->d & MemAbs)
3489
		rc = decode_abs(ctxt, &memop);
3490 3491 3492
	if (rc != X86EMUL_CONTINUE)
		goto done;

3493 3494
	if (!ctxt->has_seg_override)
		set_seg_override(ctxt, VCPU_SREG_DS);
3495

3496
	memop.addr.mem.seg = seg_override(ctxt);
3497

3498
	if (memop.type == OP_MEM && ctxt->ad_bytes != 8)
3499
		memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3500 3501 3502 3503 3504

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
3505
	switch (ctxt->d & SrcMask) {
3506 3507 3508
	case SrcNone:
		break;
	case SrcReg:
3509
		decode_register_operand(ctxt, &ctxt->src, 0);
3510 3511
		break;
	case SrcMem16:
3512
		memop.bytes = 2;
3513 3514
		goto srcmem_common;
	case SrcMem32:
3515
		memop.bytes = 4;
3516 3517
		goto srcmem_common;
	case SrcMem:
3518 3519
		memop.bytes = (ctxt->d & ByteOp) ? 1 :
							   ctxt->op_bytes;
3520
	srcmem_common:
3521 3522
		ctxt->src = memop;
		memopp = &ctxt->src;
3523
		break;
3524
	case SrcImmU16:
3525
		rc = decode_imm(ctxt, &ctxt->src, 2, false);
3526
		break;
3527
	case SrcImm:
3528
		rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), true);
3529
		break;
3530
	case SrcImmU:
3531
		rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), false);
3532 3533
		break;
	case SrcImmByte:
3534
		rc = decode_imm(ctxt, &ctxt->src, 1, true);
3535
		break;
3536
	case SrcImmUByte:
3537
		rc = decode_imm(ctxt, &ctxt->src, 1, false);
3538 3539
		break;
	case SrcAcc:
3540 3541 3542 3543
		ctxt->src.type = OP_REG;
		ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RAX];
		fetch_register_operand(&ctxt->src);
3544 3545
		break;
	case SrcOne:
3546 3547
		ctxt->src.bytes = 1;
		ctxt->src.val = 1;
3548 3549
		break;
	case SrcSI:
3550 3551 3552 3553 3554 3555
		ctxt->src.type = OP_MEM;
		ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		ctxt->src.addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
		ctxt->src.addr.mem.seg = seg_override(ctxt);
		ctxt->src.val = 0;
3556 3557
		break;
	case SrcImmFAddr:
3558 3559 3560
		ctxt->src.type = OP_IMM;
		ctxt->src.addr.mem.ea = ctxt->_eip;
		ctxt->src.bytes = ctxt->op_bytes + 2;
3561
		insn_fetch_arr(ctxt->src.valptr, ctxt->src.bytes, ctxt);
3562 3563
		break;
	case SrcMemFAddr:
3564
		memop.bytes = ctxt->op_bytes + 2;
3565
		goto srcmem_common;
3566
		break;
3567
	case SrcDX:
3568 3569 3570 3571
		ctxt->src.type = OP_REG;
		ctxt->src.bytes = 2;
		ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
		fetch_register_operand(&ctxt->src);
3572
		break;
3573 3574
	}

3575 3576 3577
	if (rc != X86EMUL_CONTINUE)
		goto done;

3578 3579 3580 3581
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
3582
	switch (ctxt->d & Src2Mask) {
3583 3584 3585
	case Src2None:
		break;
	case Src2CL:
3586
		ctxt->src2.bytes = 1;
3587
		ctxt->src2.val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3588 3589
		break;
	case Src2ImmByte:
3590
		rc = decode_imm(ctxt, &ctxt->src2, 1, true);
3591 3592
		break;
	case Src2One:
3593 3594
		ctxt->src2.bytes = 1;
		ctxt->src2.val = 1;
3595
		break;
3596
	case Src2Imm:
3597
		rc = decode_imm(ctxt, &ctxt->src2, imm_size(ctxt), true);
3598
		break;
3599 3600
	}

3601 3602 3603
	if (rc != X86EMUL_CONTINUE)
		goto done;

3604
	/* Decode and fetch the destination operand: register or memory. */
3605
	switch (ctxt->d & DstMask) {
3606
	case DstReg:
3607 3608
		decode_register_operand(ctxt, &ctxt->dst,
			 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3609
		break;
3610
	case DstImmUByte:
3611 3612 3613
		ctxt->dst.type = OP_IMM;
		ctxt->dst.addr.mem.ea = ctxt->_eip;
		ctxt->dst.bytes = 1;
3614
		ctxt->dst.val = insn_fetch(u8, ctxt);
3615
		break;
3616 3617
	case DstMem:
	case DstMem64:
3618 3619 3620 3621
		ctxt->dst = memop;
		memopp = &ctxt->dst;
		if ((ctxt->d & DstMask) == DstMem64)
			ctxt->dst.bytes = 8;
3622
		else
3623 3624 3625 3626
			ctxt->dst.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		if (ctxt->d & BitOp)
			fetch_bit_operand(ctxt);
		ctxt->dst.orig_val = ctxt->dst.val;
3627 3628
		break;
	case DstAcc:
3629 3630 3631 3632 3633
		ctxt->dst.type = OP_REG;
		ctxt->dst.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RAX];
		fetch_register_operand(&ctxt->dst);
		ctxt->dst.orig_val = ctxt->dst.val;
3634 3635
		break;
	case DstDI:
3636 3637 3638 3639 3640 3641
		ctxt->dst.type = OP_MEM;
		ctxt->dst.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		ctxt->dst.addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
		ctxt->dst.addr.mem.seg = VCPU_SREG_ES;
		ctxt->dst.val = 0;
3642
		break;
3643
	case DstDX:
3644 3645 3646 3647
		ctxt->dst.type = OP_REG;
		ctxt->dst.bytes = 2;
		ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
		fetch_register_operand(&ctxt->dst);
3648
		break;
3649 3650 3651
	case ImplicitOps:
		/* Special instructions do their own operand decoding. */
	default:
3652
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
3653
		break;
3654 3655 3656
	}

done:
3657 3658
	if (memopp && memopp->type == OP_MEM && ctxt->rip_relative)
		memopp->addr.mem.ea += ctxt->_eip;
3659

3660
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3661 3662
}

3663 3664 3665 3666 3667 3668 3669 3670 3671
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
3672 3673 3674
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
3675
		 ((ctxt->eflags & EFLG_ZF) == 0))
3676
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
3677 3678 3679 3680 3681 3682
		    ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
		return true;

	return false;
}

3683
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3684
{
3685
	struct x86_emulate_ops *ops = ctxt->ops;
3686
	u64 msr_data;
3687
	int rc = X86EMUL_CONTINUE;
3688
	int saved_dst_type = ctxt->dst.type;
3689

3690
	ctxt->mem_read.pos = 0;
3691

3692
	if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3693
		rc = emulate_ud(ctxt);
3694 3695 3696
		goto done;
	}

3697
	/* LOCK prefix is allowed only with some instructions */
3698
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3699
		rc = emulate_ud(ctxt);
3700 3701 3702
		goto done;
	}

3703
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3704
		rc = emulate_ud(ctxt);
3705 3706 3707
		goto done;
	}

3708
	if ((ctxt->d & Sse)
3709 3710
	    && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
		|| !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
A
Avi Kivity 已提交
3711 3712 3713 3714
		rc = emulate_ud(ctxt);
		goto done;
	}

3715
	if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
A
Avi Kivity 已提交
3716 3717 3718 3719
		rc = emulate_nm(ctxt);
		goto done;
	}

3720 3721
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3722
					      X86_ICPT_PRE_EXCEPT);
3723 3724 3725 3726
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3727
	/* Privileged instruction can be executed only in CPL=0 */
3728
	if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3729
		rc = emulate_gp(ctxt, 0);
3730 3731 3732
		goto done;
	}

3733
	/* Instruction can only be executed in protected mode */
3734
	if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3735 3736 3737 3738
		rc = emulate_ud(ctxt);
		goto done;
	}

3739
	/* Do instruction specific permission checks */
3740 3741
	if (ctxt->check_perm) {
		rc = ctxt->check_perm(ctxt);
3742 3743 3744 3745
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3746 3747
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3748
					      X86_ICPT_POST_EXCEPT);
3749 3750 3751 3752
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3753
	if (ctxt->rep_prefix && (ctxt->d & String)) {
3754
		/* All REP prefixes have the same first termination condition */
3755 3756
		if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
			ctxt->eip = ctxt->_eip;
3757 3758 3759 3760
			goto done;
		}
	}

3761 3762 3763
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
3764
		if (rc != X86EMUL_CONTINUE)
3765
			goto done;
3766
		ctxt->src.orig_val64 = ctxt->src.val64;
3767 3768
	}

3769 3770 3771
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
3772 3773 3774 3775
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3776
	if ((ctxt->d & DstMask) == ImplicitOps)
3777 3778 3779
		goto special_insn;


3780
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3781
		/* optimisation - avoid slow emulated read if Mov */
3782 3783
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
3784 3785
		if (rc != X86EMUL_CONTINUE)
			goto done;
3786
	}
3787
	ctxt->dst.orig_val = ctxt->dst.val;
3788

3789 3790
special_insn:

3791 3792
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3793
					      X86_ICPT_POST_MEMACCESS);
3794 3795 3796 3797
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3798 3799
	if (ctxt->execute) {
		rc = ctxt->execute(ctxt);
3800 3801 3802 3803 3804
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

3805
	if (ctxt->twobyte)
A
Avi Kivity 已提交
3806 3807
		goto twobyte_insn;

3808
	switch (ctxt->b) {
3809
	case 0x06:		/* push es */
3810
		rc = emulate_push_sreg(ctxt, VCPU_SREG_ES);
3811 3812
		break;
	case 0x07:		/* pop es */
3813
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_ES);
3814 3815
		break;
	case 0x0e:		/* push cs */
3816
		rc = emulate_push_sreg(ctxt, VCPU_SREG_CS);
3817 3818
		break;
	case 0x16:		/* push ss */
3819
		rc = emulate_push_sreg(ctxt, VCPU_SREG_SS);
3820 3821
		break;
	case 0x17:		/* pop ss */
3822
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_SS);
3823 3824
		break;
	case 0x1e:		/* push ds */
3825
		rc = emulate_push_sreg(ctxt, VCPU_SREG_DS);
3826 3827
		break;
	case 0x1f:		/* pop ds */
3828
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_DS);
3829
		break;
3830
	case 0x40 ... 0x47: /* inc r16/r32 */
3831
		emulate_1op(ctxt, "inc");
3832 3833
		break;
	case 0x48 ... 0x4f: /* dec r16/r32 */
3834
		emulate_1op(ctxt, "dec");
3835
		break;
A
Avi Kivity 已提交
3836
	case 0x63:		/* movsxd */
3837
		if (ctxt->mode != X86EMUL_MODE_PROT64)
A
Avi Kivity 已提交
3838
			goto cannot_emulate;
3839
		ctxt->dst.val = (s32) ctxt->src.val;
A
Avi Kivity 已提交
3840
		break;
3841 3842
	case 0x6c:		/* insb */
	case 0x6d:		/* insw/insd */
3843
		ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3844
		goto do_io_in;
3845 3846
	case 0x6e:		/* outsb */
	case 0x6f:		/* outsw/outsd */
3847
		ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3848
		goto do_io_out;
3849
		break;
3850
	case 0x70 ... 0x7f: /* jcc (short) */
3851 3852
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
3853
		break;
N
Nitin A Kamble 已提交
3854
	case 0x8d: /* lea r16/r32, m */
3855
		ctxt->dst.val = ctxt->src.addr.mem.ea;
N
Nitin A Kamble 已提交
3856
		break;
A
Avi Kivity 已提交
3857
	case 0x8f:		/* pop (sole member of Grp1a) */
3858
		rc = em_grp1a(ctxt);
A
Avi Kivity 已提交
3859
		break;
3860
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
3861
		if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3862
			break;
3863 3864
		rc = em_xchg(ctxt);
		break;
3865
	case 0x98: /* cbw/cwde/cdqe */
3866 3867 3868 3869
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3870 3871
		}
		break;
3872
	case 0xc0 ... 0xc1:
3873
		rc = em_grp2(ctxt);
3874
		break;
3875
	case 0xc4:		/* les */
3876
		rc = emulate_load_segment(ctxt, VCPU_SREG_ES);
3877 3878
		break;
	case 0xc5:		/* lds */
3879
		rc = emulate_load_segment(ctxt, VCPU_SREG_DS);
3880
		break;
3881
	case 0xcc:		/* int3 */
3882 3883
		rc = emulate_int(ctxt, 3);
		break;
3884
	case 0xcd:		/* int n */
3885
		rc = emulate_int(ctxt, ctxt->src.val);
3886 3887
		break;
	case 0xce:		/* into */
3888 3889
		if (ctxt->eflags & EFLG_OF)
			rc = emulate_int(ctxt, 4);
3890
		break;
3891
	case 0xd0 ... 0xd1:	/* Grp2 */
3892
		rc = em_grp2(ctxt);
3893 3894
		break;
	case 0xd2 ... 0xd3:	/* Grp2 */
3895
		ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3896
		rc = em_grp2(ctxt);
3897
		break;
3898 3899
	case 0xe4: 	/* inb */
	case 0xe5: 	/* in */
3900
		goto do_io_in;
3901 3902
	case 0xe6: /* outb */
	case 0xe7: /* out */
3903
		goto do_io_out;
3904
	case 0xe8: /* call (near) */ {
3905 3906 3907
		long int rel = ctxt->src.val;
		ctxt->src.val = (unsigned long) ctxt->_eip;
		jmp_rel(ctxt, rel);
3908
		rc = em_push(ctxt);
3909
		break;
3910 3911
	}
	case 0xe9: /* jmp rel */
3912
	case 0xeb: /* jmp rel short */
3913 3914
		jmp_rel(ctxt, ctxt->src.val);
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
3915
		break;
3916 3917
	case 0xec: /* in al,dx */
	case 0xed: /* in (e/r)ax,dx */
3918
	do_io_in:
3919 3920
		if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
				     &ctxt->dst.val))
3921 3922
			goto done; /* IO is needed */
		break;
3923 3924
	case 0xee: /* out dx,al */
	case 0xef: /* out dx,(e/r)ax */
3925
	do_io_out:
3926 3927 3928
		ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				      &ctxt->src.val, 1);
		ctxt->dst.type = OP_NONE;	/* Disable writeback. */
3929
		break;
3930
	case 0xf4:              /* hlt */
3931
		ctxt->ops->halt(ctxt);
3932
		break;
3933 3934 3935 3936 3937 3938 3939
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
		ctxt->eflags ^= EFLG_CF;
		break;
	case 0xf8: /* clc */
		ctxt->eflags &= ~EFLG_CF;
		break;
3940 3941 3942
	case 0xf9: /* stc */
		ctxt->eflags |= EFLG_CF;
		break;
3943 3944 3945 3946 3947 3948
	case 0xfc: /* cld */
		ctxt->eflags &= ~EFLG_DF;
		break;
	case 0xfd: /* std */
		ctxt->eflags |= EFLG_DF;
		break;
3949
	case 0xfe: /* Grp4 */
3950
		rc = em_grp45(ctxt);
3951
		break;
3952
	case 0xff: /* Grp5 */
3953 3954
		rc = em_grp45(ctxt);
		break;
3955 3956
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
3957
	}
3958

3959 3960 3961
	if (rc != X86EMUL_CONTINUE)
		goto done;

3962
writeback:
3963
	rc = writeback(ctxt);
3964
	if (rc != X86EMUL_CONTINUE)
3965 3966
		goto done;

3967 3968 3969 3970
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
3971
	ctxt->dst.type = saved_dst_type;
3972

3973 3974 3975
	if ((ctxt->d & SrcMask) == SrcSI)
		string_addr_inc(ctxt, seg_override(ctxt),
				VCPU_REGS_RSI, &ctxt->src);
3976

3977
	if ((ctxt->d & DstMask) == DstDI)
3978
		string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3979
				&ctxt->dst);
3980

3981 3982 3983
	if (ctxt->rep_prefix && (ctxt->d & String)) {
		struct read_cache *r = &ctxt->io_read;
		register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
3984

3985 3986 3987 3988 3989
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
3990
			if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
3991 3992 3993 3994 3995 3996
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
3997
				ctxt->mem_read.end = 0;
3998 3999 4000
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
4001
		}
4002
	}
4003

4004
	ctxt->eip = ctxt->_eip;
4005 4006

done:
4007 4008
	if (rc == X86EMUL_PROPAGATE_FAULT)
		ctxt->have_exception = true;
4009 4010 4011
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

4012
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
4013 4014

twobyte_insn:
4015
	switch (ctxt->b) {
4016
	case 0x09:		/* wbinvd */
4017
		(ctxt->ops->wbinvd)(ctxt);
4018 4019
		break;
	case 0x08:		/* invd */
4020 4021 4022 4023
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
		break;
	case 0x20: /* mov cr, reg */
4024
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4025
		break;
A
Avi Kivity 已提交
4026
	case 0x21: /* mov from dr to reg */
4027
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
A
Avi Kivity 已提交
4028
		break;
4029
	case 0x22: /* mov reg, cr */
4030
		if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4031
			emulate_gp(ctxt, 0);
4032
			rc = X86EMUL_PROPAGATE_FAULT;
4033 4034
			goto done;
		}
4035
		ctxt->dst.type = OP_NONE;
4036
		break;
A
Avi Kivity 已提交
4037
	case 0x23: /* mov from reg to dr */
4038
		if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4039
				((ctxt->mode == X86EMUL_MODE_PROT64) ?
4040
				 ~0ULL : ~0U)) < 0) {
4041
			/* #UD condition is already handled by the code above */
4042
			emulate_gp(ctxt, 0);
4043
			rc = X86EMUL_PROPAGATE_FAULT;
4044 4045 4046
			goto done;
		}

4047
		ctxt->dst.type = OP_NONE;	/* no writeback */
A
Avi Kivity 已提交
4048
		break;
4049 4050
	case 0x30:
		/* wrmsr */
4051 4052 4053
		msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
			| ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
		if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4054
			emulate_gp(ctxt, 0);
4055
			rc = X86EMUL_PROPAGATE_FAULT;
4056
			goto done;
4057 4058 4059 4060 4061
		}
		rc = X86EMUL_CONTINUE;
		break;
	case 0x32:
		/* rdmsr */
4062
		if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4063
			emulate_gp(ctxt, 0);
4064
			rc = X86EMUL_PROPAGATE_FAULT;
4065
			goto done;
4066
		} else {
4067 4068
			ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
			ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4069 4070 4071
		}
		rc = X86EMUL_CONTINUE;
		break;
A
Avi Kivity 已提交
4072
	case 0x40 ... 0x4f:	/* cmov */
4073 4074 4075
		ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
		if (!test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
4076
		break;
4077
	case 0x80 ... 0x8f: /* jnz rel, etc*/
4078 4079
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
4080
		break;
4081
	case 0x90 ... 0x9f:     /* setcc r/m8 */
4082
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4083
		break;
4084
	case 0xa0:	  /* push fs */
4085
		rc = emulate_push_sreg(ctxt, VCPU_SREG_FS);
4086 4087
		break;
	case 0xa1:	 /* pop fs */
4088
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_FS);
4089
		break;
4090 4091
	case 0xa3:
	      bt:		/* bt */
4092
		ctxt->dst.type = OP_NONE;
4093
		/* only subword offset */
4094
		ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4095
		emulate_2op_SrcV_nobyte(ctxt, "bt");
4096
		break;
4097 4098
	case 0xa4: /* shld imm8, r, r/m */
	case 0xa5: /* shld cl, r, r/m */
4099
		emulate_2op_cl(ctxt, "shld");
4100
		break;
4101
	case 0xa8:	/* push gs */
4102
		rc = emulate_push_sreg(ctxt, VCPU_SREG_GS);
4103 4104
		break;
	case 0xa9:	/* pop gs */
4105
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_GS);
4106
		break;
4107 4108
	case 0xab:
	      bts:		/* bts */
4109
		emulate_2op_SrcV_nobyte(ctxt, "bts");
4110
		break;
4111 4112
	case 0xac: /* shrd imm8, r, r/m */
	case 0xad: /* shrd cl, r, r/m */
4113
		emulate_2op_cl(ctxt, "shrd");
4114
		break;
4115 4116
	case 0xae:              /* clflush */
		break;
A
Avi Kivity 已提交
4117 4118 4119 4120 4121
	case 0xb0 ... 0xb1:	/* cmpxchg */
		/*
		 * Save real source value, then compare EAX against
		 * destination.
		 */
4122 4123
		ctxt->src.orig_val = ctxt->src.val;
		ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4124
		emulate_2op_SrcV(ctxt, "cmp");
4125
		if (ctxt->eflags & EFLG_ZF) {
A
Avi Kivity 已提交
4126
			/* Success: write back to memory. */
4127
			ctxt->dst.val = ctxt->src.orig_val;
A
Avi Kivity 已提交
4128 4129
		} else {
			/* Failure: write the value we saw to EAX. */
4130 4131
			ctxt->dst.type = OP_REG;
			ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
A
Avi Kivity 已提交
4132 4133
		}
		break;
4134
	case 0xb2:		/* lss */
4135
		rc = emulate_load_segment(ctxt, VCPU_SREG_SS);
4136
		break;
A
Avi Kivity 已提交
4137 4138
	case 0xb3:
	      btr:		/* btr */
4139
		emulate_2op_SrcV_nobyte(ctxt, "btr");
A
Avi Kivity 已提交
4140
		break;
4141
	case 0xb4:		/* lfs */
4142
		rc = emulate_load_segment(ctxt, VCPU_SREG_FS);
4143 4144
		break;
	case 0xb5:		/* lgs */
4145
		rc = emulate_load_segment(ctxt, VCPU_SREG_GS);
4146
		break;
A
Avi Kivity 已提交
4147
	case 0xb6 ... 0xb7:	/* movzx */
4148 4149 4150
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
						       : (u16) ctxt->src.val;
A
Avi Kivity 已提交
4151 4152
		break;
	case 0xba:		/* Grp8 */
4153
		switch (ctxt->modrm_reg & 3) {
A
Avi Kivity 已提交
4154 4155 4156 4157 4158 4159 4160 4161 4162 4163
		case 0:
			goto bt;
		case 1:
			goto bts;
		case 2:
			goto btr;
		case 3:
			goto btc;
		}
		break;
4164 4165
	case 0xbb:
	      btc:		/* btc */
4166
		emulate_2op_SrcV_nobyte(ctxt, "btc");
4167
		break;
4168 4169 4170
	case 0xbc: {		/* bsf */
		u8 zf;
		__asm__ ("bsf %2, %0; setz %1"
4171 4172
			 : "=r"(ctxt->dst.val), "=q"(zf)
			 : "r"(ctxt->src.val));
4173 4174 4175
		ctxt->eflags &= ~X86_EFLAGS_ZF;
		if (zf) {
			ctxt->eflags |= X86_EFLAGS_ZF;
4176
			ctxt->dst.type = OP_NONE;	/* Disable writeback. */
4177 4178 4179 4180 4181 4182
		}
		break;
	}
	case 0xbd: {		/* bsr */
		u8 zf;
		__asm__ ("bsr %2, %0; setz %1"
4183 4184
			 : "=r"(ctxt->dst.val), "=q"(zf)
			 : "r"(ctxt->src.val));
4185 4186 4187
		ctxt->eflags &= ~X86_EFLAGS_ZF;
		if (zf) {
			ctxt->eflags |= X86_EFLAGS_ZF;
4188
			ctxt->dst.type = OP_NONE;	/* Disable writeback. */
4189 4190 4191
		}
		break;
	}
A
Avi Kivity 已提交
4192
	case 0xbe ... 0xbf:	/* movsx */
4193 4194 4195
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
							(s16) ctxt->src.val;
A
Avi Kivity 已提交
4196
		break;
4197
	case 0xc0 ... 0xc1:	/* xadd */
4198
		emulate_2op_SrcV(ctxt, "add");
4199
		/* Write back the register source. */
4200 4201
		ctxt->src.val = ctxt->dst.orig_val;
		write_register_operand(&ctxt->src);
4202
		break;
4203
	case 0xc3:		/* movnti */
4204 4205 4206
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
							(u64) ctxt->src.val;
4207
		break;
A
Avi Kivity 已提交
4208
	case 0xc7:		/* Grp9 (cmpxchg8b) */
4209
		rc = em_grp9(ctxt);
4210
		break;
4211 4212
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4213
	}
4214 4215 4216 4217

	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
4218 4219 4220
	goto writeback;

cannot_emulate:
4221
	return EMULATION_FAILED;
A
Avi Kivity 已提交
4222
}