emulate.c 107.5 KB
Newer Older
A
Avi Kivity 已提交
1
/******************************************************************************
2
 * emulate.c
A
Avi Kivity 已提交
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
A
Avi Kivity 已提交
10 11
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
A
Avi Kivity 已提交
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
A
Avi Kivity 已提交
27

28
#include "x86.h"
29
#include "tss.h"
30

31 32 33
/*
 * Operand types
 */
34 35 36 37 38 39 40 41 42
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
43 44 45 46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
#define OpImm             12ull  /* Sign extended immediate */
47 48 49 50 51 52 53 54 55
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */

#define OpBits             5  /* Width of operand field */
56
#define OpMask             ((1ull << OpBits) - 1)
57

A
Avi Kivity 已提交
58 59 60 61 62 63 64 65 66 67
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
68
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
69
/* Destination operand type. */
70 71 72 73 74 75 76 77 78 79
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
#define DstMask     (OpMask << DstShift)
A
Avi Kivity 已提交
80
/* Source operand type. */
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
#define SrcDX       (OpDX << SrcShift)
#define SrcMask     (OpMask << SrcShift)
99 100 101 102 103 104 105 106 107 108
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
#define Sse         (1<<18)     /* SSE Vector instruction */
109 110 111 112
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
113
/* Misc flags */
114
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
115
#define VendorSpecific (1<<22) /* Vendor specific instruction */
116
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
117
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
118
#define Undefined   (1<<25) /* No Such Instruction */
119
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
120
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
121
#define No64	    (1<<28)
122
/* Source 2 operand type */
123 124 125 126 127 128 129
#define Src2Shift   (29)
#define Src2None    (OpNone << Src2Shift)
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
#define Src2Mask    (OpMask << Src2Shift)
A
Avi Kivity 已提交
130

131 132 133 134 135 136 137 138
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
139

140
struct opcode {
141 142
	u64 flags : 56;
	u64 intercept : 8;
143
	union {
144
		int (*execute)(struct x86_emulate_ctxt *ctxt);
145 146
		struct opcode *group;
		struct group_dual *gdual;
147
		struct gprefix *gprefix;
148
	} u;
149
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
150 151 152 153 154
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
155 156
};

157 158 159 160 161 162 163
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

A
Avi Kivity 已提交
164
/* EFLAGS bit definitions. */
165 166 167 168
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
169 170
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
171 172
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
A
Avi Kivity 已提交
173 174
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
175
#define EFLG_IF (1<<9)
176
#define EFLG_TF (1<<8)
A
Avi Kivity 已提交
177 178 179 180 181 182
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

183 184 185
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

A
Avi Kivity 已提交
186 187 188 189 190 191 192
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

193
#if defined(CONFIG_X86_64)
A
Avi Kivity 已提交
194 195 196 197 198 199 200 201 202 203 204 205 206 207
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
A
Avi Kivity 已提交
223 224 225 226 227 228 229 230 231

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

232 233 234 235 236 237
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

238
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
239 240 241 242 243
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
244 245
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
246
			  "=&r" (_tmp)					\
247
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
248
	} while (0)
249 250


A
Avi Kivity 已提交
251
/* Raw emulation: instruction has two explicit operands. */
252
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
253 254 255
	do {								\
		unsigned long _tmp;					\
									\
256
		switch ((ctxt)->dst.bytes) {				\
257
		case 2:							\
258
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
259 260
			break;						\
		case 4:							\
261
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
262 263
			break;						\
		case 8:							\
264
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
265 266
			break;						\
		}							\
A
Avi Kivity 已提交
267 268
	} while (0)

269
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
A
Avi Kivity 已提交
270
	do {								     \
271
		unsigned long _tmp;					     \
272
		switch ((ctxt)->dst.bytes) {				     \
A
Avi Kivity 已提交
273
		case 1:							     \
274
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
A
Avi Kivity 已提交
275 276
			break;						     \
		default:						     \
277
			__emulate_2op_nobyte(ctxt, _op,			     \
A
Avi Kivity 已提交
278 279 280 281 282 283
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
284 285
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
A
Avi Kivity 已提交
286 287

/* Source operand is byte, word, long or quad sized. */
288 289
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
290 291

/* Source operand is word, long or quad sized. */
292 293
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
294

295
/* Instruction has three operands and one operand is stored in ECX register */
296
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
297 298
	do {								\
		unsigned long _tmp;					\
299 300 301
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
302 303 304 305 306
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
307
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
308 309 310
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
311 312 313
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
314 315
	} while (0)

316
#define emulate_2op_cl(ctxt, _op)					\
317
	do {								\
318
		switch ((ctxt)->dst.bytes) {				\
319
		case 2:							\
320
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
321 322
			break;						\
		case 4:							\
323
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
324 325
			break;						\
		case 8:							\
326
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
327 328
			break;						\
		}							\
329 330
	} while (0)

331
#define __emulate_1op(ctxt, _op, _suffix)				\
A
Avi Kivity 已提交
332 333 334
	do {								\
		unsigned long _tmp;					\
									\
335 336 337 338
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
339
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
340 341 342 343 344
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
345
#define emulate_1op(ctxt, _op)						\
346
	do {								\
347 348 349 350 351
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
A
Avi Kivity 已提交
352 353 354
		}							\
	} while (0)

355
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
356 357
	do {								\
		unsigned long _tmp;					\
358 359
		ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX];		\
		ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX];		\
360 361 362 363 364 365 366 367 368 369 370 371
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
372 373 374 375
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val),	\
			  "a" (*rax), "d" (*rdx));			\
376 377
	} while (0)

378
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
379
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
380
	do {								\
381
		switch((ctxt)->src.bytes) {				\
382
		case 1:							\
383
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
384 385
			break;						\
		case 2:							\
386
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
387 388
			break;						\
		case 4:							\
389
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
390 391
			break;						\
		case 8: ON64(						\
392
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
393 394 395 396
			break;						\
		}							\
	} while (0)

397 398 399 400 401 402
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
403 404 405 406 407 408 409 410
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
411 412 413
		.next_rip   = ctxt->eip,
	};

414
	return ctxt->ops->intercept(ctxt, &info, stage);
415 416
}

417
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
418
{
419
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
420 421
}

A
Avi Kivity 已提交
422
/* Access/update address held in a register, based on addressing mode. */
423
static inline unsigned long
424
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
425
{
426
	if (ctxt->ad_bytes == sizeof(unsigned long))
427 428
		return reg;
	else
429
		return reg & ad_mask(ctxt);
430 431 432
}

static inline unsigned long
433
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
434
{
435
	return address_mask(ctxt, reg);
436 437
}

438
static inline void
439
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
440
{
441
	if (ctxt->ad_bytes == sizeof(unsigned long))
442 443
		*reg += inc;
	else
444
		*reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
445
}
A
Avi Kivity 已提交
446

447
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
448
{
449
	register_address_increment(ctxt, &ctxt->_eip, rel);
450
}
451

452 453 454 455 456 457 458
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

459
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
460
{
461 462
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
463 464
}

465
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
466 467 468 469
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

470
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
471 472
}

473
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
474
{
475
	if (!ctxt->has_seg_override)
476 477
		return 0;

478
	return ctxt->seg_override;
479 480
}

481 482
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
483
{
484 485 486
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
487
	return X86EMUL_PROPAGATE_FAULT;
488 489
}

490 491 492 493 494
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

495
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
496
{
497
	return emulate_exception(ctxt, GP_VECTOR, err, true);
498 499
}

500 501 502 503 504
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

505
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
506
{
507
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
508 509
}

510
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
511
{
512
	return emulate_exception(ctxt, TS_VECTOR, err, true);
513 514
}

515 516
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
517
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
518 519
}

A
Avi Kivity 已提交
520 521 522 523 524
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

545
static int __linearize(struct x86_emulate_ctxt *ctxt,
546
		     struct segmented_address addr,
547
		     unsigned size, bool write, bool fetch,
548 549
		     ulong *linear)
{
550 551
	struct desc_struct desc;
	bool usable;
552
	ulong la;
553
	u32 lim;
554
	u16 sel;
555
	unsigned cpl, rpl;
556

557
	la = seg_base(ctxt, addr.seg) + addr.ea;
558 559 560 561 562 563 564 565
	switch (ctxt->mode) {
	case X86EMUL_MODE_REAL:
		break;
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
566 567
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
568 569 570 571 572 573
		if (!usable)
			goto bad;
		/* code segment or read-only data segment */
		if (((desc.type & 8) || !(desc.type & 2)) && write)
			goto bad;
		/* unreadable code segment */
574
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
575 576 577 578 579 580 581 582 583 584 585 586 587 588
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
			/* exapand-down segment */
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
589
		cpl = ctxt->ops->cpl(ctxt);
590
		rpl = sel & 3;
591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
		cpl = max(cpl, rpl);
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
607
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
608 609 610
		la &= (u32)-1;
	*linear = la;
	return X86EMUL_CONTINUE;
611 612 613 614 615
bad:
	if (addr.seg == VCPU_SREG_SS)
		return emulate_ss(ctxt, addr.seg);
	else
		return emulate_gp(ctxt, addr.seg);
616 617
}

618 619 620 621 622 623 624 625 626
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


627 628 629 630 631
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
632 633 634
	int rc;
	ulong linear;

635
	rc = linearize(ctxt, addr, size, false, &linear);
636 637
	if (rc != X86EMUL_CONTINUE)
		return rc;
638
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
639 640
}

641 642 643 644 645 646 647 648
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
649
{
650
	struct fetch_cache *fc = &ctxt->fetch;
651
	int rc;
652
	int size, cur_size;
653

654
	if (ctxt->_eip == fc->end) {
655
		unsigned long linear;
656 657
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
658
		cur_size = fc->end - fc->start;
659 660
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
661
		rc = __linearize(ctxt, addr, size, false, true, &linear);
662
		if (unlikely(rc != X86EMUL_CONTINUE))
663
			return rc;
664 665
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
666
		if (unlikely(rc != X86EMUL_CONTINUE))
667
			return rc;
668
		fc->end += size;
669
	}
670 671
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
672
	return X86EMUL_CONTINUE;
673 674 675
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
676
			 void *dest, unsigned size)
677
{
678
	int rc;
679

680
	/* x86 instructions are limited to 15 bytes. */
681
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
682
		return X86EMUL_UNHANDLEABLE;
683
	while (size--) {
684
		rc = do_insn_fetch_byte(ctxt, dest++);
685
		if (rc != X86EMUL_CONTINUE)
686 687
			return rc;
	}
688
	return X86EMUL_CONTINUE;
689 690
}

691
/* Fetch next part of the instruction being emulated. */
692
#define insn_fetch(_type, _ctxt)					\
693
({	unsigned long _x;						\
694
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
695 696 697 698 699
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

700 701
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
702 703 704 705
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

706 707 708 709 710 711 712
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
static void *decode_register(u8 modrm_reg, unsigned long *regs,
			     int highbyte_regs)
A
Avi Kivity 已提交
713 714 715 716 717 718 719 720 721 722
{
	void *p;

	p = &regs[modrm_reg];
	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
		p = (unsigned char *)&regs[modrm_reg & 3] + 1;
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
723
			   struct segmented_address addr,
A
Avi Kivity 已提交
724 725 726 727 728 729 730
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
731
	rc = segmented_read_std(ctxt, addr, size, 2);
732
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
733
		return rc;
734
	addr.ea += 2;
735
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
736 737 738
	return rc;
}

739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
static int test_cc(unsigned int condition, unsigned int flags)
{
	int rc = 0;

	switch ((condition & 15) >> 1) {
	case 0: /* o */
		rc |= (flags & EFLG_OF);
		break;
	case 1: /* b/c/nae */
		rc |= (flags & EFLG_CF);
		break;
	case 2: /* z/e */
		rc |= (flags & EFLG_ZF);
		break;
	case 3: /* be/na */
		rc |= (flags & (EFLG_CF|EFLG_ZF));
		break;
	case 4: /* s */
		rc |= (flags & EFLG_SF);
		break;
	case 5: /* p/pe */
		rc |= (flags & EFLG_PF);
		break;
	case 7: /* le/ng */
		rc |= (flags & EFLG_ZF);
		/* fall through */
	case 6: /* l/nge */
		rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
		break;
	}

	/* Odd condition identifiers (lsb == 1) have inverted sense. */
	return (!!rc ^ (condition & 1));
}

774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

A
Avi Kivity 已提交
792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
				    struct operand *op,
849 850
				    int inhibit_bytereg)
{
851 852
	unsigned reg = ctxt->modrm_reg;
	int highbyte_regs = ctxt->rex_prefix == 0;
853

854 855
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
856

857
	if (ctxt->d & Sse) {
A
Avi Kivity 已提交
858 859 860 861 862 863 864
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}

865
	op->type = OP_REG;
866 867
	if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
		op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
868 869
		op->bytes = 1;
	} else {
870 871
		op->addr.reg = decode_register(reg, ctxt->regs, 0);
		op->bytes = ctxt->op_bytes;
872
	}
873
	fetch_register_operand(op);
874 875 876
	op->orig_val = op->val;
}

877
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
878
			struct operand *op)
879 880
{
	u8 sib;
881
	int index_reg = 0, base_reg = 0, scale;
882
	int rc = X86EMUL_CONTINUE;
883
	ulong modrm_ea = 0;
884

885 886 887 888
	if (ctxt->rex_prefix) {
		ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1;	/* REX.R */
		index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
		ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
889 890
	}

891
	ctxt->modrm = insn_fetch(u8, ctxt);
892 893 894 895
	ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
	ctxt->modrm_rm |= (ctxt->modrm & 0x07);
	ctxt->modrm_seg = VCPU_SREG_DS;
896

897
	if (ctxt->modrm_mod == 3) {
898
		op->type = OP_REG;
899 900 901 902
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.reg = decode_register(ctxt->modrm_rm,
					       ctxt->regs, ctxt->d & ByteOp);
		if (ctxt->d & Sse) {
A
Avi Kivity 已提交
903 904
			op->type = OP_XMM;
			op->bytes = 16;
905 906
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
A
Avi Kivity 已提交
907 908
			return rc;
		}
909
		fetch_register_operand(op);
910 911 912
		return rc;
	}

913 914
	op->type = OP_MEM;

915 916 917 918 919
	if (ctxt->ad_bytes == 2) {
		unsigned bx = ctxt->regs[VCPU_REGS_RBX];
		unsigned bp = ctxt->regs[VCPU_REGS_RBP];
		unsigned si = ctxt->regs[VCPU_REGS_RSI];
		unsigned di = ctxt->regs[VCPU_REGS_RDI];
920 921

		/* 16-bit ModR/M decode. */
922
		switch (ctxt->modrm_mod) {
923
		case 0:
924
			if (ctxt->modrm_rm == 6)
925
				modrm_ea += insn_fetch(u16, ctxt);
926 927
			break;
		case 1:
928
			modrm_ea += insn_fetch(s8, ctxt);
929 930
			break;
		case 2:
931
			modrm_ea += insn_fetch(u16, ctxt);
932 933
			break;
		}
934
		switch (ctxt->modrm_rm) {
935
		case 0:
936
			modrm_ea += bx + si;
937 938
			break;
		case 1:
939
			modrm_ea += bx + di;
940 941
			break;
		case 2:
942
			modrm_ea += bp + si;
943 944
			break;
		case 3:
945
			modrm_ea += bp + di;
946 947
			break;
		case 4:
948
			modrm_ea += si;
949 950
			break;
		case 5:
951
			modrm_ea += di;
952 953
			break;
		case 6:
954
			if (ctxt->modrm_mod != 0)
955
				modrm_ea += bp;
956 957
			break;
		case 7:
958
			modrm_ea += bx;
959 960
			break;
		}
961 962 963
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
964
		modrm_ea = (u16)modrm_ea;
965 966
	} else {
		/* 32/64-bit ModR/M decode. */
967
		if ((ctxt->modrm_rm & 7) == 4) {
968
			sib = insn_fetch(u8, ctxt);
969 970 971 972
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

973
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
974
				modrm_ea += insn_fetch(s32, ctxt);
975
			else
976
				modrm_ea += ctxt->regs[base_reg];
977
			if (index_reg != 4)
978 979
				modrm_ea += ctxt->regs[index_reg] << scale;
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
980
			if (ctxt->mode == X86EMUL_MODE_PROT64)
981
				ctxt->rip_relative = 1;
982
		} else
983 984
			modrm_ea += ctxt->regs[ctxt->modrm_rm];
		switch (ctxt->modrm_mod) {
985
		case 0:
986
			if (ctxt->modrm_rm == 5)
987
				modrm_ea += insn_fetch(s32, ctxt);
988 989
			break;
		case 1:
990
			modrm_ea += insn_fetch(s8, ctxt);
991 992
			break;
		case 2:
993
			modrm_ea += insn_fetch(s32, ctxt);
994 995 996
			break;
		}
	}
997
	op->addr.mem.ea = modrm_ea;
998 999 1000 1001 1002
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
1003
		      struct operand *op)
1004
{
1005
	int rc = X86EMUL_CONTINUE;
1006

1007
	op->type = OP_MEM;
1008
	switch (ctxt->ad_bytes) {
1009
	case 2:
1010
		op->addr.mem.ea = insn_fetch(u16, ctxt);
1011 1012
		break;
	case 4:
1013
		op->addr.mem.ea = insn_fetch(u32, ctxt);
1014 1015
		break;
	case 8:
1016
		op->addr.mem.ea = insn_fetch(u64, ctxt);
1017 1018 1019 1020 1021 1022
		break;
	}
done:
	return rc;
}

1023
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1024
{
1025
	long sv = 0, mask;
1026

1027 1028
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
		mask = ~(ctxt->dst.bytes * 8 - 1);
1029

1030 1031 1032 1033
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1034

1035
		ctxt->dst.addr.mem.ea += (sv >> 3);
1036
	}
1037 1038

	/* only subword offset */
1039
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1040 1041
}

1042 1043
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1044
{
1045
	int rc;
1046
	struct read_cache *mc = &ctxt->mem_read;
A
Avi Kivity 已提交
1047

1048 1049 1050 1051 1052
	while (size) {
		int n = min(size, 8u);
		size -= n;
		if (mc->pos < mc->end)
			goto read_cached;
1053

1054 1055
		rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
					      &ctxt->exception);
1056 1057 1058
		if (rc != X86EMUL_CONTINUE)
			return rc;
		mc->end += n;
A
Avi Kivity 已提交
1059

1060 1061 1062 1063 1064
	read_cached:
		memcpy(dest, mc->data + mc->pos, n);
		mc->pos += n;
		dest += n;
		addr += n;
A
Avi Kivity 已提交
1065
	}
1066 1067
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1068

1069 1070 1071 1072 1073
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1074 1075 1076
	int rc;
	ulong linear;

1077
	rc = linearize(ctxt, addr, size, false, &linear);
1078 1079
	if (rc != X86EMUL_CONTINUE)
		return rc;
1080
	return read_emulated(ctxt, linear, data, size);
1081 1082 1083 1084 1085 1086 1087
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1088 1089 1090
	int rc;
	ulong linear;

1091
	rc = linearize(ctxt, addr, size, true, &linear);
1092 1093
	if (rc != X86EMUL_CONTINUE)
		return rc;
1094 1095
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1096 1097 1098 1099 1100 1101 1102
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1103 1104 1105
	int rc;
	ulong linear;

1106
	rc = linearize(ctxt, addr, size, true, &linear);
1107 1108
	if (rc != X86EMUL_CONTINUE)
		return rc;
1109 1110
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1111 1112
}

1113 1114 1115 1116
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1117
	struct read_cache *rc = &ctxt->io_read;
1118

1119 1120
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1121 1122
		unsigned int count = ctxt->rep_prefix ?
			address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1123
		in_page = (ctxt->eflags & EFLG_DF) ?
1124 1125
			offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
			PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1126 1127 1128 1129 1130
		n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
			count);
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1131
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1132 1133
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1134 1135
	}

1136 1137 1138 1139
	memcpy(dest, rc->data + rc->pos, size);
	rc->pos += size;
	return 1;
}
A
Avi Kivity 已提交
1140

1141 1142 1143
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1144 1145
	struct x86_emulate_ops *ops = ctxt->ops;

1146 1147
	if (selector & 1 << 2) {
		struct desc_struct desc;
1148 1149
		u16 sel;

1150
		memset (dt, 0, sizeof *dt);
1151
		if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1152
			return;
1153

1154 1155 1156
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
		dt->address = get_desc_base(&desc);
	} else
1157
		ops->get_gdt(ctxt, dt);
1158
}
1159

1160 1161 1162 1163 1164 1165 1166
/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1167

1168
	get_descriptor_table_ptr(ctxt, selector, &dt);
1169

1170 1171
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1172

1173 1174 1175
	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
1176
}
1177

1178 1179 1180 1181 1182 1183 1184
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
A
Avi Kivity 已提交
1185

1186
	get_descriptor_table_ptr(ctxt, selector, &dt);
1187

1188 1189
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
A
Avi Kivity 已提交
1190

1191
	addr = dt.address + index * 8;
1192 1193
	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
				    &ctxt->exception);
1194
}
1195

1196
/* Does not support long mode */
1197 1198 1199 1200 1201 1202 1203 1204 1205
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
	struct desc_struct seg_desc;
	u8 dpl, rpl, cpl;
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
	int ret;
1206

1207
	memset(&seg_desc, 0, sizeof seg_desc);
1208

1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231
	if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
	    || ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
		goto load;
	}

	/* NULL selector is not valid for TR, CS and SS */
	if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
	    && null_selector)
		goto exception;

	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

	if (null_selector) /* for NULL selector skip all following checks */
		goto load;

1232
	ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
	err_vec = GP_VECTOR;

	/* can't load system descriptor into segment selecor */
	if (seg <= VCPU_SREG_GS && !seg_desc.s)
		goto exception;

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	rpl = selector & 3;
	dpl = seg_desc.dpl;
1250
	cpl = ctxt->ops->cpl(ctxt);
1251 1252 1253 1254 1255 1256 1257 1258 1259

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1260
		break;
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1276
		break;
1277 1278 1279 1280 1281 1282 1283 1284 1285
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1286
		/*
1287 1288 1289
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1290
		 */
1291 1292 1293 1294
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1295
		break;
1296 1297 1298 1299 1300
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
		seg_desc.type |= 1;
1301
		ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1302 1303 1304 1305
		if (ret != X86EMUL_CONTINUE)
			return ret;
	}
load:
1306
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1307 1308 1309 1310 1311 1312
	return X86EMUL_CONTINUE;
exception:
	emulate_exception(ctxt, err_vec, err_code, true);
	return X86EMUL_PROPAGATE_FAULT;
}

1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331
static void write_register_operand(struct operand *op)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (op->bytes) {
	case 1:
		*(u8 *)op->addr.reg = (u8)op->val;
		break;
	case 2:
		*(u16 *)op->addr.reg = (u16)op->val;
		break;
	case 4:
		*op->addr.reg = (u32)op->val;
		break;	/* 64b: zero-extend */
	case 8:
		*op->addr.reg = op->val;
		break;
	}
}

1332
static int writeback(struct x86_emulate_ctxt *ctxt)
1333 1334 1335
{
	int rc;

1336
	switch (ctxt->dst.type) {
1337
	case OP_REG:
1338
		write_register_operand(&ctxt->dst);
A
Avi Kivity 已提交
1339
		break;
1340
	case OP_MEM:
1341
		if (ctxt->lock_prefix)
1342
			rc = segmented_cmpxchg(ctxt,
1343 1344 1345 1346
					       ctxt->dst.addr.mem,
					       &ctxt->dst.orig_val,
					       &ctxt->dst.val,
					       ctxt->dst.bytes);
1347
		else
1348
			rc = segmented_write(ctxt,
1349 1350 1351
					     ctxt->dst.addr.mem,
					     &ctxt->dst.val,
					     ctxt->dst.bytes);
1352 1353
		if (rc != X86EMUL_CONTINUE)
			return rc;
1354
		break;
A
Avi Kivity 已提交
1355
	case OP_XMM:
1356
		write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
A
Avi Kivity 已提交
1357
		break;
1358 1359
	case OP_NONE:
		/* no writeback */
1360
		break;
1361
	default:
1362
		break;
A
Avi Kivity 已提交
1363
	}
1364 1365
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1366

1367
static int em_push(struct x86_emulate_ctxt *ctxt)
1368
{
1369
	struct segmented_address addr;
1370

1371 1372
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1373 1374 1375
	addr.seg = VCPU_SREG_SS;

	/* Disable writeback. */
1376 1377
	ctxt->dst.type = OP_NONE;
	return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1378
}
1379

1380 1381 1382 1383
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1384
	struct segmented_address addr;
1385

1386
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1387
	addr.seg = VCPU_SREG_SS;
1388
	rc = segmented_read(ctxt, addr, dest, len);
1389 1390 1391
	if (rc != X86EMUL_CONTINUE)
		return rc;

1392
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1393
	return rc;
1394 1395
}

1396 1397
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1398
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1399 1400
}

1401
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1402
			void *dest, int len)
1403 1404
{
	int rc;
1405 1406
	unsigned long val, change_mask;
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1407
	int cpl = ctxt->ops->cpl(ctxt);
1408

1409
	rc = emulate_pop(ctxt, &val, len);
1410 1411
	if (rc != X86EMUL_CONTINUE)
		return rc;
1412

1413 1414
	change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
		| EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1415

1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
			change_mask |= EFLG_IOPL;
		if (cpl <= iopl)
			change_mask |= EFLG_IF;
		break;
	case X86EMUL_MODE_VM86:
1426 1427
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1428 1429 1430 1431 1432
		change_mask |= EFLG_IF;
		break;
	default: /* real mode */
		change_mask |= (EFLG_IOPL | EFLG_IF);
		break;
1433
	}
1434 1435 1436 1437 1438

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1439 1440
}

1441 1442
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1443 1444 1445 1446
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1447 1448
}

1449
static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1450
{
1451
	ctxt->src.val = get_segment_selector(ctxt, seg);
1452

1453
	return em_push(ctxt);
1454 1455
}

1456
static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1457
{
1458 1459
	unsigned long selector;
	int rc;
1460

1461
	rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1462 1463 1464
	if (rc != X86EMUL_CONTINUE)
		return rc;

1465
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1466
	return rc;
1467 1468
}

1469
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1470
{
1471
	unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1472 1473
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1474

1475 1476
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
1477
		(ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1478

1479
		rc = em_push(ctxt);
1480 1481
		if (rc != X86EMUL_CONTINUE)
			return rc;
1482

1483
		++reg;
1484 1485
	}

1486
	return rc;
1487 1488
}

1489 1490
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
1491
	ctxt->src.val =  (unsigned long)ctxt->eflags;
1492 1493 1494
	return em_push(ctxt);
}

1495
static int em_popa(struct x86_emulate_ctxt *ctxt)
1496
{
1497 1498
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
1499

1500 1501
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
1502 1503
			register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
							ctxt->op_bytes);
1504 1505
			--reg;
		}
1506

1507
		rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1508 1509 1510
		if (rc != X86EMUL_CONTINUE)
			break;
		--reg;
1511
	}
1512
	return rc;
1513 1514
}

1515
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1516
{
1517
	struct x86_emulate_ops *ops = ctxt->ops;
1518
	int rc;
1519 1520 1521 1522 1523 1524
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
1525
	ctxt->src.val = ctxt->eflags;
1526
	rc = em_push(ctxt);
1527 1528
	if (rc != X86EMUL_CONTINUE)
		return rc;
1529 1530 1531

	ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);

1532
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1533
	rc = em_push(ctxt);
1534 1535
	if (rc != X86EMUL_CONTINUE)
		return rc;
1536

1537
	ctxt->src.val = ctxt->_eip;
1538
	rc = em_push(ctxt);
1539 1540 1541
	if (rc != X86EMUL_CONTINUE)
		return rc;

1542
	ops->get_idt(ctxt, &dt);
1543 1544 1545 1546

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

1547
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1548 1549 1550
	if (rc != X86EMUL_CONTINUE)
		return rc;

1551
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1552 1553 1554
	if (rc != X86EMUL_CONTINUE)
		return rc;

1555
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1556 1557 1558
	if (rc != X86EMUL_CONTINUE)
		return rc;

1559
	ctxt->_eip = eip;
1560 1561 1562 1563

	return rc;
}

1564
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1565 1566 1567
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1568
		return emulate_int_real(ctxt, irq);
1569 1570 1571 1572 1573 1574 1575 1576 1577 1578
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

1579
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1580
{
1581 1582 1583 1584 1585 1586 1587 1588
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
	unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
			     EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
			     EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
	unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1589

1590
	/* TODO: Add stack limit check */
1591

1592
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1593

1594 1595
	if (rc != X86EMUL_CONTINUE)
		return rc;
1596

1597 1598
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
1599

1600
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1601

1602 1603
	if (rc != X86EMUL_CONTINUE)
		return rc;
1604

1605
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1606

1607 1608
	if (rc != X86EMUL_CONTINUE)
		return rc;
1609

1610
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1611

1612 1613
	if (rc != X86EMUL_CONTINUE)
		return rc;
1614

1615
	ctxt->_eip = temp_eip;
1616 1617


1618
	if (ctxt->op_bytes == 4)
1619
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1620
	else if (ctxt->op_bytes == 2) {
1621 1622
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
1623
	}
1624 1625 1626 1627 1628

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
	ctxt->eflags |= EFLG_RESERVED_ONE_MASK;

	return rc;
1629 1630
}

1631
static int em_iret(struct x86_emulate_ctxt *ctxt)
1632
{
1633 1634
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1635
		return emulate_iret_real(ctxt);
1636 1637 1638 1639
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
1640
	default:
1641 1642
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
1643 1644 1645
	}
}

1646 1647 1648 1649 1650
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned short sel;

1651
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1652

1653
	rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1654 1655 1656
	if (rc != X86EMUL_CONTINUE)
		return rc;

1657 1658
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1659 1660 1661
	return X86EMUL_CONTINUE;
}

1662
static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1663
{
1664
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1665 1666
}

1667
static int em_grp2(struct x86_emulate_ctxt *ctxt)
1668
{
1669
	switch (ctxt->modrm_reg) {
1670
	case 0:	/* rol */
1671
		emulate_2op_SrcB(ctxt, "rol");
1672 1673
		break;
	case 1:	/* ror */
1674
		emulate_2op_SrcB(ctxt, "ror");
1675 1676
		break;
	case 2:	/* rcl */
1677
		emulate_2op_SrcB(ctxt, "rcl");
1678 1679
		break;
	case 3:	/* rcr */
1680
		emulate_2op_SrcB(ctxt, "rcr");
1681 1682 1683
		break;
	case 4:	/* sal/shl */
	case 6:	/* sal/shl */
1684
		emulate_2op_SrcB(ctxt, "sal");
1685 1686
		break;
	case 5:	/* shr */
1687
		emulate_2op_SrcB(ctxt, "shr");
1688 1689
		break;
	case 7:	/* sar */
1690
		emulate_2op_SrcB(ctxt, "sar");
1691 1692
		break;
	}
1693
	return X86EMUL_CONTINUE;
1694 1695
}

1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724
static int em_not(struct x86_emulate_ctxt *ctxt)
{
	ctxt->dst.val = ~ctxt->dst.val;
	return X86EMUL_CONTINUE;
}

static int em_neg(struct x86_emulate_ctxt *ctxt)
{
	emulate_1op(ctxt, "neg");
	return X86EMUL_CONTINUE;
}

static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "mul", ex);
	return X86EMUL_CONTINUE;
}

static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "imul", ex);
	return X86EMUL_CONTINUE;
}

static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1725
{
1726
	u8 de = 0;
1727

1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738
	emulate_1op_rax_rdx(ctxt, "div", de);
	if (de)
		return emulate_de(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 de = 0;

	emulate_1op_rax_rdx(ctxt, "idiv", de);
1739 1740
	if (de)
		return emulate_de(ctxt);
1741
	return X86EMUL_CONTINUE;
1742 1743
}

1744
static int em_grp45(struct x86_emulate_ctxt *ctxt)
1745
{
1746
	int rc = X86EMUL_CONTINUE;
1747

1748
	switch (ctxt->modrm_reg) {
1749
	case 0:	/* inc */
1750
		emulate_1op(ctxt, "inc");
1751 1752
		break;
	case 1:	/* dec */
1753
		emulate_1op(ctxt, "dec");
1754
		break;
1755 1756
	case 2: /* call near abs */ {
		long int old_eip;
1757 1758 1759
		old_eip = ctxt->_eip;
		ctxt->_eip = ctxt->src.val;
		ctxt->src.val = old_eip;
1760
		rc = em_push(ctxt);
1761 1762
		break;
	}
1763
	case 4: /* jmp abs */
1764
		ctxt->_eip = ctxt->src.val;
1765
		break;
1766 1767 1768
	case 5: /* jmp far */
		rc = em_jmp_far(ctxt);
		break;
1769
	case 6:	/* push */
1770
		rc = em_push(ctxt);
1771 1772
		break;
	}
1773
	return rc;
1774 1775
}

1776
static int em_grp9(struct x86_emulate_ctxt *ctxt)
1777
{
1778
	u64 old = ctxt->dst.orig_val64;
1779

1780 1781 1782 1783
	if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
	    ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
		ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
		ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1784
		ctxt->eflags &= ~EFLG_ZF;
1785
	} else {
1786 1787
		ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
			(u32) ctxt->regs[VCPU_REGS_RBX];
1788

1789
		ctxt->eflags |= EFLG_ZF;
1790
	}
1791
	return X86EMUL_CONTINUE;
1792 1793
}

1794 1795
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
1796 1797 1798
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
1799 1800 1801
	return em_pop(ctxt);
}

1802
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1803 1804 1805 1806
{
	int rc;
	unsigned long cs;

1807
	rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1808
	if (rc != X86EMUL_CONTINUE)
1809
		return rc;
1810 1811 1812
	if (ctxt->op_bytes == 4)
		ctxt->_eip = (u32)ctxt->_eip;
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1813
	if (rc != X86EMUL_CONTINUE)
1814
		return rc;
1815
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1816 1817 1818
	return rc;
}

1819
static int emulate_load_segment(struct x86_emulate_ctxt *ctxt, int seg)
1820 1821 1822 1823
{
	unsigned short sel;
	int rc;

1824
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1825

1826
	rc = load_segment_descriptor(ctxt, sel, seg);
1827 1828 1829
	if (rc != X86EMUL_CONTINUE)
		return rc;

1830
	ctxt->dst.val = ctxt->src.val;
1831 1832 1833
	return rc;
}

1834
static void
1835
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1836
			struct desc_struct *cs, struct desc_struct *ss)
1837
{
1838 1839
	u16 selector;

1840
	memset(cs, 0, sizeof(struct desc_struct));
1841
	ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1842
	memset(ss, 0, sizeof(struct desc_struct));
1843 1844

	cs->l = 0;		/* will be adjusted later */
1845
	set_desc_base(cs, 0);	/* flat segment */
1846
	cs->g = 1;		/* 4kb granularity */
1847
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
1848 1849 1850
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
1851 1852
	cs->p = 1;
	cs->d = 1;
1853

1854 1855
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
1856 1857 1858
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
1859
	ss->d = 1;		/* 32bit stack segment */
1860
	ss->dpl = 0;
1861
	ss->p = 1;
1862 1863
}

1864
static int em_syscall(struct x86_emulate_ctxt *ctxt)
1865
{
1866
	struct x86_emulate_ops *ops = ctxt->ops;
1867
	struct desc_struct cs, ss;
1868
	u64 msr_data;
1869
	u16 cs_sel, ss_sel;
1870
	u64 efer = 0;
1871 1872

	/* syscall is not available in real mode */
1873
	if (ctxt->mode == X86EMUL_MODE_REAL ||
1874 1875
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
1876

1877
	ops->get_msr(ctxt, MSR_EFER, &efer);
1878
	setup_syscalls_segments(ctxt, &cs, &ss);
1879

1880
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
1881
	msr_data >>= 32;
1882 1883
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
1884

1885
	if (efer & EFER_LMA) {
1886
		cs.d = 0;
1887 1888
		cs.l = 1;
	}
1889 1890
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1891

1892
	ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1893
	if (efer & EFER_LMA) {
1894
#ifdef CONFIG_X86_64
1895
		ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1896

1897
		ops->get_msr(ctxt,
1898 1899
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
1900
		ctxt->_eip = msr_data;
1901

1902
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1903 1904 1905 1906
		ctxt->eflags &= ~(msr_data | EFLG_RF);
#endif
	} else {
		/* legacy mode */
1907
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
1908
		ctxt->_eip = (u32)msr_data;
1909 1910 1911 1912

		ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
	}

1913
	return X86EMUL_CONTINUE;
1914 1915
}

1916
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1917
{
1918
	struct x86_emulate_ops *ops = ctxt->ops;
1919
	struct desc_struct cs, ss;
1920
	u64 msr_data;
1921
	u16 cs_sel, ss_sel;
1922
	u64 efer = 0;
1923

1924
	ops->get_msr(ctxt, MSR_EFER, &efer);
1925
	/* inject #GP if in real mode */
1926 1927
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
1928 1929 1930 1931

	/* XXX sysenter/sysexit have not been tested in 64bit mode.
	* Therefore, we inject an #UD.
	*/
1932 1933
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return emulate_ud(ctxt);
1934

1935
	setup_syscalls_segments(ctxt, &cs, &ss);
1936

1937
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1938 1939
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT32:
1940 1941
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
1942 1943
		break;
	case X86EMUL_MODE_PROT64:
1944 1945
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
1946 1947 1948 1949
		break;
	}

	ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1950 1951 1952 1953
	cs_sel = (u16)msr_data;
	cs_sel &= ~SELECTOR_RPL_MASK;
	ss_sel = cs_sel + 8;
	ss_sel &= ~SELECTOR_RPL_MASK;
1954
	if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1955
		cs.d = 0;
1956 1957 1958
		cs.l = 1;
	}

1959 1960
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1961

1962
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1963
	ctxt->_eip = msr_data;
1964

1965
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1966
	ctxt->regs[VCPU_REGS_RSP] = msr_data;
1967

1968
	return X86EMUL_CONTINUE;
1969 1970
}

1971
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1972
{
1973
	struct x86_emulate_ops *ops = ctxt->ops;
1974
	struct desc_struct cs, ss;
1975 1976
	u64 msr_data;
	int usermode;
X
Xiao Guangrong 已提交
1977
	u16 cs_sel = 0, ss_sel = 0;
1978

1979 1980
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
1981 1982
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
1983

1984
	setup_syscalls_segments(ctxt, &cs, &ss);
1985

1986
	if ((ctxt->rex_prefix & 0x8) != 0x0)
1987 1988 1989 1990 1991 1992
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

	cs.dpl = 3;
	ss.dpl = 3;
1993
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1994 1995
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
1996
		cs_sel = (u16)(msr_data + 16);
1997 1998
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
1999
		ss_sel = (u16)(msr_data + 24);
2000 2001
		break;
	case X86EMUL_MODE_PROT64:
2002
		cs_sel = (u16)(msr_data + 32);
2003 2004
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2005 2006
		ss_sel = cs_sel + 8;
		cs.d = 0;
2007 2008 2009
		cs.l = 1;
		break;
	}
2010 2011
	cs_sel |= SELECTOR_RPL_MASK;
	ss_sel |= SELECTOR_RPL_MASK;
2012

2013 2014
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2015

2016 2017
	ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
	ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2018

2019
	return X86EMUL_CONTINUE;
2020 2021
}

2022
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2023 2024 2025 2026 2027 2028 2029
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2030
	return ctxt->ops->cpl(ctxt) > iopl;
2031 2032 2033 2034 2035
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2036
	struct x86_emulate_ops *ops = ctxt->ops;
2037
	struct desc_struct tr_seg;
2038
	u32 base3;
2039
	int r;
2040
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2041
	unsigned mask = (1 << len) - 1;
2042
	unsigned long base;
2043

2044
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2045
	if (!tr_seg.p)
2046
		return false;
2047
	if (desc_limit_scaled(&tr_seg) < 103)
2048
		return false;
2049 2050 2051 2052
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2053
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2054 2055
	if (r != X86EMUL_CONTINUE)
		return false;
2056
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2057
		return false;
2058
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2069 2070 2071
	if (ctxt->perm_ok)
		return true;

2072 2073
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2074
			return false;
2075 2076 2077

	ctxt->perm_ok = true;

2078 2079 2080
	return true;
}

2081 2082 2083
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
2084
	tss->ip = ctxt->_eip;
2085
	tss->flag = ctxt->eflags;
2086 2087 2088 2089 2090 2091 2092 2093
	tss->ax = ctxt->regs[VCPU_REGS_RAX];
	tss->cx = ctxt->regs[VCPU_REGS_RCX];
	tss->dx = ctxt->regs[VCPU_REGS_RDX];
	tss->bx = ctxt->regs[VCPU_REGS_RBX];
	tss->sp = ctxt->regs[VCPU_REGS_RSP];
	tss->bp = ctxt->regs[VCPU_REGS_RBP];
	tss->si = ctxt->regs[VCPU_REGS_RSI];
	tss->di = ctxt->regs[VCPU_REGS_RDI];
2094

2095 2096 2097 2098 2099
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2100 2101 2102 2103 2104 2105 2106
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;

2107
	ctxt->_eip = tss->ip;
2108
	ctxt->eflags = tss->flag | 2;
2109 2110 2111 2112 2113 2114 2115 2116
	ctxt->regs[VCPU_REGS_RAX] = tss->ax;
	ctxt->regs[VCPU_REGS_RCX] = tss->cx;
	ctxt->regs[VCPU_REGS_RDX] = tss->dx;
	ctxt->regs[VCPU_REGS_RBX] = tss->bx;
	ctxt->regs[VCPU_REGS_RSP] = tss->sp;
	ctxt->regs[VCPU_REGS_RBP] = tss->bp;
	ctxt->regs[VCPU_REGS_RSI] = tss->si;
	ctxt->regs[VCPU_REGS_RDI] = tss->di;
2117 2118 2119 2120 2121

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2122 2123 2124 2125 2126
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2127 2128 2129 2130 2131

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2132
	ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2133 2134
	if (ret != X86EMUL_CONTINUE)
		return ret;
2135
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2136 2137
	if (ret != X86EMUL_CONTINUE)
		return ret;
2138
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2139 2140
	if (ret != X86EMUL_CONTINUE)
		return ret;
2141
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2142 2143
	if (ret != X86EMUL_CONTINUE)
		return ret;
2144
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2145 2146 2147 2148 2149 2150 2151 2152 2153 2154
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2155
	struct x86_emulate_ops *ops = ctxt->ops;
2156 2157
	struct tss_segment_16 tss_seg;
	int ret;
2158
	u32 new_tss_base = get_desc_base(new_desc);
2159

2160
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2161
			    &ctxt->exception);
2162
	if (ret != X86EMUL_CONTINUE)
2163 2164 2165
		/* FIXME: need to provide precise fault address */
		return ret;

2166
	save_state_to_tss16(ctxt, &tss_seg);
2167

2168
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2169
			     &ctxt->exception);
2170
	if (ret != X86EMUL_CONTINUE)
2171 2172 2173
		/* FIXME: need to provide precise fault address */
		return ret;

2174
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2175
			    &ctxt->exception);
2176
	if (ret != X86EMUL_CONTINUE)
2177 2178 2179 2180 2181 2182
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2183
		ret = ops->write_std(ctxt, new_tss_base,
2184 2185
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2186
				     &ctxt->exception);
2187
		if (ret != X86EMUL_CONTINUE)
2188 2189 2190 2191
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2192
	return load_state_from_tss16(ctxt, &tss_seg);
2193 2194 2195 2196 2197
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
2198
	tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2199
	tss->eip = ctxt->_eip;
2200
	tss->eflags = ctxt->eflags;
2201 2202 2203 2204 2205 2206 2207 2208
	tss->eax = ctxt->regs[VCPU_REGS_RAX];
	tss->ecx = ctxt->regs[VCPU_REGS_RCX];
	tss->edx = ctxt->regs[VCPU_REGS_RDX];
	tss->ebx = ctxt->regs[VCPU_REGS_RBX];
	tss->esp = ctxt->regs[VCPU_REGS_RSP];
	tss->ebp = ctxt->regs[VCPU_REGS_RBP];
	tss->esi = ctxt->regs[VCPU_REGS_RSI];
	tss->edi = ctxt->regs[VCPU_REGS_RDI];
2209

2210 2211 2212 2213 2214 2215 2216
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
	tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2217 2218 2219 2220 2221 2222 2223
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;

2224
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2225
		return emulate_gp(ctxt, 0);
2226
	ctxt->_eip = tss->eip;
2227
	ctxt->eflags = tss->eflags | 2;
2228 2229 2230 2231 2232 2233 2234 2235
	ctxt->regs[VCPU_REGS_RAX] = tss->eax;
	ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
	ctxt->regs[VCPU_REGS_RDX] = tss->edx;
	ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
	ctxt->regs[VCPU_REGS_RSP] = tss->esp;
	ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
	ctxt->regs[VCPU_REGS_RSI] = tss->esi;
	ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2236 2237 2238 2239 2240

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2241 2242 2243 2244 2245 2246 2247
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2248 2249 2250 2251 2252

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2253
	ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2254 2255
	if (ret != X86EMUL_CONTINUE)
		return ret;
2256
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2257 2258
	if (ret != X86EMUL_CONTINUE)
		return ret;
2259
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2260 2261
	if (ret != X86EMUL_CONTINUE)
		return ret;
2262
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2263 2264
	if (ret != X86EMUL_CONTINUE)
		return ret;
2265
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2266 2267
	if (ret != X86EMUL_CONTINUE)
		return ret;
2268
	ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2269 2270
	if (ret != X86EMUL_CONTINUE)
		return ret;
2271
	ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2272 2273 2274 2275 2276 2277 2278 2279 2280 2281
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2282
	struct x86_emulate_ops *ops = ctxt->ops;
2283 2284
	struct tss_segment_32 tss_seg;
	int ret;
2285
	u32 new_tss_base = get_desc_base(new_desc);
2286

2287
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2288
			    &ctxt->exception);
2289
	if (ret != X86EMUL_CONTINUE)
2290 2291 2292
		/* FIXME: need to provide precise fault address */
		return ret;

2293
	save_state_to_tss32(ctxt, &tss_seg);
2294

2295
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2296
			     &ctxt->exception);
2297
	if (ret != X86EMUL_CONTINUE)
2298 2299 2300
		/* FIXME: need to provide precise fault address */
		return ret;

2301
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2302
			    &ctxt->exception);
2303
	if (ret != X86EMUL_CONTINUE)
2304 2305 2306 2307 2308 2309
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2310
		ret = ops->write_std(ctxt, new_tss_base,
2311 2312
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2313
				     &ctxt->exception);
2314
		if (ret != X86EMUL_CONTINUE)
2315 2316 2317 2318
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2319
	return load_state_from_tss32(ctxt, &tss_seg);
2320 2321 2322
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2323 2324
				   u16 tss_selector, int reason,
				   bool has_error_code, u32 error_code)
2325
{
2326
	struct x86_emulate_ops *ops = ctxt->ops;
2327 2328
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
2329
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2330
	ulong old_tss_base =
2331
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2332
	u32 desc_limit;
2333 2334 2335

	/* FIXME: old_tss_base == ~0 ? */

2336
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2337 2338
	if (ret != X86EMUL_CONTINUE)
		return ret;
2339
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2340 2341 2342 2343 2344 2345 2346
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

	if (reason != TASK_SWITCH_IRET) {
		if ((tss_selector & 3) > next_tss_desc.dpl ||
2347
		    ops->cpl(ctxt) > next_tss_desc.dpl)
2348
			return emulate_gp(ctxt, 0);
2349 2350
	}

2351 2352 2353 2354
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
2355
		emulate_ts(ctxt, tss_selector & 0xfffc);
2356 2357 2358 2359 2360
		return X86EMUL_PROPAGATE_FAULT;
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2361
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
	   note that old_tss_sel is not used afetr this point */
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
2373
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2374 2375
				     old_tss_base, &next_tss_desc);
	else
2376
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2377
				     old_tss_base, &next_tss_desc);
2378 2379
	if (ret != X86EMUL_CONTINUE)
		return ret;
2380 2381 2382 2383 2384 2385

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
2386
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2387 2388
	}

2389
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
2390
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2391

2392
	if (has_error_code) {
2393 2394 2395
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
2396
		ret = em_push(ctxt);
2397 2398
	}

2399 2400 2401 2402
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2403 2404
			 u16 tss_selector, int reason,
			 bool has_error_code, u32 error_code)
2405 2406 2407
{
	int rc;

2408 2409
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
2410

2411
	rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2412
				     has_error_code, error_code);
2413

2414
	if (rc == X86EMUL_CONTINUE)
2415
		ctxt->eip = ctxt->_eip;
2416

2417
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2418 2419
}

2420
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2421
			    int reg, struct operand *op)
2422 2423 2424
{
	int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;

2425 2426
	register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2427
	op->addr.mem.seg = seg;
2428 2429
}

2430 2431 2432 2433 2434 2435
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
2436
	al = ctxt->dst.val;
2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

2454
	ctxt->dst.val = al;
2455
	/* Set PF, ZF, SF */
2456 2457 2458
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
2459
	emulate_2op_SrcV(ctxt, "or");
2460 2461 2462 2463 2464 2465 2466 2467
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

2468 2469 2470 2471 2472 2473
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;

2474
	old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2475
	old_eip = ctxt->_eip;
2476

2477
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2478
	if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2479 2480
		return X86EMUL_CONTINUE;

2481 2482
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2483

2484
	ctxt->src.val = old_cs;
2485
	rc = em_push(ctxt);
2486 2487 2488
	if (rc != X86EMUL_CONTINUE)
		return rc;

2489
	ctxt->src.val = old_eip;
2490
	return em_push(ctxt);
2491 2492
}

2493 2494 2495 2496
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2497 2498 2499 2500
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
	rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2501 2502
	if (rc != X86EMUL_CONTINUE)
		return rc;
2503
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2504 2505 2506
	return X86EMUL_CONTINUE;
}

2507 2508
static int em_add(struct x86_emulate_ctxt *ctxt)
{
2509
	emulate_2op_SrcV(ctxt, "add");
2510 2511 2512 2513 2514
	return X86EMUL_CONTINUE;
}

static int em_or(struct x86_emulate_ctxt *ctxt)
{
2515
	emulate_2op_SrcV(ctxt, "or");
2516 2517 2518 2519 2520
	return X86EMUL_CONTINUE;
}

static int em_adc(struct x86_emulate_ctxt *ctxt)
{
2521
	emulate_2op_SrcV(ctxt, "adc");
2522 2523 2524 2525 2526
	return X86EMUL_CONTINUE;
}

static int em_sbb(struct x86_emulate_ctxt *ctxt)
{
2527
	emulate_2op_SrcV(ctxt, "sbb");
2528 2529 2530 2531 2532
	return X86EMUL_CONTINUE;
}

static int em_and(struct x86_emulate_ctxt *ctxt)
{
2533
	emulate_2op_SrcV(ctxt, "and");
2534 2535 2536 2537 2538
	return X86EMUL_CONTINUE;
}

static int em_sub(struct x86_emulate_ctxt *ctxt)
{
2539
	emulate_2op_SrcV(ctxt, "sub");
2540 2541 2542 2543 2544
	return X86EMUL_CONTINUE;
}

static int em_xor(struct x86_emulate_ctxt *ctxt)
{
2545
	emulate_2op_SrcV(ctxt, "xor");
2546 2547 2548 2549 2550
	return X86EMUL_CONTINUE;
}

static int em_cmp(struct x86_emulate_ctxt *ctxt)
{
2551
	emulate_2op_SrcV(ctxt, "cmp");
2552
	/* Disable writeback. */
2553
	ctxt->dst.type = OP_NONE;
2554 2555 2556
	return X86EMUL_CONTINUE;
}

2557 2558
static int em_test(struct x86_emulate_ctxt *ctxt)
{
2559
	emulate_2op_SrcV(ctxt, "test");
2560 2561
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
2562 2563 2564
	return X86EMUL_CONTINUE;
}

2565 2566 2567
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
2568 2569
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
2570 2571

	/* Write back the memory destination with implicit LOCK prefix. */
2572 2573
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
2574 2575 2576
	return X86EMUL_CONTINUE;
}

2577
static int em_imul(struct x86_emulate_ctxt *ctxt)
2578
{
2579
	emulate_2op_SrcV_nobyte(ctxt, "imul");
2580 2581 2582
	return X86EMUL_CONTINUE;
}

2583 2584
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
2585
	ctxt->dst.val = ctxt->src2.val;
2586 2587 2588
	return em_imul(ctxt);
}

2589 2590
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
2591 2592 2593 2594
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
	ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2595 2596 2597 2598

	return X86EMUL_CONTINUE;
}

2599 2600 2601 2602
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

2603
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2604 2605
	ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
	ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2606 2607 2608
	return X86EMUL_CONTINUE;
}

2609 2610
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
2611
	ctxt->dst.val = ctxt->src.val;
2612 2613 2614
	return X86EMUL_CONTINUE;
}

2615 2616
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
2617
	if (ctxt->modrm_reg > VCPU_SREG_GS)
2618 2619
		return emulate_ud(ctxt);

2620
	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2621 2622 2623 2624 2625
	return X86EMUL_CONTINUE;
}

static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
2626
	u16 sel = ctxt->src.val;
2627

2628
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2629 2630
		return emulate_ud(ctxt);

2631
	if (ctxt->modrm_reg == VCPU_SREG_SS)
2632 2633 2634
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
2635 2636
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2637 2638
}

2639 2640
static int em_movdqu(struct x86_emulate_ctxt *ctxt)
{
2641
	memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2642 2643 2644
	return X86EMUL_CONTINUE;
}

2645 2646
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
2647 2648 2649
	int rc;
	ulong linear;

2650
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2651
	if (rc == X86EMUL_CONTINUE)
2652
		ctxt->ops->invlpg(ctxt, linear);
2653
	/* Disable writeback. */
2654
	ctxt->dst.type = OP_NONE;
2655 2656 2657
	return X86EMUL_CONTINUE;
}

2658 2659 2660 2661 2662 2663 2664 2665 2666 2667
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

2668 2669 2670 2671
static int em_vmcall(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2672
	if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2673 2674 2675 2676 2677 2678 2679
		return X86EMUL_UNHANDLEABLE;

	rc = ctxt->ops->fix_hypercall(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
2680
	ctxt->_eip = ctxt->eip;
2681
	/* Disable writeback. */
2682
	ctxt->dst.type = OP_NONE;
2683 2684 2685 2686 2687 2688 2689 2690
	return X86EMUL_CONTINUE;
}

static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

2691
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2692
			     &desc_ptr.size, &desc_ptr.address,
2693
			     ctxt->op_bytes);
2694 2695 2696 2697
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_gdt(ctxt, &desc_ptr);
	/* Disable writeback. */
2698
	ctxt->dst.type = OP_NONE;
2699 2700 2701
	return X86EMUL_CONTINUE;
}

2702
static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2703 2704 2705
{
	int rc;

2706 2707
	rc = ctxt->ops->fix_hypercall(ctxt);

2708
	/* Disable writeback. */
2709
	ctxt->dst.type = OP_NONE;
2710 2711 2712 2713 2714 2715 2716 2717
	return rc;
}

static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

2718
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2719
			     &desc_ptr.size, &desc_ptr.address,
2720
			     ctxt->op_bytes);
2721 2722 2723 2724
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_idt(ctxt, &desc_ptr);
	/* Disable writeback. */
2725
	ctxt->dst.type = OP_NONE;
2726 2727 2728 2729 2730
	return X86EMUL_CONTINUE;
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
2731 2732
	ctxt->dst.bytes = 2;
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2733 2734 2735 2736 2737 2738
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2739 2740
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
2741 2742 2743
	return X86EMUL_CONTINUE;
}

2744 2745
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
2746 2747 2748 2749
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
	if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
		jmp_rel(ctxt, ctxt->src.val);
2750 2751 2752 2753 2754 2755

	return X86EMUL_CONTINUE;
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
2756 2757
	if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
		jmp_rel(ctxt, ctxt->src.val);
2758 2759 2760 2761

	return X86EMUL_CONTINUE;
}

2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
2795
	if (!valid_cr(ctxt->modrm_reg))
2796 2797 2798 2799 2800 2801 2802
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
2803 2804
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
2805
	u64 efer = 0;
2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
2823
		u64 cr4;
2824 2825 2826 2827
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

2828 2829
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2830 2831 2832 2833 2834 2835 2836 2837 2838 2839

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

2840 2841
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
2842
			rsvd = CR3_L_MODE_RESERVED_BITS;
2843
		else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2844
			rsvd = CR3_PAE_RESERVED_BITS;
2845
		else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2846 2847 2848 2849 2850 2851 2852 2853
			rsvd = CR3_NONPAE_RESERVED_BITS;

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
2854
		u64 cr4;
2855

2856 2857
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

2869 2870 2871 2872
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

2873
	ctxt->ops->get_dr(ctxt, 7, &dr7);
2874 2875 2876 2877 2878 2879 2880

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
2881
	int dr = ctxt->modrm_reg;
2882 2883 2884 2885 2886
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

2887
	cr4 = ctxt->ops->get_cr(ctxt, 4);
2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

	if (check_dr7_gd(ctxt))
		return emulate_db(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
2899 2900
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
2901 2902 2903 2904 2905 2906 2907

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

2908 2909 2910 2911
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
	u64 efer;

2912
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2913 2914 2915 2916 2917 2918 2919 2920 2921

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
2922
	u64 rax = ctxt->regs[VCPU_REGS_RAX];
2923 2924

	/* Valid physical address? */
2925
	if (rax & 0xffff000000000000ULL)
2926 2927 2928 2929 2930
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

2931 2932
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
2933
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2934

2935
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2936 2937 2938 2939 2940
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

2941 2942
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
2943
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2944
	u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2945

2946
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2947 2948 2949 2950 2951 2952
	    (rcx > 3))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

2953 2954
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
2955 2956
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2957 2958 2959 2960 2961 2962 2963
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
2964 2965
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2966 2967 2968 2969 2970
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

2971
#define D(_y) { .flags = (_y) }
2972
#define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2973 2974
#define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
		      .check_perm = (_p) }
2975
#define N    D(0)
2976
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2977
#define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2978
#define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2979
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2980 2981
#define II(_f, _e, _i) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2982 2983 2984
#define IIP(_f, _e, _i, _p) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
	  .check_perm = (_p) }
2985
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2986

2987
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
2988
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2989 2990
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)

2991 2992 2993
#define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2994

2995 2996 2997 2998 2999 3000
static struct opcode group7_rm1[] = {
	DI(SrcNone | ModRM | Priv, monitor),
	DI(SrcNone | ModRM | Priv, mwait),
	N, N, N, N, N, N,
};

3001 3002
static struct opcode group7_rm3[] = {
	DIP(SrcNone | ModRM | Prot | Priv, vmrun,   check_svme_pa),
3003
	II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3004 3005 3006 3007 3008 3009 3010
	DIP(SrcNone | ModRM | Prot | Priv, vmload,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, vmsave,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, stgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, clgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, skinit,  check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
};
3011

3012 3013 3014 3015 3016
static struct opcode group7_rm7[] = {
	N,
	DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
	N, N, N, N, N, N,
};
3017

3018
static struct opcode group1[] = {
3019 3020 3021 3022 3023 3024 3025 3026
	I(Lock, em_add),
	I(Lock, em_or),
	I(Lock, em_adc),
	I(Lock, em_sbb),
	I(Lock, em_and),
	I(Lock, em_sub),
	I(Lock, em_xor),
	I(0, em_cmp),
3027 3028 3029 3030 3031 3032 3033
};

static struct opcode group1A[] = {
	D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
};

static struct opcode group3[] = {
3034 3035 3036 3037 3038 3039 3040 3041
	I(DstMem | SrcImm | ModRM, em_test),
	I(DstMem | SrcImm | ModRM, em_test),
	I(DstMem | SrcNone | ModRM | Lock, em_not),
	I(DstMem | SrcNone | ModRM | Lock, em_neg),
	I(SrcMem | ModRM, em_mul_ex),
	I(SrcMem | ModRM, em_imul_ex),
	I(SrcMem | ModRM, em_div_ex),
	I(SrcMem | ModRM, em_idiv_ex),
3042 3043 3044 3045 3046 3047 3048 3049 3050
};

static struct opcode group4[] = {
	D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
	N, N, N, N, N, N,
};

static struct opcode group5[] = {
	D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3051 3052
	D(SrcMem | ModRM | Stack),
	I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3053 3054 3055 3056
	D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
	D(SrcMem | ModRM | Stack), N,
};

3057 3058 3059 3060 3061 3062 3063 3064
static struct opcode group6[] = {
	DI(ModRM | Prot,        sldt),
	DI(ModRM | Prot,        str),
	DI(ModRM | Prot | Priv, lldt),
	DI(ModRM | Prot | Priv, ltr),
	N, N, N, N,
};

3065
static struct group_dual group7 = { {
3066 3067
	DI(ModRM | Mov | DstMem | Priv, sgdt),
	DI(ModRM | Mov | DstMem | Priv, sidt),
3068 3069 3070 3071 3072
	II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
	II(ModRM | SrcMem | Priv, em_lidt, lidt),
	II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
	II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
	II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3073
}, {
3074 3075
	I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
	EXT(0, group7_rm1),
3076
	N, EXT(0, group7_rm3),
3077 3078
	II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
	II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092
} };

static struct opcode group8[] = {
	N, N, N, N,
	D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
	D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
};

static struct group_dual group9 = { {
	N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
}, {
	N, N, N, N, N, N, N, N,
} };

3093 3094 3095 3096
static struct opcode group11[] = {
	I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
};

3097 3098 3099 3100
static struct gprefix pfx_0f_6f_0f_7f = {
	N, N, N, I(Sse, em_movdqu),
};

3101 3102
static struct opcode opcode_table[256] = {
	/* 0x00 - 0x07 */
3103
	I6ALU(Lock, em_add),
3104 3105
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	/* 0x08 - 0x0F */
3106
	I6ALU(Lock, em_or),
3107 3108
	D(ImplicitOps | Stack | No64), N,
	/* 0x10 - 0x17 */
3109
	I6ALU(Lock, em_adc),
3110 3111
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	/* 0x18 - 0x1F */
3112
	I6ALU(Lock, em_sbb),
3113 3114
	D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
	/* 0x20 - 0x27 */
3115
	I6ALU(Lock, em_and), N, N,
3116
	/* 0x28 - 0x2F */
3117
	I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3118
	/* 0x30 - 0x37 */
3119
	I6ALU(Lock, em_xor), N, N,
3120
	/* 0x38 - 0x3F */
3121
	I6ALU(0, em_cmp), N, N,
3122 3123 3124
	/* 0x40 - 0x4F */
	X16(D(DstReg)),
	/* 0x50 - 0x57 */
3125
	X8(I(SrcReg | Stack, em_push)),
3126
	/* 0x58 - 0x5F */
3127
	X8(I(DstReg | Stack, em_pop)),
3128
	/* 0x60 - 0x67 */
3129 3130
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
3131 3132 3133
	N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
	N, N, N, N,
	/* 0x68 - 0x6F */
3134 3135
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3136 3137
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3138 3139
	D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
	D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3140 3141 3142 3143 3144 3145 3146
	/* 0x70 - 0x7F */
	X16(D(SrcImmByte)),
	/* 0x80 - 0x87 */
	G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
	G(DstMem | SrcImm | ModRM | Group, group1),
	G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
	G(DstMem | SrcImmByte | ModRM | Group, group1),
3147
	I2bv(DstMem | SrcReg | ModRM, em_test),
3148
	I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3149
	/* 0x88 - 0x8F */
3150 3151
	I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3152 3153 3154 3155
	I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
3156
	/* 0x90 - 0x97 */
3157
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3158
	/* 0x98 - 0x9F */
3159
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3160
	I(SrcImmFAddr | No64, em_call_far), N,
3161 3162
	II(ImplicitOps | Stack, em_pushf, pushf),
	II(ImplicitOps | Stack, em_popf, popf), N, N,
3163
	/* 0xA0 - 0xA7 */
3164 3165 3166
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
	I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
	I2bv(SrcSI | DstDI | Mov | String, em_mov),
3167
	I2bv(SrcSI | DstDI | String, em_cmp),
3168
	/* 0xA8 - 0xAF */
3169
	I2bv(DstAcc | SrcImm, em_test),
3170 3171
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3172
	I2bv(SrcAcc | DstDI | String, em_cmp),
3173
	/* 0xB0 - 0xB7 */
3174
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3175
	/* 0xB8 - 0xBF */
3176
	X8(I(DstReg | SrcImm | Mov, em_mov)),
3177
	/* 0xC0 - 0xC7 */
3178
	D2bv(DstMem | SrcImmByte | ModRM),
3179
	I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3180
	I(ImplicitOps | Stack, em_ret),
3181
	D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3182
	G(ByteOp, group11), G(0, group11),
3183
	/* 0xC8 - 0xCF */
3184
	N, N, N, I(ImplicitOps | Stack, em_ret_far),
3185
	D(ImplicitOps), DI(SrcImmByte, intn),
3186
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3187
	/* 0xD0 - 0xD7 */
3188
	D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3189 3190 3191 3192
	N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
3193 3194
	X3(I(SrcImmByte, em_loop)),
	I(SrcImmByte, em_jcxz),
3195 3196
	D2bvIP(SrcImmUByte | DstAcc, in,  check_perm_in),
	D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3197 3198
	/* 0xE8 - 0xEF */
	D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3199
	I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3200 3201
	D2bvIP(SrcDX | DstAcc, in,  check_perm_in),
	D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3202
	/* 0xF0 - 0xF7 */
3203
	N, DI(ImplicitOps, icebp), N, N,
3204 3205
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
3206
	/* 0xF8 - 0xFF */
3207 3208
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3209 3210 3211 3212 3213
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

static struct opcode twobyte_table[256] = {
	/* 0x00 - 0x0F */
3214
	G(0, group6), GD(0, &group7), N, N,
3215 3216
	N, I(ImplicitOps | VendorSpecific, em_syscall),
	II(ImplicitOps | Priv, em_clts, clts), N,
3217
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3218 3219 3220 3221
	N, D(ImplicitOps | ModRM), N, N,
	/* 0x10 - 0x1F */
	N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
	/* 0x20 - 0x2F */
3222
	DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3223
	DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3224
	DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3225
	DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3226 3227 3228
	N, N, N, N,
	N, N, N, N, N, N, N, N,
	/* 0x30 - 0x3F */
3229 3230 3231 3232
	DI(ImplicitOps | Priv, wrmsr),
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
	DI(ImplicitOps | Priv, rdmsr),
	DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3233 3234
	I(ImplicitOps | VendorSpecific, em_sysenter),
	I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3235
	N, N,
3236 3237 3238 3239 3240 3241
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
	X16(D(DstReg | SrcMem | ModRM | Mov)),
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
3242 3243 3244 3245
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3246
	/* 0x70 - 0x7F */
3247 3248 3249 3250
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3251 3252 3253
	/* 0x80 - 0x8F */
	X16(D(SrcImm)),
	/* 0x90 - 0x9F */
3254
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3255 3256
	/* 0xA0 - 0xA7 */
	D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3257
	DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3258 3259 3260 3261
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM), N, N,
	/* 0xA8 - 0xAF */
	D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3262
	DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3263 3264
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM),
3265
	D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3266
	/* 0xB0 - 0xB7 */
3267
	D2bv(DstMem | SrcReg | ModRM | Lock),
3268 3269 3270
	D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
	D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3271 3272
	/* 0xB8 - 0xBF */
	N, N,
3273
	G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3274 3275
	D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3276
	/* 0xC0 - 0xCF */
3277
	D2bv(DstMem | SrcReg | ModRM | Lock),
3278
	N, D(DstMem | SrcReg | ModRM | Mov),
3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293
	N, N, N, GD(0, &group9),
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

#undef D
#undef N
#undef G
#undef GD
#undef I
3294
#undef GP
3295
#undef EXT
3296

3297
#undef D2bv
3298
#undef D2bvIP
3299
#undef I2bv
3300
#undef I6ALU
3301

3302
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3303 3304 3305
{
	unsigned size;

3306
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
3319
	op->addr.mem.ea = ctxt->_eip;
3320 3321 3322
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
3323
		op->val = insn_fetch(s8, ctxt);
3324 3325
		break;
	case 2:
3326
		op->val = insn_fetch(s16, ctxt);
3327 3328
		break;
	case 4:
3329
		op->val = insn_fetch(s32, ctxt);
3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348
		break;
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

3349 3350 3351 3352 3353 3354 3355 3356
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
			  unsigned d)
{
	int rc = X86EMUL_CONTINUE;

	switch (d) {
	case OpReg:
		decode_register_operand(ctxt, op,
3357
			 op == &ctxt->dst &&
3358 3359 3360
			 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
		break;
	case OpImmUByte:
3361
		rc = decode_imm(ctxt, op, 1, false);
3362 3363 3364 3365
		break;
	case OpMem:
	case OpMem64:
		if (d == OpMem64)
3366
			ctxt->memop.bytes = 8;
3367
		else
3368 3369 3370 3371 3372
			ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
	mem_common:
		*op = ctxt->memop;
		ctxt->memopp = op;
		if ((ctxt->d & BitOp) && op == &ctxt->dst)
3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396
			fetch_bit_operand(ctxt);
		op->orig_val = op->val;
		break;
	case OpAcc:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
	case OpDI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
		op->addr.mem.seg = VCPU_SREG_ES;
		op->val = 0;
		break;
	case OpDX:
		op->type = OP_REG;
		op->bytes = 2;
		op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
		fetch_register_operand(op);
		break;
3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410
	case OpCL:
		op->bytes = 1;
		op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
		break;
	case OpImmByte:
		rc = decode_imm(ctxt, op, 1, true);
		break;
	case OpOne:
		op->bytes = 1;
		op->val = 1;
		break;
	case OpImm:
		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
		break;
3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439
	case OpMem16:
		ctxt->memop.bytes = 2;
		goto mem_common;
	case OpMem32:
		ctxt->memop.bytes = 4;
		goto mem_common;
	case OpImmU16:
		rc = decode_imm(ctxt, op, 2, false);
		break;
	case OpImmU:
		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
		break;
	case OpSI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
		op->addr.mem.seg = seg_override(ctxt);
		op->val = 0;
		break;
	case OpImmFAddr:
		op->type = OP_IMM;
		op->addr.mem.ea = ctxt->_eip;
		op->bytes = ctxt->op_bytes + 2;
		insn_fetch_arr(op->valptr, op->bytes, ctxt);
		break;
	case OpMemFAddr:
		ctxt->memop.bytes = ctxt->op_bytes + 2;
		goto mem_common;
3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450
	case OpImplicit:
		/* Special instructions do their own operand decoding. */
	default:
		op->type = OP_NONE; /* Disable writeback. */
		break;
	}

done:
	return rc;
}

3451
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3452 3453 3454
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
3455
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3456
	bool op_prefix = false;
3457
	struct opcode opcode;
3458

3459 3460
	ctxt->memop.type = OP_NONE;
	ctxt->memopp = NULL;
3461 3462 3463
	ctxt->_eip = ctxt->eip;
	ctxt->fetch.start = ctxt->_eip;
	ctxt->fetch.end = ctxt->fetch.start + insn_len;
3464
	if (insn_len > 0)
3465
		memcpy(ctxt->fetch.data, insn, insn_len);
3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
3483
		return EMULATION_FAILED;
3484 3485
	}

3486 3487
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
3488 3489 3490

	/* Legacy prefixes. */
	for (;;) {
3491
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
3492
		case 0x66:	/* operand-size override */
3493
			op_prefix = true;
3494
			/* switch between 2/4 bytes */
3495
			ctxt->op_bytes = def_op_bytes ^ 6;
3496 3497 3498 3499
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
3500
				ctxt->ad_bytes = def_ad_bytes ^ 12;
3501 3502
			else
				/* switch between 2/4 bytes */
3503
				ctxt->ad_bytes = def_ad_bytes ^ 6;
3504 3505 3506 3507 3508
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
3509
			set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3510 3511 3512
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
3513
			set_seg_override(ctxt, ctxt->b & 7);
3514 3515 3516 3517
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
3518
			ctxt->rex_prefix = ctxt->b;
3519 3520
			continue;
		case 0xf0:	/* LOCK */
3521
			ctxt->lock_prefix = 1;
3522 3523 3524
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
3525
			ctxt->rep_prefix = ctxt->b;
3526 3527 3528 3529 3530 3531 3532
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

3533
		ctxt->rex_prefix = 0;
3534 3535 3536 3537 3538
	}

done_prefixes:

	/* REX prefix. */
3539 3540
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
3541 3542

	/* Opcode byte(s). */
3543
	opcode = opcode_table[ctxt->b];
3544
	/* Two-byte opcode? */
3545 3546
	if (ctxt->b == 0x0f) {
		ctxt->twobyte = 1;
3547
		ctxt->b = insn_fetch(u8, ctxt);
3548
		opcode = twobyte_table[ctxt->b];
3549
	}
3550
	ctxt->d = opcode.flags;
3551

3552 3553
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
3554
		case Group:
3555
			ctxt->modrm = insn_fetch(u8, ctxt);
3556 3557
			--ctxt->_eip;
			goffset = (ctxt->modrm >> 3) & 7;
3558 3559 3560
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
3561
			ctxt->modrm = insn_fetch(u8, ctxt);
3562 3563 3564
			--ctxt->_eip;
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
3565 3566 3567 3568 3569
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
3570
			goffset = ctxt->modrm & 7;
3571
			opcode = opcode.u.group[goffset];
3572 3573
			break;
		case Prefix:
3574
			if (ctxt->rep_prefix && op_prefix)
3575
				return EMULATION_FAILED;
3576
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3577 3578 3579 3580 3581 3582 3583 3584
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
		default:
3585
			return EMULATION_FAILED;
3586
		}
3587

3588
		ctxt->d &= ~(u64)GroupMask;
3589
		ctxt->d |= opcode.flags;
3590 3591
	}

3592 3593 3594
	ctxt->execute = opcode.u.execute;
	ctxt->check_perm = opcode.check_perm;
	ctxt->intercept = opcode.intercept;
3595 3596

	/* Unrecognised? */
3597
	if (ctxt->d == 0 || (ctxt->d & Undefined))
3598
		return EMULATION_FAILED;
3599

3600
	if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3601
		return EMULATION_FAILED;
3602

3603 3604
	if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
		ctxt->op_bytes = 8;
3605

3606
	if (ctxt->d & Op3264) {
3607
		if (mode == X86EMUL_MODE_PROT64)
3608
			ctxt->op_bytes = 8;
3609
		else
3610
			ctxt->op_bytes = 4;
3611 3612
	}

3613 3614
	if (ctxt->d & Sse)
		ctxt->op_bytes = 16;
A
Avi Kivity 已提交
3615

3616
	/* ModRM and SIB bytes. */
3617
	if (ctxt->d & ModRM) {
3618
		rc = decode_modrm(ctxt, &ctxt->memop);
3619 3620 3621
		if (!ctxt->has_seg_override)
			set_seg_override(ctxt, ctxt->modrm_seg);
	} else if (ctxt->d & MemAbs)
3622
		rc = decode_abs(ctxt, &ctxt->memop);
3623 3624 3625
	if (rc != X86EMUL_CONTINUE)
		goto done;

3626 3627
	if (!ctxt->has_seg_override)
		set_seg_override(ctxt, VCPU_SREG_DS);
3628

3629
	ctxt->memop.addr.mem.seg = seg_override(ctxt);
3630

3631 3632
	if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3633 3634 3635 3636 3637

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
3638
	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3639 3640 3641
	if (rc != X86EMUL_CONTINUE)
		goto done;

3642 3643 3644 3645
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
3646
	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3647 3648 3649
	if (rc != X86EMUL_CONTINUE)
		goto done;

3650
	/* Decode and fetch the destination operand: register or memory. */
3651
	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3652 3653

done:
3654 3655
	if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
		ctxt->memopp->addr.mem.ea += ctxt->_eip;
3656

3657
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3658 3659
}

3660 3661 3662 3663 3664 3665 3666 3667 3668
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
3669 3670 3671
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
3672
		 ((ctxt->eflags & EFLG_ZF) == 0))
3673
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
3674 3675 3676 3677 3678 3679
		    ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
		return true;

	return false;
}

3680
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3681
{
3682
	struct x86_emulate_ops *ops = ctxt->ops;
3683
	u64 msr_data;
3684
	int rc = X86EMUL_CONTINUE;
3685
	int saved_dst_type = ctxt->dst.type;
3686

3687
	ctxt->mem_read.pos = 0;
3688

3689
	if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3690
		rc = emulate_ud(ctxt);
3691 3692 3693
		goto done;
	}

3694
	/* LOCK prefix is allowed only with some instructions */
3695
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3696
		rc = emulate_ud(ctxt);
3697 3698 3699
		goto done;
	}

3700
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3701
		rc = emulate_ud(ctxt);
3702 3703 3704
		goto done;
	}

3705
	if ((ctxt->d & Sse)
3706 3707
	    && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
		|| !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
A
Avi Kivity 已提交
3708 3709 3710 3711
		rc = emulate_ud(ctxt);
		goto done;
	}

3712
	if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
A
Avi Kivity 已提交
3713 3714 3715 3716
		rc = emulate_nm(ctxt);
		goto done;
	}

3717 3718
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3719
					      X86_ICPT_PRE_EXCEPT);
3720 3721 3722 3723
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3724
	/* Privileged instruction can be executed only in CPL=0 */
3725
	if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3726
		rc = emulate_gp(ctxt, 0);
3727 3728 3729
		goto done;
	}

3730
	/* Instruction can only be executed in protected mode */
3731
	if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3732 3733 3734 3735
		rc = emulate_ud(ctxt);
		goto done;
	}

3736
	/* Do instruction specific permission checks */
3737 3738
	if (ctxt->check_perm) {
		rc = ctxt->check_perm(ctxt);
3739 3740 3741 3742
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3743 3744
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3745
					      X86_ICPT_POST_EXCEPT);
3746 3747 3748 3749
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3750
	if (ctxt->rep_prefix && (ctxt->d & String)) {
3751
		/* All REP prefixes have the same first termination condition */
3752 3753
		if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
			ctxt->eip = ctxt->_eip;
3754 3755 3756 3757
			goto done;
		}
	}

3758 3759 3760
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
3761
		if (rc != X86EMUL_CONTINUE)
3762
			goto done;
3763
		ctxt->src.orig_val64 = ctxt->src.val64;
3764 3765
	}

3766 3767 3768
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
3769 3770 3771 3772
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3773
	if ((ctxt->d & DstMask) == ImplicitOps)
3774 3775 3776
		goto special_insn;


3777
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3778
		/* optimisation - avoid slow emulated read if Mov */
3779 3780
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
3781 3782
		if (rc != X86EMUL_CONTINUE)
			goto done;
3783
	}
3784
	ctxt->dst.orig_val = ctxt->dst.val;
3785

3786 3787
special_insn:

3788 3789
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3790
					      X86_ICPT_POST_MEMACCESS);
3791 3792 3793 3794
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3795 3796
	if (ctxt->execute) {
		rc = ctxt->execute(ctxt);
3797 3798 3799 3800 3801
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

3802
	if (ctxt->twobyte)
A
Avi Kivity 已提交
3803 3804
		goto twobyte_insn;

3805
	switch (ctxt->b) {
3806
	case 0x06:		/* push es */
3807
		rc = emulate_push_sreg(ctxt, VCPU_SREG_ES);
3808 3809
		break;
	case 0x07:		/* pop es */
3810
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_ES);
3811 3812
		break;
	case 0x0e:		/* push cs */
3813
		rc = emulate_push_sreg(ctxt, VCPU_SREG_CS);
3814 3815
		break;
	case 0x16:		/* push ss */
3816
		rc = emulate_push_sreg(ctxt, VCPU_SREG_SS);
3817 3818
		break;
	case 0x17:		/* pop ss */
3819
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_SS);
3820 3821
		break;
	case 0x1e:		/* push ds */
3822
		rc = emulate_push_sreg(ctxt, VCPU_SREG_DS);
3823 3824
		break;
	case 0x1f:		/* pop ds */
3825
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_DS);
3826
		break;
3827
	case 0x40 ... 0x47: /* inc r16/r32 */
3828
		emulate_1op(ctxt, "inc");
3829 3830
		break;
	case 0x48 ... 0x4f: /* dec r16/r32 */
3831
		emulate_1op(ctxt, "dec");
3832
		break;
A
Avi Kivity 已提交
3833
	case 0x63:		/* movsxd */
3834
		if (ctxt->mode != X86EMUL_MODE_PROT64)
A
Avi Kivity 已提交
3835
			goto cannot_emulate;
3836
		ctxt->dst.val = (s32) ctxt->src.val;
A
Avi Kivity 已提交
3837
		break;
3838 3839
	case 0x6c:		/* insb */
	case 0x6d:		/* insw/insd */
3840
		ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3841
		goto do_io_in;
3842 3843
	case 0x6e:		/* outsb */
	case 0x6f:		/* outsw/outsd */
3844
		ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3845
		goto do_io_out;
3846
		break;
3847
	case 0x70 ... 0x7f: /* jcc (short) */
3848 3849
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
3850
		break;
N
Nitin A Kamble 已提交
3851
	case 0x8d: /* lea r16/r32, m */
3852
		ctxt->dst.val = ctxt->src.addr.mem.ea;
N
Nitin A Kamble 已提交
3853
		break;
A
Avi Kivity 已提交
3854
	case 0x8f:		/* pop (sole member of Grp1a) */
3855
		rc = em_grp1a(ctxt);
A
Avi Kivity 已提交
3856
		break;
3857
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
3858
		if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3859
			break;
3860 3861
		rc = em_xchg(ctxt);
		break;
3862
	case 0x98: /* cbw/cwde/cdqe */
3863 3864 3865 3866
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3867 3868
		}
		break;
3869
	case 0xc0 ... 0xc1:
3870
		rc = em_grp2(ctxt);
3871
		break;
3872
	case 0xc4:		/* les */
3873
		rc = emulate_load_segment(ctxt, VCPU_SREG_ES);
3874 3875
		break;
	case 0xc5:		/* lds */
3876
		rc = emulate_load_segment(ctxt, VCPU_SREG_DS);
3877
		break;
3878
	case 0xcc:		/* int3 */
3879 3880
		rc = emulate_int(ctxt, 3);
		break;
3881
	case 0xcd:		/* int n */
3882
		rc = emulate_int(ctxt, ctxt->src.val);
3883 3884
		break;
	case 0xce:		/* into */
3885 3886
		if (ctxt->eflags & EFLG_OF)
			rc = emulate_int(ctxt, 4);
3887
		break;
3888
	case 0xd0 ... 0xd1:	/* Grp2 */
3889
		rc = em_grp2(ctxt);
3890 3891
		break;
	case 0xd2 ... 0xd3:	/* Grp2 */
3892
		ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3893
		rc = em_grp2(ctxt);
3894
		break;
3895 3896
	case 0xe4: 	/* inb */
	case 0xe5: 	/* in */
3897
		goto do_io_in;
3898 3899
	case 0xe6: /* outb */
	case 0xe7: /* out */
3900
		goto do_io_out;
3901
	case 0xe8: /* call (near) */ {
3902 3903 3904
		long int rel = ctxt->src.val;
		ctxt->src.val = (unsigned long) ctxt->_eip;
		jmp_rel(ctxt, rel);
3905
		rc = em_push(ctxt);
3906
		break;
3907 3908
	}
	case 0xe9: /* jmp rel */
3909
	case 0xeb: /* jmp rel short */
3910 3911
		jmp_rel(ctxt, ctxt->src.val);
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
3912
		break;
3913 3914
	case 0xec: /* in al,dx */
	case 0xed: /* in (e/r)ax,dx */
3915
	do_io_in:
3916 3917
		if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
				     &ctxt->dst.val))
3918 3919
			goto done; /* IO is needed */
		break;
3920 3921
	case 0xee: /* out dx,al */
	case 0xef: /* out dx,(e/r)ax */
3922
	do_io_out:
3923 3924 3925
		ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				      &ctxt->src.val, 1);
		ctxt->dst.type = OP_NONE;	/* Disable writeback. */
3926
		break;
3927
	case 0xf4:              /* hlt */
3928
		ctxt->ops->halt(ctxt);
3929
		break;
3930 3931 3932 3933 3934 3935 3936
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
		ctxt->eflags ^= EFLG_CF;
		break;
	case 0xf8: /* clc */
		ctxt->eflags &= ~EFLG_CF;
		break;
3937 3938 3939
	case 0xf9: /* stc */
		ctxt->eflags |= EFLG_CF;
		break;
3940 3941 3942 3943 3944 3945
	case 0xfc: /* cld */
		ctxt->eflags &= ~EFLG_DF;
		break;
	case 0xfd: /* std */
		ctxt->eflags |= EFLG_DF;
		break;
3946
	case 0xfe: /* Grp4 */
3947
		rc = em_grp45(ctxt);
3948
		break;
3949
	case 0xff: /* Grp5 */
3950 3951
		rc = em_grp45(ctxt);
		break;
3952 3953
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
3954
	}
3955

3956 3957 3958
	if (rc != X86EMUL_CONTINUE)
		goto done;

3959
writeback:
3960
	rc = writeback(ctxt);
3961
	if (rc != X86EMUL_CONTINUE)
3962 3963
		goto done;

3964 3965 3966 3967
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
3968
	ctxt->dst.type = saved_dst_type;
3969

3970 3971 3972
	if ((ctxt->d & SrcMask) == SrcSI)
		string_addr_inc(ctxt, seg_override(ctxt),
				VCPU_REGS_RSI, &ctxt->src);
3973

3974
	if ((ctxt->d & DstMask) == DstDI)
3975
		string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3976
				&ctxt->dst);
3977

3978 3979 3980
	if (ctxt->rep_prefix && (ctxt->d & String)) {
		struct read_cache *r = &ctxt->io_read;
		register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
3981

3982 3983 3984 3985 3986
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
3987
			if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
3988 3989 3990 3991 3992 3993
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
3994
				ctxt->mem_read.end = 0;
3995 3996 3997
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
3998
		}
3999
	}
4000

4001
	ctxt->eip = ctxt->_eip;
4002 4003

done:
4004 4005
	if (rc == X86EMUL_PROPAGATE_FAULT)
		ctxt->have_exception = true;
4006 4007 4008
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

4009
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
4010 4011

twobyte_insn:
4012
	switch (ctxt->b) {
4013
	case 0x09:		/* wbinvd */
4014
		(ctxt->ops->wbinvd)(ctxt);
4015 4016
		break;
	case 0x08:		/* invd */
4017 4018 4019 4020
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
		break;
	case 0x20: /* mov cr, reg */
4021
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4022
		break;
A
Avi Kivity 已提交
4023
	case 0x21: /* mov from dr to reg */
4024
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
A
Avi Kivity 已提交
4025
		break;
4026
	case 0x22: /* mov reg, cr */
4027
		if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4028
			emulate_gp(ctxt, 0);
4029
			rc = X86EMUL_PROPAGATE_FAULT;
4030 4031
			goto done;
		}
4032
		ctxt->dst.type = OP_NONE;
4033
		break;
A
Avi Kivity 已提交
4034
	case 0x23: /* mov from reg to dr */
4035
		if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4036
				((ctxt->mode == X86EMUL_MODE_PROT64) ?
4037
				 ~0ULL : ~0U)) < 0) {
4038
			/* #UD condition is already handled by the code above */
4039
			emulate_gp(ctxt, 0);
4040
			rc = X86EMUL_PROPAGATE_FAULT;
4041 4042 4043
			goto done;
		}

4044
		ctxt->dst.type = OP_NONE;	/* no writeback */
A
Avi Kivity 已提交
4045
		break;
4046 4047
	case 0x30:
		/* wrmsr */
4048 4049 4050
		msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
			| ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
		if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4051
			emulate_gp(ctxt, 0);
4052
			rc = X86EMUL_PROPAGATE_FAULT;
4053
			goto done;
4054 4055 4056 4057 4058
		}
		rc = X86EMUL_CONTINUE;
		break;
	case 0x32:
		/* rdmsr */
4059
		if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4060
			emulate_gp(ctxt, 0);
4061
			rc = X86EMUL_PROPAGATE_FAULT;
4062
			goto done;
4063
		} else {
4064 4065
			ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
			ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4066 4067 4068
		}
		rc = X86EMUL_CONTINUE;
		break;
A
Avi Kivity 已提交
4069
	case 0x40 ... 0x4f:	/* cmov */
4070 4071 4072
		ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
		if (!test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
4073
		break;
4074
	case 0x80 ... 0x8f: /* jnz rel, etc*/
4075 4076
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
4077
		break;
4078
	case 0x90 ... 0x9f:     /* setcc r/m8 */
4079
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4080
		break;
4081
	case 0xa0:	  /* push fs */
4082
		rc = emulate_push_sreg(ctxt, VCPU_SREG_FS);
4083 4084
		break;
	case 0xa1:	 /* pop fs */
4085
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_FS);
4086
		break;
4087 4088
	case 0xa3:
	      bt:		/* bt */
4089
		ctxt->dst.type = OP_NONE;
4090
		/* only subword offset */
4091
		ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4092
		emulate_2op_SrcV_nobyte(ctxt, "bt");
4093
		break;
4094 4095
	case 0xa4: /* shld imm8, r, r/m */
	case 0xa5: /* shld cl, r, r/m */
4096
		emulate_2op_cl(ctxt, "shld");
4097
		break;
4098
	case 0xa8:	/* push gs */
4099
		rc = emulate_push_sreg(ctxt, VCPU_SREG_GS);
4100 4101
		break;
	case 0xa9:	/* pop gs */
4102
		rc = emulate_pop_sreg(ctxt, VCPU_SREG_GS);
4103
		break;
4104 4105
	case 0xab:
	      bts:		/* bts */
4106
		emulate_2op_SrcV_nobyte(ctxt, "bts");
4107
		break;
4108 4109
	case 0xac: /* shrd imm8, r, r/m */
	case 0xad: /* shrd cl, r, r/m */
4110
		emulate_2op_cl(ctxt, "shrd");
4111
		break;
4112 4113
	case 0xae:              /* clflush */
		break;
A
Avi Kivity 已提交
4114 4115 4116 4117 4118
	case 0xb0 ... 0xb1:	/* cmpxchg */
		/*
		 * Save real source value, then compare EAX against
		 * destination.
		 */
4119 4120
		ctxt->src.orig_val = ctxt->src.val;
		ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4121
		emulate_2op_SrcV(ctxt, "cmp");
4122
		if (ctxt->eflags & EFLG_ZF) {
A
Avi Kivity 已提交
4123
			/* Success: write back to memory. */
4124
			ctxt->dst.val = ctxt->src.orig_val;
A
Avi Kivity 已提交
4125 4126
		} else {
			/* Failure: write the value we saw to EAX. */
4127 4128
			ctxt->dst.type = OP_REG;
			ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
A
Avi Kivity 已提交
4129 4130
		}
		break;
4131
	case 0xb2:		/* lss */
4132
		rc = emulate_load_segment(ctxt, VCPU_SREG_SS);
4133
		break;
A
Avi Kivity 已提交
4134 4135
	case 0xb3:
	      btr:		/* btr */
4136
		emulate_2op_SrcV_nobyte(ctxt, "btr");
A
Avi Kivity 已提交
4137
		break;
4138
	case 0xb4:		/* lfs */
4139
		rc = emulate_load_segment(ctxt, VCPU_SREG_FS);
4140 4141
		break;
	case 0xb5:		/* lgs */
4142
		rc = emulate_load_segment(ctxt, VCPU_SREG_GS);
4143
		break;
A
Avi Kivity 已提交
4144
	case 0xb6 ... 0xb7:	/* movzx */
4145 4146 4147
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
						       : (u16) ctxt->src.val;
A
Avi Kivity 已提交
4148 4149
		break;
	case 0xba:		/* Grp8 */
4150
		switch (ctxt->modrm_reg & 3) {
A
Avi Kivity 已提交
4151 4152 4153 4154 4155 4156 4157 4158 4159 4160
		case 0:
			goto bt;
		case 1:
			goto bts;
		case 2:
			goto btr;
		case 3:
			goto btc;
		}
		break;
4161 4162
	case 0xbb:
	      btc:		/* btc */
4163
		emulate_2op_SrcV_nobyte(ctxt, "btc");
4164
		break;
4165 4166 4167
	case 0xbc: {		/* bsf */
		u8 zf;
		__asm__ ("bsf %2, %0; setz %1"
4168 4169
			 : "=r"(ctxt->dst.val), "=q"(zf)
			 : "r"(ctxt->src.val));
4170 4171 4172
		ctxt->eflags &= ~X86_EFLAGS_ZF;
		if (zf) {
			ctxt->eflags |= X86_EFLAGS_ZF;
4173
			ctxt->dst.type = OP_NONE;	/* Disable writeback. */
4174 4175 4176 4177 4178 4179
		}
		break;
	}
	case 0xbd: {		/* bsr */
		u8 zf;
		__asm__ ("bsr %2, %0; setz %1"
4180 4181
			 : "=r"(ctxt->dst.val), "=q"(zf)
			 : "r"(ctxt->src.val));
4182 4183 4184
		ctxt->eflags &= ~X86_EFLAGS_ZF;
		if (zf) {
			ctxt->eflags |= X86_EFLAGS_ZF;
4185
			ctxt->dst.type = OP_NONE;	/* Disable writeback. */
4186 4187 4188
		}
		break;
	}
A
Avi Kivity 已提交
4189
	case 0xbe ... 0xbf:	/* movsx */
4190 4191 4192
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
							(s16) ctxt->src.val;
A
Avi Kivity 已提交
4193
		break;
4194
	case 0xc0 ... 0xc1:	/* xadd */
4195
		emulate_2op_SrcV(ctxt, "add");
4196
		/* Write back the register source. */
4197 4198
		ctxt->src.val = ctxt->dst.orig_val;
		write_register_operand(&ctxt->src);
4199
		break;
4200
	case 0xc3:		/* movnti */
4201 4202 4203
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
							(u64) ctxt->src.val;
4204
		break;
A
Avi Kivity 已提交
4205
	case 0xc7:		/* Grp9 (cmpxchg8b) */
4206
		rc = em_grp9(ctxt);
4207
		break;
4208 4209
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4210
	}
4211 4212 4213 4214

	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
4215 4216 4217
	goto writeback;

cannot_emulate:
4218
	return EMULATION_FAILED;
A
Avi Kivity 已提交
4219
}