emulate.c 125.1 KB
Newer Older
A
Avi Kivity 已提交
1
/******************************************************************************
2
 * emulate.c
A
Avi Kivity 已提交
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
A
Avi Kivity 已提交
10 11
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
A
Avi Kivity 已提交
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
27
#include <linux/stringify.h>
A
Avi Kivity 已提交
28

29
#include "x86.h"
30
#include "tss.h"
31

32 33 34
/*
 * Operand types
 */
35 36 37 38 39 40 41 42 43
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
44 45 46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
47
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
48 49 50 51 52 53 54
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
55 56 57 58 59 60
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
61
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
62
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
63 64

#define OpBits             5  /* Width of operand field */
65
#define OpMask             ((1ull << OpBits) - 1)
66

A
Avi Kivity 已提交
67 68 69 70 71 72 73 74 75 76
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
77
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
78
/* Destination operand type. */
79 80 81 82 83 84 85 86 87 88
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
#define DstMask     (OpMask << DstShift)
A
Avi Kivity 已提交
89
/* Source operand type. */
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
106
#define SrcImm64    (OpImm64 << SrcShift)
107
#define SrcDX       (OpDX << SrcShift)
108
#define SrcMem8     (OpMem8 << SrcShift)
109
#define SrcMask     (OpMask << SrcShift)
110 111 112 113 114 115 116 117 118
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
119
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
120
#define Sse         (1<<18)     /* SSE Vector instruction */
121 122 123 124
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
125
/* Misc flags */
126
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
127
#define VendorSpecific (1<<22) /* Vendor specific instruction */
128
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
129
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
130
#define Undefined   (1<<25) /* No Such Instruction */
131
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
132
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
133
#define No64	    (1<<28)
134
#define PageTable   (1 << 29)   /* instruction used to write page table */
135
/* Source 2 operand type */
136
#define Src2Shift   (30)
137 138 139 140 141
#define Src2None    (OpNone << Src2Shift)
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
142 143 144 145 146 147
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
148
#define Src2Mask    (OpMask << Src2Shift)
A
Avi Kivity 已提交
149
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
150 151 152
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
153
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
154
#define NoWrite     ((u64)1 << 45)  /* No writeback */
A
Avi Kivity 已提交
155

156 157 158 159 160 161 162 163
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
164

165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

/*
 * fastop functions have a special calling convention:
 *
 * dst:    [rdx]:rax  (in/out)
 * src:    rbx        (in/out)
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 *
 * fastop functions are declared as taking a never-defined fastop parameter,
 * so they can't be called from C directly.
 */

struct fastop;

186
struct opcode {
187 188
	u64 flags : 56;
	u64 intercept : 8;
189
	union {
190
		int (*execute)(struct x86_emulate_ctxt *ctxt);
191 192 193
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
194
		const struct escape *esc;
195
		void (*fastop)(struct fastop *fake);
196
	} u;
197
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
198 199 200 201 202
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
203 204
};

205 206 207 208 209 210 211
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

212 213 214 215 216
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

A
Avi Kivity 已提交
217
/* EFLAGS bit definitions. */
218 219 220 221
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
222 223
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
224 225
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
A
Avi Kivity 已提交
226 227
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
228
#define EFLG_IF (1<<9)
229
#define EFLG_TF (1<<8)
A
Avi Kivity 已提交
230 231 232 233 234 235
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

236 237 238
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

A
Avi Kivity 已提交
275 276 277 278 279 280 281
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

282
#if defined(CONFIG_X86_64)
A
Avi Kivity 已提交
283 284 285 286 287 288 289 290 291 292 293 294 295 296
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
A
Avi Kivity 已提交
312 313 314 315 316 317 318 319 320

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

321 322 323 324 325 326
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

327
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
328 329 330 331 332
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
333 334
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
335
			  "=&r" (_tmp)					\
336
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
337
	} while (0)
338 339


A
Avi Kivity 已提交
340
/* Raw emulation: instruction has two explicit operands. */
341
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
342 343 344
	do {								\
		unsigned long _tmp;					\
									\
345
		switch ((ctxt)->dst.bytes) {				\
346
		case 2:							\
347
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
348 349
			break;						\
		case 4:							\
350
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
351 352
			break;						\
		case 8:							\
353
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
354 355
			break;						\
		}							\
A
Avi Kivity 已提交
356 357
	} while (0)

358
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
A
Avi Kivity 已提交
359
	do {								     \
360
		unsigned long _tmp;					     \
361
		switch ((ctxt)->dst.bytes) {				     \
A
Avi Kivity 已提交
362
		case 1:							     \
363
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
A
Avi Kivity 已提交
364 365
			break;						     \
		default:						     \
366
			__emulate_2op_nobyte(ctxt, _op,			     \
A
Avi Kivity 已提交
367 368 369 370 371 372
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
373 374
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
A
Avi Kivity 已提交
375 376

/* Source operand is byte, word, long or quad sized. */
377 378
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
379 380

/* Source operand is word, long or quad sized. */
381 382
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
A
Avi Kivity 已提交
383

384
/* Instruction has three operands and one operand is stored in ECX register */
385
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
386 387
	do {								\
		unsigned long _tmp;					\
388 389 390
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
391 392 393 394 395
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
396
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
397 398 399
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
400 401 402
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
403 404
	} while (0)

405
#define emulate_2op_cl(ctxt, _op)					\
406
	do {								\
407
		switch ((ctxt)->dst.bytes) {				\
408
		case 2:							\
409
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
410 411
			break;						\
		case 4:							\
412
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
413 414
			break;						\
		case 8:							\
415
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
416 417
			break;						\
		}							\
418 419
	} while (0)

420
#define __emulate_1op(ctxt, _op, _suffix)				\
A
Avi Kivity 已提交
421 422 423
	do {								\
		unsigned long _tmp;					\
									\
424 425 426 427
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
428
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
429 430 431 432 433
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
434
#define emulate_1op(ctxt, _op)						\
435
	do {								\
436 437 438 439 440
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
A
Avi Kivity 已提交
441 442 443
		}							\
	} while (0)

444 445 446 447 448 449 450 451 452 453 454 455 456
#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
#define FOP_RET   "ret \n\t"

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
            FOP_ALIGN \
	    "em_" #op ": \n\t"

#define FOP_END \
	    ".popsection")

457 458
#define FOPNOP() FOP_ALIGN FOP_RET

459 460 461 462 463 464 465 466 467 468 469
#define FOP1E(op,  dst) \
	FOP_ALIGN #op " %" #dst " \n\t" FOP_RET

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

470 471 472 473 474 475 476 477 478 479 480
#define FOP2E(op,  dst, src)	   \
	FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET

#define FASTOP2(op) \
	FOP_START(op) \
	FOP2E(op##b, al, bl) \
	FOP2E(op##w, ax, bx) \
	FOP2E(op##l, eax, ebx) \
	ON64(FOP2E(op##q, rax, rbx)) \
	FOP_END

481 482 483 484 485 486 487 488 489
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

490 491 492 493 494 495 496 497 498 499 500 501
#define FOP3E(op,  dst, src, src2) \
	FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
	FOP3E(op##w, ax, bx, cl) \
	FOP3E(op##l, eax, ebx, cl) \
	ON64(FOP3E(op##q, rax, rbx, cl)) \
	FOP_END

502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523
/* Special case for SETcc - 1 instruction per cc */
#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"

FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

524
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
525 526
	do {								\
		unsigned long _tmp;					\
527 528
		ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX);		\
		ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX);		\
529 530 531 532 533 534 535 536 537 538 539 540
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
541 542
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
543
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val));	\
544 545
	} while (0)

546
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
547
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
548
	do {								\
549
		switch((ctxt)->src.bytes) {				\
550
		case 1:							\
551
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
552 553
			break;						\
		case 2:							\
554
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
555 556
			break;						\
		case 4:							\
557
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
558 559
			break;						\
		case 8: ON64(						\
560
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
561 562 563 564
			break;						\
		}							\
	} while (0)

565 566 567 568 569 570
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
571 572 573 574 575 576 577 578
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
579 580 581
		.next_rip   = ctxt->eip,
	};

582
	return ctxt->ops->intercept(ctxt, &info, stage);
583 584
}

A
Avi Kivity 已提交
585 586 587 588 589
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

590
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
591
{
592
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
593 594
}

A
Avi Kivity 已提交
595 596 597 598 599 600 601 602 603 604 605
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

A
Avi Kivity 已提交
606 607 608 609 610
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

A
Avi Kivity 已提交
611
/* Access/update address held in a register, based on addressing mode. */
612
static inline unsigned long
613
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
614
{
615
	if (ctxt->ad_bytes == sizeof(unsigned long))
616 617
		return reg;
	else
618
		return reg & ad_mask(ctxt);
619 620 621
}

static inline unsigned long
622
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
623
{
624
	return address_mask(ctxt, reg);
625 626
}

627 628 629 630 631
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

632
static inline void
633
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
634
{
635 636
	ulong mask;

637
	if (ctxt->ad_bytes == sizeof(unsigned long))
638
		mask = ~0UL;
639
	else
640 641 642 643 644 645
		mask = ad_mask(ctxt);
	masked_increment(reg, mask, inc);
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
646
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
647
}
A
Avi Kivity 已提交
648

649
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
650
{
651
	register_address_increment(ctxt, &ctxt->_eip, rel);
652
}
653

654 655 656 657 658 659 660
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

661
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
662
{
663 664
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
665 666
}

667
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
668 669 670 671
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

672
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
673 674
}

675
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
676
{
677
	if (!ctxt->has_seg_override)
678 679
		return 0;

680
	return ctxt->seg_override;
681 682
}

683 684
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
685
{
686 687 688
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
689
	return X86EMUL_PROPAGATE_FAULT;
690 691
}

692 693 694 695 696
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

697
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
698
{
699
	return emulate_exception(ctxt, GP_VECTOR, err, true);
700 701
}

702 703 704 705 706
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

707
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
708
{
709
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
710 711
}

712
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
713
{
714
	return emulate_exception(ctxt, TS_VECTOR, err, true);
715 716
}

717 718
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
719
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
720 721
}

A
Avi Kivity 已提交
722 723 724 725 726
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.
 */
static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
{
	if (likely(size < 16))
		return false;

	if (ctxt->d & Aligned)
		return true;
	else if (ctxt->d & Unaligned)
		return false;
	else if (ctxt->d & Avx)
		return false;
	else
		return true;
}

770
static int __linearize(struct x86_emulate_ctxt *ctxt,
771
		     struct segmented_address addr,
772
		     unsigned size, bool write, bool fetch,
773 774
		     ulong *linear)
{
775 776
	struct desc_struct desc;
	bool usable;
777
	ulong la;
778
	u32 lim;
779
	u16 sel;
780
	unsigned cpl;
781

782
	la = seg_base(ctxt, addr.seg) + addr.ea;
783 784 785 786 787 788
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
789 790
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
791 792
		if (!usable)
			goto bad;
793 794 795
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
796 797
			goto bad;
		/* unreadable code segment */
798
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
799 800 801 802 803 804 805
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
G
Guo Chao 已提交
806
			/* expand-down segment */
807 808 809 810 811 812
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
813
		cpl = ctxt->ops->cpl(ctxt);
814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
829
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
830
		la &= (u32)-1;
831 832
	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
		return emulate_gp(ctxt, 0);
833 834
	*linear = la;
	return X86EMUL_CONTINUE;
835 836
bad:
	if (addr.seg == VCPU_SREG_SS)
837
		return emulate_ss(ctxt, sel);
838
	else
839
		return emulate_gp(ctxt, sel);
840 841
}

842 843 844 845 846 847 848 849 850
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


851 852 853 854 855
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
856 857 858
	int rc;
	ulong linear;

859
	rc = linearize(ctxt, addr, size, false, &linear);
860 861
	if (rc != X86EMUL_CONTINUE)
		return rc;
862
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
863 864
}

865 866 867 868 869 870 871 872
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
873
{
874
	struct fetch_cache *fc = &ctxt->fetch;
875
	int rc;
876
	int size, cur_size;
877

878
	if (ctxt->_eip == fc->end) {
879
		unsigned long linear;
880 881
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
882
		cur_size = fc->end - fc->start;
883 884
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
885
		rc = __linearize(ctxt, addr, size, false, true, &linear);
886
		if (unlikely(rc != X86EMUL_CONTINUE))
887
			return rc;
888 889
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
890
		if (unlikely(rc != X86EMUL_CONTINUE))
891
			return rc;
892
		fc->end += size;
893
	}
894 895
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
896
	return X86EMUL_CONTINUE;
897 898 899
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
900
			 void *dest, unsigned size)
901
{
902
	int rc;
903

904
	/* x86 instructions are limited to 15 bytes. */
905
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
906
		return X86EMUL_UNHANDLEABLE;
907
	while (size--) {
908
		rc = do_insn_fetch_byte(ctxt, dest++);
909
		if (rc != X86EMUL_CONTINUE)
910 911
			return rc;
	}
912
	return X86EMUL_CONTINUE;
913 914
}

915
/* Fetch next part of the instruction being emulated. */
916
#define insn_fetch(_type, _ctxt)					\
917
({	unsigned long _x;						\
918
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
919 920 921 922 923
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

924 925
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
926 927 928 929
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

930 931 932 933 934
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
935
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
936
			     int highbyte_regs)
A
Avi Kivity 已提交
937 938 939 940
{
	void *p;

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
941 942 943
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
A
Avi Kivity 已提交
944 945 946 947
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
948
			   struct segmented_address addr,
A
Avi Kivity 已提交
949 950 951 952 953 954 955
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
956
	rc = segmented_read_std(ctxt, addr, size, 2);
957
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
958
		return rc;
959
	addr.ea += 2;
960
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
961 962 963
	return rc;
}

964
static u8 test_cc(unsigned int condition, unsigned long flags)
965
{
966 967
	u8 rc;
	void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
968

969 970 971 972
	flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
	asm("pushq %[flags]; popf; call *%[fastop]"
	    : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
	return rc;
973 974
}

975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

A
Avi Kivity 已提交
993 994 995 996
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
997 998 999 1000 1001 1002 1003 1004
	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
1005
#ifdef CONFIG_X86_64
1006 1007 1008 1009 1010 1011 1012 1013
	case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
1025 1026 1027 1028 1029 1030 1031 1032
	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
A
Avi Kivity 已提交
1033
#ifdef CONFIG_X86_64
1034 1035 1036 1037 1038 1039 1040 1041
	case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
A
Avi Kivity 已提交
1042 1043 1044 1045 1046 1047
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

A
Avi Kivity 已提交
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
	case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
	case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
	case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
	case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
	case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
	case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
	case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
	case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
	case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
	case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
static int em_fninit(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fninit");
	ctxt->ops->put_fpu(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
{
	u16 fcw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstcw %0": "+m"(fcw));
	ctxt->ops->put_fpu(ctxt);

	/* force 2 byte destination */
	ctxt->dst.bytes = 2;
	ctxt->dst.val = fcw;

	return X86EMUL_CONTINUE;
}

static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
{
	u16 fsw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstsw %0": "+m"(fsw));
	ctxt->ops->put_fpu(ctxt);

	/* force 2 byte destination */
	ctxt->dst.bytes = 2;
	ctxt->dst.val = fsw;

	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1129
static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1130
				    struct operand *op)
1131
{
1132 1133
	unsigned reg = ctxt->modrm_reg;
	int highbyte_regs = ctxt->rex_prefix == 0;
1134

1135 1136
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
1137

1138
	if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1139 1140 1141 1142 1143 1144
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}
A
Avi Kivity 已提交
1145 1146 1147 1148 1149 1150 1151
	if (ctxt->d & Mmx) {
		reg &= 7;
		op->type = OP_MM;
		op->bytes = 8;
		op->addr.mm = reg;
		return;
	}
A
Avi Kivity 已提交
1152

1153
	op->type = OP_REG;
1154
	if (ctxt->d & ByteOp) {
1155
		op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1156 1157
		op->bytes = 1;
	} else {
1158
		op->addr.reg = decode_register(ctxt, reg, 0);
1159
		op->bytes = ctxt->op_bytes;
1160
	}
1161
	fetch_register_operand(op);
1162 1163 1164
	op->orig_val = op->val;
}

1165 1166 1167 1168 1169 1170
static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
{
	if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
		ctxt->modrm_seg = VCPU_SREG_SS;
}

1171
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1172
			struct operand *op)
1173 1174
{
	u8 sib;
1175
	int index_reg = 0, base_reg = 0, scale;
1176
	int rc = X86EMUL_CONTINUE;
1177
	ulong modrm_ea = 0;
1178

1179 1180 1181 1182
	if (ctxt->rex_prefix) {
		ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1;	/* REX.R */
		index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
		ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1183 1184
	}

1185 1186 1187 1188
	ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
	ctxt->modrm_rm |= (ctxt->modrm & 0x07);
	ctxt->modrm_seg = VCPU_SREG_DS;
1189

1190
	if (ctxt->modrm_mod == 3) {
1191
		op->type = OP_REG;
1192
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1193
		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1194
		if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1195 1196
			op->type = OP_XMM;
			op->bytes = 16;
1197 1198
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
A
Avi Kivity 已提交
1199 1200
			return rc;
		}
A
Avi Kivity 已提交
1201 1202 1203 1204 1205 1206
		if (ctxt->d & Mmx) {
			op->type = OP_MM;
			op->bytes = 8;
			op->addr.xmm = ctxt->modrm_rm & 7;
			return rc;
		}
1207
		fetch_register_operand(op);
1208 1209 1210
		return rc;
	}

1211 1212
	op->type = OP_MEM;

1213
	if (ctxt->ad_bytes == 2) {
1214 1215 1216 1217
		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1218 1219

		/* 16-bit ModR/M decode. */
1220
		switch (ctxt->modrm_mod) {
1221
		case 0:
1222
			if (ctxt->modrm_rm == 6)
1223
				modrm_ea += insn_fetch(u16, ctxt);
1224 1225
			break;
		case 1:
1226
			modrm_ea += insn_fetch(s8, ctxt);
1227 1228
			break;
		case 2:
1229
			modrm_ea += insn_fetch(u16, ctxt);
1230 1231
			break;
		}
1232
		switch (ctxt->modrm_rm) {
1233
		case 0:
1234
			modrm_ea += bx + si;
1235 1236
			break;
		case 1:
1237
			modrm_ea += bx + di;
1238 1239
			break;
		case 2:
1240
			modrm_ea += bp + si;
1241 1242
			break;
		case 3:
1243
			modrm_ea += bp + di;
1244 1245
			break;
		case 4:
1246
			modrm_ea += si;
1247 1248
			break;
		case 5:
1249
			modrm_ea += di;
1250 1251
			break;
		case 6:
1252
			if (ctxt->modrm_mod != 0)
1253
				modrm_ea += bp;
1254 1255
			break;
		case 7:
1256
			modrm_ea += bx;
1257 1258
			break;
		}
1259 1260 1261
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
1262
		modrm_ea = (u16)modrm_ea;
1263 1264
	} else {
		/* 32/64-bit ModR/M decode. */
1265
		if ((ctxt->modrm_rm & 7) == 4) {
1266
			sib = insn_fetch(u8, ctxt);
1267 1268 1269 1270
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

1271
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1272
				modrm_ea += insn_fetch(s32, ctxt);
1273
			else {
1274
				modrm_ea += reg_read(ctxt, base_reg);
1275 1276
				adjust_modrm_seg(ctxt, base_reg);
			}
1277
			if (index_reg != 4)
1278
				modrm_ea += reg_read(ctxt, index_reg) << scale;
1279
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1280
			if (ctxt->mode == X86EMUL_MODE_PROT64)
1281
				ctxt->rip_relative = 1;
1282 1283
		} else {
			base_reg = ctxt->modrm_rm;
1284
			modrm_ea += reg_read(ctxt, base_reg);
1285 1286
			adjust_modrm_seg(ctxt, base_reg);
		}
1287
		switch (ctxt->modrm_mod) {
1288
		case 0:
1289
			if (ctxt->modrm_rm == 5)
1290
				modrm_ea += insn_fetch(s32, ctxt);
1291 1292
			break;
		case 1:
1293
			modrm_ea += insn_fetch(s8, ctxt);
1294 1295
			break;
		case 2:
1296
			modrm_ea += insn_fetch(s32, ctxt);
1297 1298 1299
			break;
		}
	}
1300
	op->addr.mem.ea = modrm_ea;
1301 1302 1303 1304 1305
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
1306
		      struct operand *op)
1307
{
1308
	int rc = X86EMUL_CONTINUE;
1309

1310
	op->type = OP_MEM;
1311
	switch (ctxt->ad_bytes) {
1312
	case 2:
1313
		op->addr.mem.ea = insn_fetch(u16, ctxt);
1314 1315
		break;
	case 4:
1316
		op->addr.mem.ea = insn_fetch(u32, ctxt);
1317 1318
		break;
	case 8:
1319
		op->addr.mem.ea = insn_fetch(u64, ctxt);
1320 1321 1322 1323 1324 1325
		break;
	}
done:
	return rc;
}

1326
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1327
{
1328
	long sv = 0, mask;
1329

1330 1331
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
		mask = ~(ctxt->dst.bytes * 8 - 1);
1332

1333 1334 1335 1336
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1337

1338
		ctxt->dst.addr.mem.ea += (sv >> 3);
1339
	}
1340 1341

	/* only subword offset */
1342
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1343 1344
}

1345 1346
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1347
{
1348
	int rc;
1349
	struct read_cache *mc = &ctxt->mem_read;
A
Avi Kivity 已提交
1350

1351 1352
	if (mc->pos < mc->end)
		goto read_cached;
A
Avi Kivity 已提交
1353

1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365
	WARN_ON((mc->end + size) >= sizeof(mc->data));

	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
				      &ctxt->exception);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	mc->end += size;

read_cached:
	memcpy(dest, mc->data + mc->pos, size);
	mc->pos += size;
1366 1367
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1368

1369 1370 1371 1372 1373
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1374 1375 1376
	int rc;
	ulong linear;

1377
	rc = linearize(ctxt, addr, size, false, &linear);
1378 1379
	if (rc != X86EMUL_CONTINUE)
		return rc;
1380
	return read_emulated(ctxt, linear, data, size);
1381 1382 1383 1384 1385 1386 1387
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1388 1389 1390
	int rc;
	ulong linear;

1391
	rc = linearize(ctxt, addr, size, true, &linear);
1392 1393
	if (rc != X86EMUL_CONTINUE)
		return rc;
1394 1395
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1396 1397 1398 1399 1400 1401 1402
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1403 1404 1405
	int rc;
	ulong linear;

1406
	rc = linearize(ctxt, addr, size, true, &linear);
1407 1408
	if (rc != X86EMUL_CONTINUE)
		return rc;
1409 1410
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1411 1412
}

1413 1414 1415 1416
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1417
	struct read_cache *rc = &ctxt->io_read;
1418

1419 1420
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1421
		unsigned int count = ctxt->rep_prefix ?
1422
			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1423
		in_page = (ctxt->eflags & EFLG_DF) ?
1424 1425
			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1426 1427 1428 1429 1430
		n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
			count);
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1431
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1432 1433
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1434 1435
	}

1436 1437 1438 1439 1440 1441 1442 1443 1444
	if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
		ctxt->dst.data = rc->data + rc->pos;
		ctxt->dst.type = OP_MEM_STR;
		ctxt->dst.count = (rc->end - rc->pos) / size;
		rc->pos = rc->end;
	} else {
		memcpy(dest, rc->data + rc->pos, size);
		rc->pos += size;
	}
1445 1446
	return 1;
}
A
Avi Kivity 已提交
1447

1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463
static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
				     u16 index, struct desc_struct *desc)
{
	struct desc_ptr dt;
	ulong addr;

	ctxt->ops->get_idt(ctxt, &dt);

	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, index << 3 | 0x2);

	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
}

1464 1465 1466
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1467
	const struct x86_emulate_ops *ops = ctxt->ops;
1468

1469 1470
	if (selector & 1 << 2) {
		struct desc_struct desc;
1471 1472
		u16 sel;

1473
		memset (dt, 0, sizeof *dt);
1474
		if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1475
			return;
1476

1477 1478 1479
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
		dt->address = get_desc_base(&desc);
	} else
1480
		ops->get_gdt(ctxt, dt);
1481
}
1482

1483 1484
/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1485 1486
				   u16 selector, struct desc_struct *desc,
				   ulong *desc_addr_p)
1487 1488 1489 1490
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1491

1492
	get_descriptor_table_ptr(ctxt, selector, &dt);
1493

1494 1495
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1496

1497
	*desc_addr_p = addr = dt.address + index * 8;
1498 1499
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
1500
}
1501

1502 1503 1504 1505 1506 1507 1508
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
A
Avi Kivity 已提交
1509

1510
	get_descriptor_table_ptr(ctxt, selector, &dt);
1511

1512 1513
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
A
Avi Kivity 已提交
1514

1515
	addr = dt.address + index * 8;
1516 1517
	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
				    &ctxt->exception);
1518
}
1519

1520
/* Does not support long mode */
1521 1522 1523
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
1524
	struct desc_struct seg_desc, old_desc;
1525 1526 1527 1528
	u8 dpl, rpl, cpl;
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1529
	ulong desc_addr;
1530
	int ret;
1531
	u16 dummy;
1532

1533
	memset(&seg_desc, 0, sizeof seg_desc);
1534

1535 1536 1537
	if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
	    || ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor */
1538
		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1539 1540 1541 1542
		set_desc_base(&seg_desc, selector << 4);
		goto load;
	}

1543 1544 1545 1546 1547 1548 1549 1550
	rpl = selector & 3;
	cpl = ctxt->ops->cpl(ctxt);

	/* NULL selector is not valid for TR, CS and SS (except for long mode) */
	if ((seg == VCPU_SREG_CS
	     || (seg == VCPU_SREG_SS
		 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
	     || seg == VCPU_SREG_TR)
1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
	    && null_selector)
		goto exception;

	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

	if (null_selector) /* for NULL selector skip all following checks */
		goto load;

1561
	ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1562 1563 1564 1565 1566 1567
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
	err_vec = GP_VECTOR;

G
Guo Chao 已提交
1568
	/* can't load system descriptor into segment selector */
1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
	if (seg <= VCPU_SREG_GS && !seg_desc.s)
		goto exception;

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	dpl = seg_desc.dpl;

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1587
		break;
1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1603
		break;
1604 1605 1606
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
1607 1608 1609 1610 1611 1612
		old_desc = seg_desc;
		seg_desc.type |= 2; /* busy */
		ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
						  sizeof(seg_desc), &ctxt->exception);
		if (ret != X86EMUL_CONTINUE)
			return ret;
1613 1614 1615 1616 1617 1618
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1619
		/*
1620 1621 1622
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1623
		 */
1624 1625 1626 1627
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1628
		break;
1629 1630 1631 1632 1633
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
		seg_desc.type |= 1;
1634
		ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1635 1636 1637 1638
		if (ret != X86EMUL_CONTINUE)
			return ret;
	}
load:
1639
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1640 1641 1642 1643 1644 1645
	return X86EMUL_CONTINUE;
exception:
	emulate_exception(ctxt, err_vec, err_code, true);
	return X86EMUL_PROPAGATE_FAULT;
}

1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664
static void write_register_operand(struct operand *op)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (op->bytes) {
	case 1:
		*(u8 *)op->addr.reg = (u8)op->val;
		break;
	case 2:
		*(u16 *)op->addr.reg = (u16)op->val;
		break;
	case 4:
		*op->addr.reg = (u32)op->val;
		break;	/* 64b: zero-extend */
	case 8:
		*op->addr.reg = op->val;
		break;
	}
}

1665
static int writeback(struct x86_emulate_ctxt *ctxt)
1666 1667 1668
{
	int rc;

1669 1670 1671
	if (ctxt->d & NoWrite)
		return X86EMUL_CONTINUE;

1672
	switch (ctxt->dst.type) {
1673
	case OP_REG:
1674
		write_register_operand(&ctxt->dst);
A
Avi Kivity 已提交
1675
		break;
1676
	case OP_MEM:
1677
		if (ctxt->lock_prefix)
1678
			rc = segmented_cmpxchg(ctxt,
1679 1680 1681 1682
					       ctxt->dst.addr.mem,
					       &ctxt->dst.orig_val,
					       &ctxt->dst.val,
					       ctxt->dst.bytes);
1683
		else
1684
			rc = segmented_write(ctxt,
1685 1686 1687
					     ctxt->dst.addr.mem,
					     &ctxt->dst.val,
					     ctxt->dst.bytes);
1688 1689
		if (rc != X86EMUL_CONTINUE)
			return rc;
1690
		break;
1691 1692 1693 1694 1695 1696 1697 1698
	case OP_MEM_STR:
		rc = segmented_write(ctxt,
				ctxt->dst.addr.mem,
				ctxt->dst.data,
				ctxt->dst.bytes * ctxt->dst.count);
		if (rc != X86EMUL_CONTINUE)
			return rc;
		break;
A
Avi Kivity 已提交
1699
	case OP_XMM:
1700
		write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
A
Avi Kivity 已提交
1701
		break;
A
Avi Kivity 已提交
1702 1703 1704
	case OP_MM:
		write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
		break;
1705 1706
	case OP_NONE:
		/* no writeback */
1707
		break;
1708
	default:
1709
		break;
A
Avi Kivity 已提交
1710
	}
1711 1712
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1713

1714
static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1715
{
1716
	struct segmented_address addr;
1717

1718
	rsp_increment(ctxt, -bytes);
1719
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1720 1721
	addr.seg = VCPU_SREG_SS;

1722 1723 1724 1725 1726
	return segmented_write(ctxt, addr, data, bytes);
}

static int em_push(struct x86_emulate_ctxt *ctxt)
{
1727
	/* Disable writeback. */
1728
	ctxt->dst.type = OP_NONE;
1729
	return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1730
}
1731

1732 1733 1734 1735
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1736
	struct segmented_address addr;
1737

1738
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1739
	addr.seg = VCPU_SREG_SS;
1740
	rc = segmented_read(ctxt, addr, dest, len);
1741 1742 1743
	if (rc != X86EMUL_CONTINUE)
		return rc;

1744
	rsp_increment(ctxt, len);
1745
	return rc;
1746 1747
}

1748 1749
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1750
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1751 1752
}

1753
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1754
			void *dest, int len)
1755 1756
{
	int rc;
1757 1758
	unsigned long val, change_mask;
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1759
	int cpl = ctxt->ops->cpl(ctxt);
1760

1761
	rc = emulate_pop(ctxt, &val, len);
1762 1763
	if (rc != X86EMUL_CONTINUE)
		return rc;
1764

1765 1766
	change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
		| EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1767

1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
			change_mask |= EFLG_IOPL;
		if (cpl <= iopl)
			change_mask |= EFLG_IF;
		break;
	case X86EMUL_MODE_VM86:
1778 1779
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1780 1781 1782 1783 1784
		change_mask |= EFLG_IF;
		break;
	default: /* real mode */
		change_mask |= (EFLG_IOPL | EFLG_IF);
		break;
1785
	}
1786 1787 1788 1789 1790

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1791 1792
}

1793 1794
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1795 1796 1797 1798
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1799 1800
}

A
Avi Kivity 已提交
1801 1802 1803 1804 1805
static int em_enter(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned frame_size = ctxt->src.val;
	unsigned nesting_level = ctxt->src2.val & 31;
1806
	ulong rbp;
A
Avi Kivity 已提交
1807 1808 1809 1810

	if (nesting_level)
		return X86EMUL_UNHANDLEABLE;

1811 1812
	rbp = reg_read(ctxt, VCPU_REGS_RBP);
	rc = push(ctxt, &rbp, stack_size(ctxt));
A
Avi Kivity 已提交
1813 1814
	if (rc != X86EMUL_CONTINUE)
		return rc;
1815
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
A
Avi Kivity 已提交
1816
		      stack_mask(ctxt));
1817 1818
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
A
Avi Kivity 已提交
1819 1820 1821 1822
		      stack_mask(ctxt));
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1823 1824
static int em_leave(struct x86_emulate_ctxt *ctxt)
{
1825
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
A
Avi Kivity 已提交
1826
		      stack_mask(ctxt));
1827
	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
A
Avi Kivity 已提交
1828 1829
}

1830
static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1831
{
1832 1833
	int seg = ctxt->src2.val;

1834
	ctxt->src.val = get_segment_selector(ctxt, seg);
1835

1836
	return em_push(ctxt);
1837 1838
}

1839
static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1840
{
1841
	int seg = ctxt->src2.val;
1842 1843
	unsigned long selector;
	int rc;
1844

1845
	rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1846 1847 1848
	if (rc != X86EMUL_CONTINUE)
		return rc;

1849
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1850
	return rc;
1851 1852
}

1853
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1854
{
1855
	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1856 1857
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1858

1859 1860
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
1861
		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1862

1863
		rc = em_push(ctxt);
1864 1865
		if (rc != X86EMUL_CONTINUE)
			return rc;
1866

1867
		++reg;
1868 1869
	}

1870
	return rc;
1871 1872
}

1873 1874
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
1875
	ctxt->src.val =  (unsigned long)ctxt->eflags;
1876 1877 1878
	return em_push(ctxt);
}

1879
static int em_popa(struct x86_emulate_ctxt *ctxt)
1880
{
1881 1882
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
1883

1884 1885
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
1886
			rsp_increment(ctxt, ctxt->op_bytes);
1887 1888
			--reg;
		}
1889

1890
		rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1891 1892 1893
		if (rc != X86EMUL_CONTINUE)
			break;
		--reg;
1894
	}
1895
	return rc;
1896 1897
}

1898
static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1899
{
1900
	const struct x86_emulate_ops *ops = ctxt->ops;
1901
	int rc;
1902 1903 1904 1905 1906 1907
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
1908
	ctxt->src.val = ctxt->eflags;
1909
	rc = em_push(ctxt);
1910 1911
	if (rc != X86EMUL_CONTINUE)
		return rc;
1912 1913 1914

	ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);

1915
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1916
	rc = em_push(ctxt);
1917 1918
	if (rc != X86EMUL_CONTINUE)
		return rc;
1919

1920
	ctxt->src.val = ctxt->_eip;
1921
	rc = em_push(ctxt);
1922 1923 1924
	if (rc != X86EMUL_CONTINUE)
		return rc;

1925
	ops->get_idt(ctxt, &dt);
1926 1927 1928 1929

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

1930
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1931 1932 1933
	if (rc != X86EMUL_CONTINUE)
		return rc;

1934
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1935 1936 1937
	if (rc != X86EMUL_CONTINUE)
		return rc;

1938
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1939 1940 1941
	if (rc != X86EMUL_CONTINUE)
		return rc;

1942
	ctxt->_eip = eip;
1943 1944 1945 1946

	return rc;
}

1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
{
	int rc;

	invalidate_registers(ctxt);
	rc = __emulate_int_real(ctxt, irq);
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);
	return rc;
}

1958
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1959 1960 1961
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1962
		return __emulate_int_real(ctxt, irq);
1963 1964 1965 1966 1967 1968 1969 1970 1971 1972
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

1973
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1974
{
1975 1976 1977 1978 1979 1980 1981 1982
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
	unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
			     EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
			     EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
	unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1983

1984
	/* TODO: Add stack limit check */
1985

1986
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1987

1988 1989
	if (rc != X86EMUL_CONTINUE)
		return rc;
1990

1991 1992
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
1993

1994
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1995

1996 1997
	if (rc != X86EMUL_CONTINUE)
		return rc;
1998

1999
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2000

2001 2002
	if (rc != X86EMUL_CONTINUE)
		return rc;
2003

2004
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2005

2006 2007
	if (rc != X86EMUL_CONTINUE)
		return rc;
2008

2009
	ctxt->_eip = temp_eip;
2010 2011


2012
	if (ctxt->op_bytes == 4)
2013
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2014
	else if (ctxt->op_bytes == 2) {
2015 2016
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
2017
	}
2018 2019 2020 2021 2022

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
	ctxt->eflags |= EFLG_RESERVED_ONE_MASK;

	return rc;
2023 2024
}

2025
static int em_iret(struct x86_emulate_ctxt *ctxt)
2026
{
2027 2028
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
2029
		return emulate_iret_real(ctxt);
2030 2031 2032 2033
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
2034
	default:
2035 2036
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
2037 2038 2039
	}
}

2040 2041 2042 2043 2044
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned short sel;

2045
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2046

2047
	rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2048 2049 2050
	if (rc != X86EMUL_CONTINUE)
		return rc;

2051 2052
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2053 2054 2055
	return X86EMUL_CONTINUE;
}

2056 2057
FASTOP1(not);
FASTOP1(neg);
2058 2059
FASTOP1(inc);
FASTOP1(dec);
2060

2061 2062 2063 2064 2065 2066 2067 2068
FASTOP2CL(rol);
FASTOP2CL(ror);
FASTOP2CL(rcl);
FASTOP2CL(rcr);
FASTOP2CL(shl);
FASTOP2CL(shr);
FASTOP2CL(sar);

2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "mul", ex);
	return X86EMUL_CONTINUE;
}

static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "imul", ex);
	return X86EMUL_CONTINUE;
}

static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2086
{
2087
	u8 de = 0;
2088

2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099
	emulate_1op_rax_rdx(ctxt, "div", de);
	if (de)
		return emulate_de(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 de = 0;

	emulate_1op_rax_rdx(ctxt, "idiv", de);
2100 2101
	if (de)
		return emulate_de(ctxt);
2102
	return X86EMUL_CONTINUE;
2103 2104
}

2105
static int em_grp45(struct x86_emulate_ctxt *ctxt)
2106
{
2107
	int rc = X86EMUL_CONTINUE;
2108

2109
	switch (ctxt->modrm_reg) {
2110 2111
	case 2: /* call near abs */ {
		long int old_eip;
2112 2113 2114
		old_eip = ctxt->_eip;
		ctxt->_eip = ctxt->src.val;
		ctxt->src.val = old_eip;
2115
		rc = em_push(ctxt);
2116 2117
		break;
	}
2118
	case 4: /* jmp abs */
2119
		ctxt->_eip = ctxt->src.val;
2120
		break;
2121 2122 2123
	case 5: /* jmp far */
		rc = em_jmp_far(ctxt);
		break;
2124
	case 6:	/* push */
2125
		rc = em_push(ctxt);
2126 2127
		break;
	}
2128
	return rc;
2129 2130
}

2131
static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2132
{
2133
	u64 old = ctxt->dst.orig_val64;
2134

2135 2136 2137 2138
	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2139
		ctxt->eflags &= ~EFLG_ZF;
2140
	} else {
2141 2142
		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
			(u32) reg_read(ctxt, VCPU_REGS_RBX);
2143

2144
		ctxt->eflags |= EFLG_ZF;
2145
	}
2146
	return X86EMUL_CONTINUE;
2147 2148
}

2149 2150
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
2151 2152 2153
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
2154 2155 2156
	return em_pop(ctxt);
}

2157
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2158 2159 2160 2161
{
	int rc;
	unsigned long cs;

2162
	rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2163
	if (rc != X86EMUL_CONTINUE)
2164
		return rc;
2165 2166 2167
	if (ctxt->op_bytes == 4)
		ctxt->_eip = (u32)ctxt->_eip;
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2168
	if (rc != X86EMUL_CONTINUE)
2169
		return rc;
2170
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2171 2172 2173
	return rc;
}

2174 2175 2176 2177
static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
{
	/* Save real source value, then compare EAX against destination. */
	ctxt->src.orig_val = ctxt->src.val;
2178
	ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2179 2180 2181 2182 2183 2184 2185 2186
	emulate_2op_SrcV(ctxt, "cmp");

	if (ctxt->eflags & EFLG_ZF) {
		/* Success: write back to memory. */
		ctxt->dst.val = ctxt->src.orig_val;
	} else {
		/* Failure: write the value we saw to EAX. */
		ctxt->dst.type = OP_REG;
2187
		ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2188 2189 2190 2191
	}
	return X86EMUL_CONTINUE;
}

2192
static int em_lseg(struct x86_emulate_ctxt *ctxt)
2193
{
2194
	int seg = ctxt->src2.val;
2195 2196 2197
	unsigned short sel;
	int rc;

2198
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2199

2200
	rc = load_segment_descriptor(ctxt, sel, seg);
2201 2202 2203
	if (rc != X86EMUL_CONTINUE)
		return rc;

2204
	ctxt->dst.val = ctxt->src.val;
2205 2206 2207
	return rc;
}

2208
static void
2209
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2210
			struct desc_struct *cs, struct desc_struct *ss)
2211 2212
{
	cs->l = 0;		/* will be adjusted later */
2213
	set_desc_base(cs, 0);	/* flat segment */
2214
	cs->g = 1;		/* 4kb granularity */
2215
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
2216 2217 2218
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
2219 2220
	cs->p = 1;
	cs->d = 1;
2221
	cs->avl = 0;
2222

2223 2224
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
2225 2226 2227
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
2228
	ss->d = 1;		/* 32bit stack segment */
2229
	ss->dpl = 0;
2230
	ss->p = 1;
2231 2232
	ss->l = 0;
	ss->avl = 0;
2233 2234
}

2235 2236 2237 2238 2239
static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

	eax = ecx = 0;
2240 2241
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2242 2243 2244 2245
		&& ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
		&& edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
}

2246 2247
static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
{
2248
	const struct x86_emulate_ops *ops = ctxt->ops;
2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259
	u32 eax, ebx, ecx, edx;

	/*
	 * syscall should always be enabled in longmode - so only become
	 * vendor specific (cpuid) if other modes are active...
	 */
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return true;

	eax = 0x00000000;
	ecx = 0x00000000;
2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284
	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	/*
	 * Intel ("GenuineIntel")
	 * remark: Intel CPUs only support "syscall" in 64bit
	 * longmode. Also an 64bit guest with a
	 * 32bit compat-app running will #UD !! While this
	 * behaviour can be fixed (by emulating) into AMD
	 * response - CPUs of AMD can't behave like Intel.
	 */
	if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
		return false;

	/* AMD ("AuthenticAMD") */
	if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
		return true;

	/* AMD ("AMDisbetter!") */
	if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
		return true;
2285 2286 2287 2288 2289

	/* default: (not Intel, not AMD), apply Intel's stricter rules... */
	return false;
}

2290
static int em_syscall(struct x86_emulate_ctxt *ctxt)
2291
{
2292
	const struct x86_emulate_ops *ops = ctxt->ops;
2293
	struct desc_struct cs, ss;
2294
	u64 msr_data;
2295
	u16 cs_sel, ss_sel;
2296
	u64 efer = 0;
2297 2298

	/* syscall is not available in real mode */
2299
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2300 2301
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
2302

2303 2304 2305
	if (!(em_syscall_is_enabled(ctxt)))
		return emulate_ud(ctxt);

2306
	ops->get_msr(ctxt, MSR_EFER, &efer);
2307
	setup_syscalls_segments(ctxt, &cs, &ss);
2308

2309 2310 2311
	if (!(efer & EFER_SCE))
		return emulate_ud(ctxt);

2312
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
2313
	msr_data >>= 32;
2314 2315
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
2316

2317
	if (efer & EFER_LMA) {
2318
		cs.d = 0;
2319 2320
		cs.l = 1;
	}
2321 2322
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2323

2324
	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2325
	if (efer & EFER_LMA) {
2326
#ifdef CONFIG_X86_64
2327
		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2328

2329
		ops->get_msr(ctxt,
2330 2331
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
2332
		ctxt->_eip = msr_data;
2333

2334
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2335 2336 2337 2338
		ctxt->eflags &= ~(msr_data | EFLG_RF);
#endif
	} else {
		/* legacy mode */
2339
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
2340
		ctxt->_eip = (u32)msr_data;
2341 2342 2343 2344

		ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
	}

2345
	return X86EMUL_CONTINUE;
2346 2347
}

2348
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2349
{
2350
	const struct x86_emulate_ops *ops = ctxt->ops;
2351
	struct desc_struct cs, ss;
2352
	u64 msr_data;
2353
	u16 cs_sel, ss_sel;
2354
	u64 efer = 0;
2355

2356
	ops->get_msr(ctxt, MSR_EFER, &efer);
2357
	/* inject #GP if in real mode */
2358 2359
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
2360

2361 2362 2363 2364 2365 2366 2367 2368
	/*
	 * Not recognized on AMD in compat mode (but is recognized in legacy
	 * mode).
	 */
	if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
	    && !vendor_intel(ctxt))
		return emulate_ud(ctxt);

2369 2370 2371
	/* XXX sysenter/sysexit have not been tested in 64bit mode.
	* Therefore, we inject an #UD.
	*/
2372 2373
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return emulate_ud(ctxt);
2374

2375
	setup_syscalls_segments(ctxt, &cs, &ss);
2376

2377
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2378 2379
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT32:
2380 2381
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2382 2383
		break;
	case X86EMUL_MODE_PROT64:
2384 2385
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2386
		break;
2387 2388
	default:
		break;
2389 2390 2391
	}

	ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2392 2393 2394 2395
	cs_sel = (u16)msr_data;
	cs_sel &= ~SELECTOR_RPL_MASK;
	ss_sel = cs_sel + 8;
	ss_sel &= ~SELECTOR_RPL_MASK;
2396
	if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2397
		cs.d = 0;
2398 2399 2400
		cs.l = 1;
	}

2401 2402
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2403

2404
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2405
	ctxt->_eip = msr_data;
2406

2407
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2408
	*reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2409

2410
	return X86EMUL_CONTINUE;
2411 2412
}

2413
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2414
{
2415
	const struct x86_emulate_ops *ops = ctxt->ops;
2416
	struct desc_struct cs, ss;
2417 2418
	u64 msr_data;
	int usermode;
X
Xiao Guangrong 已提交
2419
	u16 cs_sel = 0, ss_sel = 0;
2420

2421 2422
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2423 2424
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
2425

2426
	setup_syscalls_segments(ctxt, &cs, &ss);
2427

2428
	if ((ctxt->rex_prefix & 0x8) != 0x0)
2429 2430 2431 2432 2433 2434
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

	cs.dpl = 3;
	ss.dpl = 3;
2435
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2436 2437
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
2438
		cs_sel = (u16)(msr_data + 16);
2439 2440
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2441
		ss_sel = (u16)(msr_data + 24);
2442 2443
		break;
	case X86EMUL_MODE_PROT64:
2444
		cs_sel = (u16)(msr_data + 32);
2445 2446
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2447 2448
		ss_sel = cs_sel + 8;
		cs.d = 0;
2449 2450 2451
		cs.l = 1;
		break;
	}
2452 2453
	cs_sel |= SELECTOR_RPL_MASK;
	ss_sel |= SELECTOR_RPL_MASK;
2454

2455 2456
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2457

2458 2459
	ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
	*reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2460

2461
	return X86EMUL_CONTINUE;
2462 2463
}

2464
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2465 2466 2467 2468 2469 2470 2471
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2472
	return ctxt->ops->cpl(ctxt) > iopl;
2473 2474 2475 2476 2477
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2478
	const struct x86_emulate_ops *ops = ctxt->ops;
2479
	struct desc_struct tr_seg;
2480
	u32 base3;
2481
	int r;
2482
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2483
	unsigned mask = (1 << len) - 1;
2484
	unsigned long base;
2485

2486
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2487
	if (!tr_seg.p)
2488
		return false;
2489
	if (desc_limit_scaled(&tr_seg) < 103)
2490
		return false;
2491 2492 2493 2494
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2495
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2496 2497
	if (r != X86EMUL_CONTINUE)
		return false;
2498
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2499
		return false;
2500
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2501 2502 2503 2504 2505 2506 2507 2508 2509 2510
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2511 2512 2513
	if (ctxt->perm_ok)
		return true;

2514 2515
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2516
			return false;
2517 2518 2519

	ctxt->perm_ok = true;

2520 2521 2522
	return true;
}

2523 2524 2525
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
2526
	tss->ip = ctxt->_eip;
2527
	tss->flag = ctxt->eflags;
2528 2529 2530 2531 2532 2533 2534 2535
	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->si = reg_read(ctxt, VCPU_REGS_RSI);
	tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2536

2537 2538 2539 2540 2541
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2542 2543 2544 2545 2546 2547 2548
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;

2549
	ctxt->_eip = tss->ip;
2550
	ctxt->eflags = tss->flag | 2;
2551 2552 2553 2554 2555 2556 2557 2558
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2559 2560 2561 2562 2563

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2564 2565 2566 2567 2568
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2569 2570

	/*
G
Guo Chao 已提交
2571
	 * Now load segment descriptors. If fault happens at this stage
2572 2573
	 * it is handled in a context of new task
	 */
2574
	ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2575 2576
	if (ret != X86EMUL_CONTINUE)
		return ret;
2577
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2578 2579
	if (ret != X86EMUL_CONTINUE)
		return ret;
2580
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2581 2582
	if (ret != X86EMUL_CONTINUE)
		return ret;
2583
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2584 2585
	if (ret != X86EMUL_CONTINUE)
		return ret;
2586
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2597
	const struct x86_emulate_ops *ops = ctxt->ops;
2598 2599
	struct tss_segment_16 tss_seg;
	int ret;
2600
	u32 new_tss_base = get_desc_base(new_desc);
2601

2602
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2603
			    &ctxt->exception);
2604
	if (ret != X86EMUL_CONTINUE)
2605 2606 2607
		/* FIXME: need to provide precise fault address */
		return ret;

2608
	save_state_to_tss16(ctxt, &tss_seg);
2609

2610
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2611
			     &ctxt->exception);
2612
	if (ret != X86EMUL_CONTINUE)
2613 2614 2615
		/* FIXME: need to provide precise fault address */
		return ret;

2616
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2617
			    &ctxt->exception);
2618
	if (ret != X86EMUL_CONTINUE)
2619 2620 2621 2622 2623 2624
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2625
		ret = ops->write_std(ctxt, new_tss_base,
2626 2627
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2628
				     &ctxt->exception);
2629
		if (ret != X86EMUL_CONTINUE)
2630 2631 2632 2633
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2634
	return load_state_from_tss16(ctxt, &tss_seg);
2635 2636 2637 2638 2639
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
2640
	tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2641
	tss->eip = ctxt->_eip;
2642
	tss->eflags = ctxt->eflags;
2643 2644 2645 2646 2647 2648 2649 2650
	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2651

2652 2653 2654 2655 2656 2657 2658
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
	tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2659 2660 2661 2662 2663 2664 2665
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;

2666
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2667
		return emulate_gp(ctxt, 0);
2668
	ctxt->_eip = tss->eip;
2669
	ctxt->eflags = tss->eflags | 2;
2670 2671

	/* General purpose registers */
2672 2673 2674 2675 2676 2677 2678 2679
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2680 2681 2682 2683 2684

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2685 2686 2687 2688 2689 2690 2691
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2692

2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710
	/*
	 * If we're switching between Protected Mode and VM86, we need to make
	 * sure to update the mode before loading the segment descriptors so
	 * that the selectors are interpreted correctly.
	 *
	 * Need to get rflags to the vcpu struct immediately because it
	 * influences the CPL which is checked at least when loading the segment
	 * descriptors and when pushing an error code to the new kernel stack.
	 *
	 * TODO Introduce a separate ctxt->ops->set_cpl callback
	 */
	if (ctxt->eflags & X86_EFLAGS_VM)
		ctxt->mode = X86EMUL_MODE_VM86;
	else
		ctxt->mode = X86EMUL_MODE_PROT32;

	ctxt->ops->set_rflags(ctxt, ctxt->eflags);

2711 2712 2713 2714
	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2715
	ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2716 2717
	if (ret != X86EMUL_CONTINUE)
		return ret;
2718
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2719 2720
	if (ret != X86EMUL_CONTINUE)
		return ret;
2721
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2722 2723
	if (ret != X86EMUL_CONTINUE)
		return ret;
2724
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2725 2726
	if (ret != X86EMUL_CONTINUE)
		return ret;
2727
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2728 2729
	if (ret != X86EMUL_CONTINUE)
		return ret;
2730
	ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2731 2732
	if (ret != X86EMUL_CONTINUE)
		return ret;
2733
	ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2734 2735 2736 2737 2738 2739 2740 2741 2742 2743
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2744
	const struct x86_emulate_ops *ops = ctxt->ops;
2745 2746
	struct tss_segment_32 tss_seg;
	int ret;
2747
	u32 new_tss_base = get_desc_base(new_desc);
2748

2749
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2750
			    &ctxt->exception);
2751
	if (ret != X86EMUL_CONTINUE)
2752 2753 2754
		/* FIXME: need to provide precise fault address */
		return ret;

2755
	save_state_to_tss32(ctxt, &tss_seg);
2756

2757
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2758
			     &ctxt->exception);
2759
	if (ret != X86EMUL_CONTINUE)
2760 2761 2762
		/* FIXME: need to provide precise fault address */
		return ret;

2763
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2764
			    &ctxt->exception);
2765
	if (ret != X86EMUL_CONTINUE)
2766 2767 2768 2769 2770 2771
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2772
		ret = ops->write_std(ctxt, new_tss_base,
2773 2774
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2775
				     &ctxt->exception);
2776
		if (ret != X86EMUL_CONTINUE)
2777 2778 2779 2780
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2781
	return load_state_from_tss32(ctxt, &tss_seg);
2782 2783 2784
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2785
				   u16 tss_selector, int idt_index, int reason,
2786
				   bool has_error_code, u32 error_code)
2787
{
2788
	const struct x86_emulate_ops *ops = ctxt->ops;
2789 2790
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
2791
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2792
	ulong old_tss_base =
2793
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2794
	u32 desc_limit;
2795
	ulong desc_addr;
2796 2797 2798

	/* FIXME: old_tss_base == ~0 ? */

2799
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2800 2801
	if (ret != X86EMUL_CONTINUE)
		return ret;
2802
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2803 2804 2805 2806 2807
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

2808 2809 2810 2811 2812
	/*
	 * Check privileges. The three cases are task switch caused by...
	 *
	 * 1. jmp/call/int to task gate: Check against DPL of the task gate
	 * 2. Exception/IRQ/iret: No check is performed
G
Guo Chao 已提交
2813
	 * 3. jmp/call to TSS: Check against DPL of the TSS
2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833
	 */
	if (reason == TASK_SWITCH_GATE) {
		if (idt_index != -1) {
			/* Software interrupts */
			struct desc_struct task_gate_desc;
			int dpl;

			ret = read_interrupt_descriptor(ctxt, idt_index,
							&task_gate_desc);
			if (ret != X86EMUL_CONTINUE)
				return ret;

			dpl = task_gate_desc.dpl;
			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
				return emulate_gp(ctxt, (idt_index << 3) | 0x2);
		}
	} else if (reason != TASK_SWITCH_IRET) {
		int dpl = next_tss_desc.dpl;
		if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
			return emulate_gp(ctxt, tss_selector);
2834 2835
	}

2836

2837 2838 2839 2840
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
2841
		emulate_ts(ctxt, tss_selector & 0xfffc);
2842 2843 2844 2845 2846
		return X86EMUL_PROPAGATE_FAULT;
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2847
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2848 2849 2850 2851 2852 2853
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
G
Guo Chao 已提交
2854
	   note that old_tss_sel is not used after this point */
2855 2856 2857 2858
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
2859
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2860 2861
				     old_tss_base, &next_tss_desc);
	else
2862
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2863
				     old_tss_base, &next_tss_desc);
2864 2865
	if (ret != X86EMUL_CONTINUE)
		return ret;
2866 2867 2868 2869 2870 2871

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
2872
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2873 2874
	}

2875
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
2876
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2877

2878
	if (has_error_code) {
2879 2880 2881
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
2882
		ret = em_push(ctxt);
2883 2884
	}

2885 2886 2887 2888
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2889
			 u16 tss_selector, int idt_index, int reason,
2890
			 bool has_error_code, u32 error_code)
2891 2892 2893
{
	int rc;

2894
	invalidate_registers(ctxt);
2895 2896
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
2897

2898
	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2899
				     has_error_code, error_code);
2900

2901
	if (rc == X86EMUL_CONTINUE) {
2902
		ctxt->eip = ctxt->_eip;
2903 2904
		writeback_registers(ctxt);
	}
2905

2906
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2907 2908
}

2909 2910
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
		struct operand *op)
2911
{
2912
	int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2913

2914 2915
	register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2916 2917
}

2918 2919 2920 2921 2922 2923
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
2924
	al = ctxt->dst.val;
2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

2942
	ctxt->dst.val = al;
2943
	/* Set PF, ZF, SF */
2944 2945 2946
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
2947
	emulate_2op_SrcV(ctxt, "or");
2948 2949 2950 2951 2952 2953 2954 2955
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976
static int em_aad(struct x86_emulate_ctxt *ctxt)
{
	u8 al = ctxt->dst.val & 0xff;
	u8 ah = (ctxt->dst.val >> 8) & 0xff;

	al = (al + (ah * ctxt->src.val)) & 0xff;

	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;

	ctxt->eflags &= ~(X86_EFLAGS_PF | X86_EFLAGS_SF | X86_EFLAGS_ZF);

	if (!al)
		ctxt->eflags |= X86_EFLAGS_ZF;
	if (!(al & 1))
		ctxt->eflags |= X86_EFLAGS_PF;
	if (al & 0x80)
		ctxt->eflags |= X86_EFLAGS_SF;

	return X86EMUL_CONTINUE;
}

2977 2978 2979 2980 2981 2982 2983 2984 2985
static int em_call(struct x86_emulate_ctxt *ctxt)
{
	long rel = ctxt->src.val;

	ctxt->src.val = (unsigned long)ctxt->_eip;
	jmp_rel(ctxt, rel);
	return em_push(ctxt);
}

2986 2987 2988 2989 2990 2991
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;

2992
	old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2993
	old_eip = ctxt->_eip;
2994

2995
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2996
	if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2997 2998
		return X86EMUL_CONTINUE;

2999 3000
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3001

3002
	ctxt->src.val = old_cs;
3003
	rc = em_push(ctxt);
3004 3005 3006
	if (rc != X86EMUL_CONTINUE)
		return rc;

3007
	ctxt->src.val = old_eip;
3008
	return em_push(ctxt);
3009 3010
}

3011 3012 3013 3014
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;

3015 3016 3017 3018
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
	rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3019 3020
	if (rc != X86EMUL_CONTINUE)
		return rc;
3021
	rsp_increment(ctxt, ctxt->src.val);
3022 3023 3024
	return X86EMUL_CONTINUE;
}

3025 3026 3027 3028 3029 3030 3031 3032 3033
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);
3034

3035 3036 3037
FASTOP3WCL(shld);
FASTOP3WCL(shrd);

3038 3039 3040
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
3041 3042
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
3043 3044

	/* Write back the memory destination with implicit LOCK prefix. */
3045 3046
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
3047 3048 3049
	return X86EMUL_CONTINUE;
}

3050
static int em_imul(struct x86_emulate_ctxt *ctxt)
3051
{
3052
	emulate_2op_SrcV_nobyte(ctxt, "imul");
3053 3054 3055
	return X86EMUL_CONTINUE;
}

3056 3057
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
3058
	ctxt->dst.val = ctxt->src2.val;
3059 3060 3061
	return em_imul(ctxt);
}

3062 3063
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
3064 3065
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
3066
	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3067
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3068 3069 3070 3071

	return X86EMUL_CONTINUE;
}

3072 3073 3074 3075
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

3076
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3077 3078
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3079 3080 3081
	return X86EMUL_CONTINUE;
}

3082 3083 3084 3085
static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
{
	u64 pmc;

3086
	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3087
		return emulate_gp(ctxt, 0);
3088 3089
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3090 3091 3092
	return X86EMUL_CONTINUE;
}

3093 3094
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
S
Stefan Hajnoczi 已提交
3095
	memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3096 3097 3098
	return X86EMUL_CONTINUE;
}

3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126
static int em_cr_write(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_dr_write(struct x86_emulate_ctxt *ctxt)
{
	unsigned long val;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		val = ctxt->src.val & ~0ULL;
	else
		val = ctxt->src.val & ~0U;

	/* #UD condition is already handled. */
	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3127 3128 3129 3130
static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3131 3132 3133
	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
	if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3134 3135 3136 3137 3138 3139 3140 3141 3142
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3143
	if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3144 3145
		return emulate_gp(ctxt, 0);

3146 3147
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
	*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3148 3149 3150
	return X86EMUL_CONTINUE;
}

3151 3152
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
3153
	if (ctxt->modrm_reg > VCPU_SREG_GS)
3154 3155
		return emulate_ud(ctxt);

3156
	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3157 3158 3159 3160 3161
	return X86EMUL_CONTINUE;
}

static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
3162
	u16 sel = ctxt->src.val;
3163

3164
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3165 3166
		return emulate_ud(ctxt);

3167
	if (ctxt->modrm_reg == VCPU_SREG_SS)
3168 3169 3170
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
3171 3172
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3173 3174
}

A
Avi Kivity 已提交
3175 3176 3177 3178 3179 3180 3181 3182 3183
static int em_lldt(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
}

A
Avi Kivity 已提交
3184 3185 3186 3187 3188 3189 3190 3191 3192
static int em_ltr(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
}

3193 3194
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
3195 3196 3197
	int rc;
	ulong linear;

3198
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3199
	if (rc == X86EMUL_CONTINUE)
3200
		ctxt->ops->invlpg(ctxt, linear);
3201
	/* Disable writeback. */
3202
	ctxt->dst.type = OP_NONE;
3203 3204 3205
	return X86EMUL_CONTINUE;
}

3206 3207 3208 3209 3210 3211 3212 3213 3214 3215
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

3216 3217 3218 3219
static int em_vmcall(struct x86_emulate_ctxt *ctxt)
{
	int rc;

3220
	if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3221 3222 3223 3224 3225 3226 3227
		return X86EMUL_UNHANDLEABLE;

	rc = ctxt->ops->fix_hypercall(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
3228
	ctxt->_eip = ctxt->eip;
3229
	/* Disable writeback. */
3230
	ctxt->dst.type = OP_NONE;
3231 3232 3233
	return X86EMUL_CONTINUE;
}

3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262
static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
				  void (*get)(struct x86_emulate_ctxt *ctxt,
					      struct desc_ptr *ptr))
{
	struct desc_ptr desc_ptr;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
	get(ctxt, &desc_ptr);
	if (ctxt->op_bytes == 2) {
		ctxt->op_bytes = 4;
		desc_ptr.address &= 0x00ffffff;
	}
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return segmented_write(ctxt, ctxt->dst.addr.mem,
			       &desc_ptr, 2 + ctxt->op_bytes);
}

static int em_sgdt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
}

static int em_sidt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
}

3263 3264 3265 3266 3267
static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

3268 3269
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
3270
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3271
			     &desc_ptr.size, &desc_ptr.address,
3272
			     ctxt->op_bytes);
3273 3274 3275 3276
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_gdt(ctxt, &desc_ptr);
	/* Disable writeback. */
3277
	ctxt->dst.type = OP_NONE;
3278 3279 3280
	return X86EMUL_CONTINUE;
}

3281
static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3282 3283 3284
{
	int rc;

3285 3286
	rc = ctxt->ops->fix_hypercall(ctxt);

3287
	/* Disable writeback. */
3288
	ctxt->dst.type = OP_NONE;
3289 3290 3291 3292 3293 3294 3295 3296
	return rc;
}

static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

3297 3298
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
3299
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3300
			     &desc_ptr.size, &desc_ptr.address,
3301
			     ctxt->op_bytes);
3302 3303 3304 3305
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_idt(ctxt, &desc_ptr);
	/* Disable writeback. */
3306
	ctxt->dst.type = OP_NONE;
3307 3308 3309 3310 3311
	return X86EMUL_CONTINUE;
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
3312 3313
	ctxt->dst.bytes = 2;
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3314 3315 3316 3317 3318 3319
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3320 3321
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
3322 3323 3324
	return X86EMUL_CONTINUE;
}

3325 3326
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
3327 3328
	register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3329 3330
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
		jmp_rel(ctxt, ctxt->src.val);
3331 3332 3333 3334 3335 3336

	return X86EMUL_CONTINUE;
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
3337
	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3338
		jmp_rel(ctxt, ctxt->src.val);
3339 3340 3341 3342

	return X86EMUL_CONTINUE;
}

3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360
static int em_in(struct x86_emulate_ctxt *ctxt)
{
	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
			     &ctxt->dst.val))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
}

static int em_out(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				    &ctxt->src.val, 1);
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408
static int em_bt(struct x86_emulate_ctxt *ctxt)
{
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	/* only subword offset */
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;

	emulate_2op_SrcV_nobyte(ctxt, "bt");
	return X86EMUL_CONTINUE;
}

static int em_bts(struct x86_emulate_ctxt *ctxt)
{
	emulate_2op_SrcV_nobyte(ctxt, "bts");
	return X86EMUL_CONTINUE;
}

static int em_btr(struct x86_emulate_ctxt *ctxt)
{
	emulate_2op_SrcV_nobyte(ctxt, "btr");
	return X86EMUL_CONTINUE;
}

static int em_btc(struct x86_emulate_ctxt *ctxt)
{
	emulate_2op_SrcV_nobyte(ctxt, "btc");
	return X86EMUL_CONTINUE;
}

3409 3410
static int em_bsf(struct x86_emulate_ctxt *ctxt)
{
3411
	emulate_2op_SrcV_nobyte(ctxt, "bsf");
3412 3413 3414 3415 3416
	return X86EMUL_CONTINUE;
}

static int em_bsr(struct x86_emulate_ctxt *ctxt)
{
3417
	emulate_2op_SrcV_nobyte(ctxt, "bsr");
3418 3419 3420
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3421 3422 3423 3424
static int em_cpuid(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

3425 3426
	eax = reg_read(ctxt, VCPU_REGS_RAX);
	ecx = reg_read(ctxt, VCPU_REGS_RCX);
A
Avi Kivity 已提交
3427
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3428 3429 3430 3431
	*reg_write(ctxt, VCPU_REGS_RAX) = eax;
	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;
	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = edx;
A
Avi Kivity 已提交
3432 3433 3434
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3435 3436
static int em_lahf(struct x86_emulate_ctxt *ctxt)
{
3437 3438
	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
A
Avi Kivity 已提交
3439 3440 3441
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456
static int em_bswap(struct x86_emulate_ctxt *ctxt)
{
	switch (ctxt->op_bytes) {
#ifdef CONFIG_X86_64
	case 8:
		asm("bswap %0" : "+r"(ctxt->dst.val));
		break;
#endif
	default:
		asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
		break;
	}
	return X86EMUL_CONTINUE;
}

3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
3471
	if (!valid_cr(ctxt->modrm_reg))
3472 3473 3474 3475 3476 3477 3478
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
3479 3480
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
3481
	u64 efer = 0;
3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
3499
		u64 cr4;
3500 3501 3502 3503
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

3504 3505
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3506 3507 3508 3509 3510 3511 3512 3513 3514 3515

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

3516 3517
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
3518
			rsvd = CR3_L_MODE_RESERVED_BITS;
3519
		else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3520
			rsvd = CR3_PAE_RESERVED_BITS;
3521
		else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3522 3523 3524 3525 3526 3527 3528 3529
			rsvd = CR3_NONPAE_RESERVED_BITS;

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
3530
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

3542 3543 3544 3545
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

3546
	ctxt->ops->get_dr(ctxt, 7, &dr7);
3547 3548 3549 3550 3551 3552 3553

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
3554
	int dr = ctxt->modrm_reg;
3555 3556 3557 3558 3559
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

3560
	cr4 = ctxt->ops->get_cr(ctxt, 4);
3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

	if (check_dr7_gd(ctxt))
		return emulate_db(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
3572 3573
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
3574 3575 3576 3577 3578 3579 3580

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

3581 3582 3583 3584
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
	u64 efer;

3585
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3586 3587 3588 3589 3590 3591 3592 3593 3594

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
3595
	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3596 3597

	/* Valid physical address? */
3598
	if (rax & 0xffff000000000000ULL)
3599 3600 3601 3602 3603
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

3604 3605
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
3606
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3607

3608
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3609 3610 3611 3612 3613
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

3614 3615
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
3616
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3617
	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3618

3619
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3620 3621 3622 3623 3624 3625
	    (rcx > 3))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3626 3627
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
3628 3629
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3630 3631 3632 3633 3634 3635 3636
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
3637 3638
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3639 3640 3641 3642 3643
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3644
#define D(_y) { .flags = (_y) }
3645
#define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3646 3647
#define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
		      .check_perm = (_p) }
3648
#define N    D(0)
3649
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3650 3651
#define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
#define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3652
#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3653
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3654
#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3655 3656
#define II(_f, _e, _i) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3657 3658 3659
#define IIP(_f, _e, _i, _p) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
	  .check_perm = (_p) }
3660
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3661

3662
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
3663
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3664
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
3665
#define F2bv(_f, _e)  F((_f) | ByteOp, _e), F(_f, _e)
3666 3667
#define I2bvIP(_f, _e, _i, _p) \
	IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3668

3669 3670 3671
#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3672

3673
static const struct opcode group7_rm1[] = {
3674 3675
	DI(SrcNone | Priv, monitor),
	DI(SrcNone | Priv, mwait),
3676 3677 3678
	N, N, N, N, N, N,
};

3679
static const struct opcode group7_rm3[] = {
3680 3681 3682 3683 3684 3685 3686 3687
	DIP(SrcNone | Prot | Priv,		vmrun,		check_svme_pa),
	II(SrcNone  | Prot | VendorSpecific,	em_vmmcall,	vmmcall),
	DIP(SrcNone | Prot | Priv,		vmload,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		vmsave,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		stgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		clgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		skinit,		check_svme),
	DIP(SrcNone | Prot | Priv,		invlpga,	check_svme),
3688
};
3689

3690
static const struct opcode group7_rm7[] = {
3691
	N,
3692
	DIP(SrcNone, rdtscp, check_rdtsc),
3693 3694
	N, N, N, N, N, N,
};
3695

3696
static const struct opcode group1[] = {
3697 3698 3699 3700 3701 3702 3703 3704
	F(Lock, em_add),
	F(Lock | PageTable, em_or),
	F(Lock, em_adc),
	F(Lock, em_sbb),
	F(Lock | PageTable, em_and),
	F(Lock, em_sub),
	F(Lock, em_xor),
	F(NoWrite, em_cmp),
3705 3706
};

3707
static const struct opcode group1A[] = {
3708
	I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3709 3710
};

3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721
static const struct opcode group2[] = {
	F(DstMem | ModRM, em_rol),
	F(DstMem | ModRM, em_ror),
	F(DstMem | ModRM, em_rcl),
	F(DstMem | ModRM, em_rcr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_shr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_sar),
};

3722
static const struct opcode group3[] = {
3723 3724
	F(DstMem | SrcImm | NoWrite, em_test),
	F(DstMem | SrcImm | NoWrite, em_test),
3725 3726
	F(DstMem | SrcNone | Lock, em_not),
	F(DstMem | SrcNone | Lock, em_neg),
3727 3728 3729 3730
	I(SrcMem, em_mul_ex),
	I(SrcMem, em_imul_ex),
	I(SrcMem, em_div_ex),
	I(SrcMem, em_idiv_ex),
3731 3732
};

3733
static const struct opcode group4[] = {
3734 3735
	F(ByteOp | DstMem | SrcNone | Lock, em_inc),
	F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3736 3737 3738
	N, N, N, N, N, N,
};

3739
static const struct opcode group5[] = {
3740 3741
	F(DstMem | SrcNone | Lock,		em_inc),
	F(DstMem | SrcNone | Lock,		em_dec),
3742 3743 3744 3745 3746
	I(SrcMem | Stack,			em_grp45),
	I(SrcMemFAddr | ImplicitOps | Stack,	em_call_far),
	I(SrcMem | Stack,			em_grp45),
	I(SrcMemFAddr | ImplicitOps,		em_grp45),
	I(SrcMem | Stack,			em_grp45), N,
3747 3748
};

3749
static const struct opcode group6[] = {
3750 3751
	DI(Prot,	sldt),
	DI(Prot,	str),
A
Avi Kivity 已提交
3752
	II(Prot | Priv | SrcMem16, em_lldt, lldt),
A
Avi Kivity 已提交
3753
	II(Prot | Priv | SrcMem16, em_ltr, ltr),
3754 3755 3756
	N, N, N, N,
};

3757
static const struct group_dual group7 = { {
3758 3759
	II(Mov | DstMem | Priv,			em_sgdt, sgdt),
	II(Mov | DstMem | Priv,			em_sidt, sidt),
3760 3761 3762 3763 3764
	II(SrcMem | Priv,			em_lgdt, lgdt),
	II(SrcMem | Priv,			em_lidt, lidt),
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	II(SrcMem | ByteOp | Priv | NoAccess,	em_invlpg, invlpg),
3765
}, {
3766
	I(SrcNone | Priv | VendorSpecific,	em_vmcall),
3767
	EXT(0, group7_rm1),
3768
	N, EXT(0, group7_rm3),
3769 3770 3771
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	EXT(0, group7_rm7),
3772 3773
} };

3774
static const struct opcode group8[] = {
3775
	N, N, N, N,
3776 3777 3778 3779
	I(DstMem | SrcImmByte,				em_bt),
	I(DstMem | SrcImmByte | Lock | PageTable,	em_bts),
	I(DstMem | SrcImmByte | Lock,			em_btr),
	I(DstMem | SrcImmByte | Lock | PageTable,	em_btc),
3780 3781
};

3782
static const struct group_dual group9 = { {
3783
	N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3784 3785 3786 3787
}, {
	N, N, N, N, N, N, N, N,
} };

3788
static const struct opcode group11[] = {
3789
	I(DstMem | SrcImm | Mov | PageTable, em_mov),
3790
	X7(D(Undefined)),
3791 3792
};

3793
static const struct gprefix pfx_0f_6f_0f_7f = {
3794
	I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3795 3796
};

3797
static const struct gprefix pfx_vmovntpx = {
3798 3799 3800
	I(0, em_mov), N, N, N,
};

3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863
static const struct escape escape_d9 = { {
	N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_db = { {
	N, N, N, N, N, N, N, N,
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_dd = { {
	N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

3864
static const struct opcode opcode_table[256] = {
3865
	/* 0x00 - 0x07 */
3866
	F6ALU(Lock, em_add),
3867 3868
	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3869
	/* 0x08 - 0x0F */
3870
	F6ALU(Lock | PageTable, em_or),
3871 3872
	I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
	N,
3873
	/* 0x10 - 0x17 */
3874
	F6ALU(Lock, em_adc),
3875 3876
	I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3877
	/* 0x18 - 0x1F */
3878
	F6ALU(Lock, em_sbb),
3879 3880
	I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3881
	/* 0x20 - 0x27 */
3882
	F6ALU(Lock | PageTable, em_and), N, N,
3883
	/* 0x28 - 0x2F */
3884
	F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3885
	/* 0x30 - 0x37 */
3886
	F6ALU(Lock, em_xor), N, N,
3887
	/* 0x38 - 0x3F */
3888
	F6ALU(NoWrite, em_cmp), N, N,
3889
	/* 0x40 - 0x4F */
3890
	X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3891
	/* 0x50 - 0x57 */
3892
	X8(I(SrcReg | Stack, em_push)),
3893
	/* 0x58 - 0x5F */
3894
	X8(I(DstReg | Stack, em_pop)),
3895
	/* 0x60 - 0x67 */
3896 3897
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
3898 3899 3900
	N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
	N, N, N, N,
	/* 0x68 - 0x6F */
3901 3902
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3903 3904
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3905
	I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3906
	I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3907 3908 3909
	/* 0x70 - 0x7F */
	X16(D(SrcImmByte)),
	/* 0x80 - 0x87 */
3910 3911 3912 3913
	G(ByteOp | DstMem | SrcImm, group1),
	G(DstMem | SrcImm, group1),
	G(ByteOp | DstMem | SrcImm | No64, group1),
	G(DstMem | SrcImmByte, group1),
3914
	F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3915
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3916
	/* 0x88 - 0x8F */
3917
	I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3918
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3919
	I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3920 3921 3922
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
3923
	/* 0x90 - 0x97 */
3924
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3925
	/* 0x98 - 0x9F */
3926
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3927
	I(SrcImmFAddr | No64, em_call_far), N,
3928
	II(ImplicitOps | Stack, em_pushf, pushf),
A
Avi Kivity 已提交
3929
	II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3930
	/* 0xA0 - 0xA7 */
3931
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3932
	I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3933
	I2bv(SrcSI | DstDI | Mov | String, em_mov),
3934
	F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3935
	/* 0xA8 - 0xAF */
3936
	F2bv(DstAcc | SrcImm | NoWrite, em_test),
3937 3938
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3939
	F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3940
	/* 0xB0 - 0xB7 */
3941
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3942
	/* 0xB8 - 0xBF */
3943
	X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3944
	/* 0xC0 - 0xC7 */
3945
	G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3946
	I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3947
	I(ImplicitOps | Stack, em_ret),
3948 3949
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3950
	G(ByteOp, group11), G(0, group11),
3951
	/* 0xC8 - 0xCF */
A
Avi Kivity 已提交
3952 3953
	I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
	N, I(ImplicitOps | Stack, em_ret_far),
3954
	D(ImplicitOps), DI(SrcImmByte, intn),
3955
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3956
	/* 0xD0 - 0xD7 */
3957 3958
	G(Src2One | ByteOp, group2), G(Src2One, group2),
	G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3959
	N, I(DstAcc | SrcImmByte | No64, em_aad), N, N,
3960
	/* 0xD8 - 0xDF */
3961
	N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3962
	/* 0xE0 - 0xE7 */
3963 3964
	X3(I(SrcImmByte, em_loop)),
	I(SrcImmByte, em_jcxz),
3965 3966
	I2bvIP(SrcImmUByte | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3967
	/* 0xE8 - 0xEF */
3968
	I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3969
	I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3970 3971
	I2bvIP(SrcDX | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3972
	/* 0xF0 - 0xF7 */
3973
	N, DI(ImplicitOps, icebp), N, N,
3974 3975
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
3976
	/* 0xF8 - 0xFF */
3977 3978
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3979 3980 3981
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

3982
static const struct opcode twobyte_table[256] = {
3983
	/* 0x00 - 0x0F */
3984
	G(0, group6), GD(0, &group7), N, N,
3985 3986
	N, I(ImplicitOps | VendorSpecific, em_syscall),
	II(ImplicitOps | Priv, em_clts, clts), N,
3987
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3988 3989 3990 3991
	N, D(ImplicitOps | ModRM), N, N,
	/* 0x10 - 0x1F */
	N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
	/* 0x20 - 0x2F */
3992
	DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3993
	DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3994 3995
	IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
	IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3996
	N, N, N, N,
3997 3998
	N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
	N, N, N, N,
3999
	/* 0x30 - 0x3F */
4000
	II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4001
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4002
	II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4003
	IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4004 4005
	I(ImplicitOps | VendorSpecific, em_sysenter),
	I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
4006
	N, N,
4007 4008 4009 4010 4011 4012
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
	X16(D(DstReg | SrcMem | ModRM | Mov)),
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
4013 4014 4015 4016
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4017
	/* 0x70 - 0x7F */
4018 4019 4020 4021
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4022 4023 4024
	/* 0x80 - 0x8F */
	X16(D(SrcImm)),
	/* 0x90 - 0x9F */
4025
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4026
	/* 0xA0 - 0xA7 */
4027
	I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
A
Avi Kivity 已提交
4028
	II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
4029 4030
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4031
	/* 0xA8 - 0xAF */
4032
	I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4033
	DI(ImplicitOps, rsm),
4034
	I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4035 4036
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4037
	D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
4038
	/* 0xB0 - 0xB7 */
4039
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4040
	I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4041
	I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4042 4043
	I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4044
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4045 4046
	/* 0xB8 - 0xBF */
	N, N,
4047 4048
	G(BitOp, group8),
	I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4049
	I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
4050
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
A
Avi Kivity 已提交
4051
	/* 0xC0 - 0xC7 */
4052
	D2bv(DstMem | SrcReg | ModRM | Lock),
4053
	N, D(DstMem | SrcReg | ModRM | Mov),
4054
	N, N, N, GD(0, &group9),
A
Avi Kivity 已提交
4055 4056
	/* 0xC8 - 0xCF */
	X8(I(DstReg, em_bswap)),
4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

#undef D
#undef N
#undef G
#undef GD
#undef I
4070
#undef GP
4071
#undef EXT
4072

4073
#undef D2bv
4074
#undef D2bvIP
4075
#undef I2bv
4076
#undef I2bvIP
4077
#undef I6ALU
4078

4079
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4080 4081 4082
{
	unsigned size;

4083
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
4096
	op->addr.mem.ea = ctxt->_eip;
4097 4098 4099
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
4100
		op->val = insn_fetch(s8, ctxt);
4101 4102
		break;
	case 2:
4103
		op->val = insn_fetch(s16, ctxt);
4104 4105
		break;
	case 4:
4106
		op->val = insn_fetch(s32, ctxt);
4107
		break;
4108 4109 4110
	case 8:
		op->val = insn_fetch(s64, ctxt);
		break;
4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

4129 4130 4131 4132 4133 4134 4135
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
			  unsigned d)
{
	int rc = X86EMUL_CONTINUE;

	switch (d) {
	case OpReg:
4136
		decode_register_operand(ctxt, op);
4137 4138
		break;
	case OpImmUByte:
4139
		rc = decode_imm(ctxt, op, 1, false);
4140 4141
		break;
	case OpMem:
4142
		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4143 4144 4145 4146
	mem_common:
		*op = ctxt->memop;
		ctxt->memopp = op;
		if ((ctxt->d & BitOp) && op == &ctxt->dst)
4147 4148 4149
			fetch_bit_operand(ctxt);
		op->orig_val = op->val;
		break;
4150 4151 4152
	case OpMem64:
		ctxt->memop.bytes = 8;
		goto mem_common;
4153 4154 4155
	case OpAcc:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4156
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4157 4158 4159 4160 4161 4162 4163
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
	case OpDI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4164
			register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4165 4166
		op->addr.mem.seg = VCPU_SREG_ES;
		op->val = 0;
4167
		op->count = 1;
4168 4169 4170 4171
		break;
	case OpDX:
		op->type = OP_REG;
		op->bytes = 2;
4172
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4173 4174
		fetch_register_operand(op);
		break;
4175 4176
	case OpCL:
		op->bytes = 1;
4177
		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188
		break;
	case OpImmByte:
		rc = decode_imm(ctxt, op, 1, true);
		break;
	case OpOne:
		op->bytes = 1;
		op->val = 1;
		break;
	case OpImm:
		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
		break;
4189 4190 4191
	case OpImm64:
		rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
		break;
4192 4193 4194
	case OpMem8:
		ctxt->memop.bytes = 1;
		goto mem_common;
4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210
	case OpMem16:
		ctxt->memop.bytes = 2;
		goto mem_common;
	case OpMem32:
		ctxt->memop.bytes = 4;
		goto mem_common;
	case OpImmU16:
		rc = decode_imm(ctxt, op, 2, false);
		break;
	case OpImmU:
		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
		break;
	case OpSI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4211
			register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4212 4213
		op->addr.mem.seg = seg_override(ctxt);
		op->val = 0;
4214
		op->count = 1;
4215 4216 4217 4218 4219 4220 4221 4222 4223 4224
		break;
	case OpImmFAddr:
		op->type = OP_IMM;
		op->addr.mem.ea = ctxt->_eip;
		op->bytes = ctxt->op_bytes + 2;
		insn_fetch_arr(op->valptr, op->bytes, ctxt);
		break;
	case OpMemFAddr:
		ctxt->memop.bytes = ctxt->op_bytes + 2;
		goto mem_common;
4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242
	case OpES:
		op->val = VCPU_SREG_ES;
		break;
	case OpCS:
		op->val = VCPU_SREG_CS;
		break;
	case OpSS:
		op->val = VCPU_SREG_SS;
		break;
	case OpDS:
		op->val = VCPU_SREG_DS;
		break;
	case OpFS:
		op->val = VCPU_SREG_FS;
		break;
	case OpGS:
		op->val = VCPU_SREG_GS;
		break;
4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253
	case OpImplicit:
		/* Special instructions do their own operand decoding. */
	default:
		op->type = OP_NONE; /* Disable writeback. */
		break;
	}

done:
	return rc;
}

4254
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4255 4256 4257
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
4258
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4259
	bool op_prefix = false;
4260
	struct opcode opcode;
4261

4262 4263
	ctxt->memop.type = OP_NONE;
	ctxt->memopp = NULL;
4264 4265 4266
	ctxt->_eip = ctxt->eip;
	ctxt->fetch.start = ctxt->_eip;
	ctxt->fetch.end = ctxt->fetch.start + insn_len;
4267
	if (insn_len > 0)
4268
		memcpy(ctxt->fetch.data, insn, insn_len);
4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
4286
		return EMULATION_FAILED;
4287 4288
	}

4289 4290
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
4291 4292 4293

	/* Legacy prefixes. */
	for (;;) {
4294
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
4295
		case 0x66:	/* operand-size override */
4296
			op_prefix = true;
4297
			/* switch between 2/4 bytes */
4298
			ctxt->op_bytes = def_op_bytes ^ 6;
4299 4300 4301 4302
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
4303
				ctxt->ad_bytes = def_ad_bytes ^ 12;
4304 4305
			else
				/* switch between 2/4 bytes */
4306
				ctxt->ad_bytes = def_ad_bytes ^ 6;
4307 4308 4309 4310 4311
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
4312
			set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4313 4314 4315
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
4316
			set_seg_override(ctxt, ctxt->b & 7);
4317 4318 4319 4320
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
4321
			ctxt->rex_prefix = ctxt->b;
4322 4323
			continue;
		case 0xf0:	/* LOCK */
4324
			ctxt->lock_prefix = 1;
4325 4326 4327
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
4328
			ctxt->rep_prefix = ctxt->b;
4329 4330 4331 4332 4333 4334 4335
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

4336
		ctxt->rex_prefix = 0;
4337 4338 4339 4340 4341
	}

done_prefixes:

	/* REX prefix. */
4342 4343
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
4344 4345

	/* Opcode byte(s). */
4346
	opcode = opcode_table[ctxt->b];
4347
	/* Two-byte opcode? */
4348 4349
	if (ctxt->b == 0x0f) {
		ctxt->twobyte = 1;
4350
		ctxt->b = insn_fetch(u8, ctxt);
4351
		opcode = twobyte_table[ctxt->b];
4352
	}
4353
	ctxt->d = opcode.flags;
4354

4355 4356 4357
	if (ctxt->d & ModRM)
		ctxt->modrm = insn_fetch(u8, ctxt);

4358 4359
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
4360
		case Group:
4361
			goffset = (ctxt->modrm >> 3) & 7;
4362 4363 4364
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
4365 4366
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
4367 4368 4369 4370 4371
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
4372
			goffset = ctxt->modrm & 7;
4373
			opcode = opcode.u.group[goffset];
4374 4375
			break;
		case Prefix:
4376
			if (ctxt->rep_prefix && op_prefix)
4377
				return EMULATION_FAILED;
4378
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4379 4380 4381 4382 4383 4384 4385
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
4386 4387 4388 4389 4390 4391
		case Escape:
			if (ctxt->modrm > 0xbf)
				opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
			else
				opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
			break;
4392
		default:
4393
			return EMULATION_FAILED;
4394
		}
4395

4396
		ctxt->d &= ~(u64)GroupMask;
4397
		ctxt->d |= opcode.flags;
4398 4399
	}

4400 4401 4402
	ctxt->execute = opcode.u.execute;
	ctxt->check_perm = opcode.check_perm;
	ctxt->intercept = opcode.intercept;
4403 4404

	/* Unrecognised? */
4405
	if (ctxt->d == 0 || (ctxt->d & Undefined))
4406
		return EMULATION_FAILED;
4407

4408
	if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4409
		return EMULATION_FAILED;
4410

4411 4412
	if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
		ctxt->op_bytes = 8;
4413

4414
	if (ctxt->d & Op3264) {
4415
		if (mode == X86EMUL_MODE_PROT64)
4416
			ctxt->op_bytes = 8;
4417
		else
4418
			ctxt->op_bytes = 4;
4419 4420
	}

4421 4422
	if (ctxt->d & Sse)
		ctxt->op_bytes = 16;
A
Avi Kivity 已提交
4423 4424
	else if (ctxt->d & Mmx)
		ctxt->op_bytes = 8;
A
Avi Kivity 已提交
4425

4426
	/* ModRM and SIB bytes. */
4427
	if (ctxt->d & ModRM) {
4428
		rc = decode_modrm(ctxt, &ctxt->memop);
4429 4430 4431
		if (!ctxt->has_seg_override)
			set_seg_override(ctxt, ctxt->modrm_seg);
	} else if (ctxt->d & MemAbs)
4432
		rc = decode_abs(ctxt, &ctxt->memop);
4433 4434 4435
	if (rc != X86EMUL_CONTINUE)
		goto done;

4436 4437
	if (!ctxt->has_seg_override)
		set_seg_override(ctxt, VCPU_SREG_DS);
4438

4439
	ctxt->memop.addr.mem.seg = seg_override(ctxt);
4440

4441 4442
	if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4443 4444 4445 4446 4447

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
4448
	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4449 4450 4451
	if (rc != X86EMUL_CONTINUE)
		goto done;

4452 4453 4454 4455
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
4456
	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4457 4458 4459
	if (rc != X86EMUL_CONTINUE)
		goto done;

4460
	/* Decode and fetch the destination operand: register or memory. */
4461
	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4462 4463

done:
4464 4465
	if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
		ctxt->memopp->addr.mem.ea += ctxt->_eip;
4466

4467
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4468 4469
}

4470 4471 4472 4473 4474
bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
{
	return ctxt->d & PageTable;
}

4475 4476 4477 4478 4479 4480 4481 4482 4483
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
4484 4485 4486
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
4487
		 ((ctxt->eflags & EFLG_ZF) == 0))
4488
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
4489 4490 4491 4492 4493 4494
		    ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
		return true;

	return false;
}

A
Avi Kivity 已提交
4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507
static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
{
	bool fault = false;

	ctxt->ops->get_fpu(ctxt);
	asm volatile("1: fwait \n\t"
		     "2: \n\t"
		     ".pushsection .fixup,\"ax\" \n\t"
		     "3: \n\t"
		     "movb $1, %[fault] \n\t"
		     "jmp 2b \n\t"
		     ".popsection \n\t"
		     _ASM_EXTABLE(1b, 3b)
4508
		     : [fault]"+qm"(fault));
A
Avi Kivity 已提交
4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523
	ctxt->ops->put_fpu(ctxt);

	if (unlikely(fault))
		return emulate_exception(ctxt, MF_VECTOR, 0, false);

	return X86EMUL_CONTINUE;
}

static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
				       struct operand *op)
{
	if (op->type == OP_MM)
		read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
}

4524 4525 4526 4527 4528 4529 4530 4531 4532 4533
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
{
	ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
	fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
	asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
	    : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
	: "c"(ctxt->src2.val), [fastop]"S"(fop));
	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
	return X86EMUL_CONTINUE;
}
4534

4535
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4536
{
4537
	const struct x86_emulate_ops *ops = ctxt->ops;
4538
	int rc = X86EMUL_CONTINUE;
4539
	int saved_dst_type = ctxt->dst.type;
4540

4541
	ctxt->mem_read.pos = 0;
4542

4543
	if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4544
		rc = emulate_ud(ctxt);
4545 4546 4547
		goto done;
	}

4548
	/* LOCK prefix is allowed only with some instructions */
4549
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4550
		rc = emulate_ud(ctxt);
4551 4552 4553
		goto done;
	}

4554
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4555
		rc = emulate_ud(ctxt);
4556 4557 4558
		goto done;
	}

A
Avi Kivity 已提交
4559 4560
	if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
	    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
A
Avi Kivity 已提交
4561 4562 4563 4564
		rc = emulate_ud(ctxt);
		goto done;
	}

A
Avi Kivity 已提交
4565
	if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
A
Avi Kivity 已提交
4566 4567 4568 4569
		rc = emulate_nm(ctxt);
		goto done;
	}

A
Avi Kivity 已提交
4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583
	if (ctxt->d & Mmx) {
		rc = flush_pending_x87_faults(ctxt);
		if (rc != X86EMUL_CONTINUE)
			goto done;
		/*
		 * Now that we know the fpu is exception safe, we can fetch
		 * operands from it.
		 */
		fetch_possible_mmx_operand(ctxt, &ctxt->src);
		fetch_possible_mmx_operand(ctxt, &ctxt->src2);
		if (!(ctxt->d & Mov))
			fetch_possible_mmx_operand(ctxt, &ctxt->dst);
	}

4584 4585
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
4586
					      X86_ICPT_PRE_EXCEPT);
4587 4588 4589 4590
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4591
	/* Privileged instruction can be executed only in CPL=0 */
4592
	if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4593
		rc = emulate_gp(ctxt, 0);
4594 4595 4596
		goto done;
	}

4597
	/* Instruction can only be executed in protected mode */
4598
	if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4599 4600 4601 4602
		rc = emulate_ud(ctxt);
		goto done;
	}

4603
	/* Do instruction specific permission checks */
4604 4605
	if (ctxt->check_perm) {
		rc = ctxt->check_perm(ctxt);
4606 4607 4608 4609
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4610 4611
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
4612
					      X86_ICPT_POST_EXCEPT);
4613 4614 4615 4616
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4617
	if (ctxt->rep_prefix && (ctxt->d & String)) {
4618
		/* All REP prefixes have the same first termination condition */
4619
		if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4620
			ctxt->eip = ctxt->_eip;
4621 4622 4623 4624
			goto done;
		}
	}

4625 4626 4627
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
4628
		if (rc != X86EMUL_CONTINUE)
4629
			goto done;
4630
		ctxt->src.orig_val64 = ctxt->src.val64;
4631 4632
	}

4633 4634 4635
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
4636 4637 4638 4639
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4640
	if ((ctxt->d & DstMask) == ImplicitOps)
4641 4642 4643
		goto special_insn;


4644
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4645
		/* optimisation - avoid slow emulated read if Mov */
4646 4647
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
4648 4649
		if (rc != X86EMUL_CONTINUE)
			goto done;
4650
	}
4651
	ctxt->dst.orig_val = ctxt->dst.val;
4652

4653 4654
special_insn:

4655 4656
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
4657
					      X86_ICPT_POST_MEMACCESS);
4658 4659 4660 4661
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4662
	if (ctxt->execute) {
4663 4664 4665 4666 4667 4668 4669
		if (ctxt->d & Fastop) {
			void (*fop)(struct fastop *) = (void *)ctxt->execute;
			rc = fastop(ctxt, fop);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			goto writeback;
		}
4670
		rc = ctxt->execute(ctxt);
4671 4672 4673 4674 4675
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

4676
	if (ctxt->twobyte)
A
Avi Kivity 已提交
4677 4678
		goto twobyte_insn;

4679
	switch (ctxt->b) {
A
Avi Kivity 已提交
4680
	case 0x63:		/* movsxd */
4681
		if (ctxt->mode != X86EMUL_MODE_PROT64)
A
Avi Kivity 已提交
4682
			goto cannot_emulate;
4683
		ctxt->dst.val = (s32) ctxt->src.val;
A
Avi Kivity 已提交
4684
		break;
4685
	case 0x70 ... 0x7f: /* jcc (short) */
4686 4687
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
4688
		break;
N
Nitin A Kamble 已提交
4689
	case 0x8d: /* lea r16/r32, m */
4690
		ctxt->dst.val = ctxt->src.addr.mem.ea;
N
Nitin A Kamble 已提交
4691
		break;
4692
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
4693
		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4694
			break;
4695 4696
		rc = em_xchg(ctxt);
		break;
4697
	case 0x98: /* cbw/cwde/cdqe */
4698 4699 4700 4701
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4702 4703
		}
		break;
4704
	case 0xcc:		/* int3 */
4705 4706
		rc = emulate_int(ctxt, 3);
		break;
4707
	case 0xcd:		/* int n */
4708
		rc = emulate_int(ctxt, ctxt->src.val);
4709 4710
		break;
	case 0xce:		/* into */
4711 4712
		if (ctxt->eflags & EFLG_OF)
			rc = emulate_int(ctxt, 4);
4713
		break;
4714
	case 0xe9: /* jmp rel */
4715
	case 0xeb: /* jmp rel short */
4716 4717
		jmp_rel(ctxt, ctxt->src.val);
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
4718
		break;
4719
	case 0xf4:              /* hlt */
4720
		ctxt->ops->halt(ctxt);
4721
		break;
4722 4723 4724 4725 4726 4727 4728
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
		ctxt->eflags ^= EFLG_CF;
		break;
	case 0xf8: /* clc */
		ctxt->eflags &= ~EFLG_CF;
		break;
4729 4730 4731
	case 0xf9: /* stc */
		ctxt->eflags |= EFLG_CF;
		break;
4732 4733 4734 4735 4736 4737
	case 0xfc: /* cld */
		ctxt->eflags &= ~EFLG_DF;
		break;
	case 0xfd: /* std */
		ctxt->eflags |= EFLG_DF;
		break;
4738 4739
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4740
	}
4741

4742 4743 4744
	if (rc != X86EMUL_CONTINUE)
		goto done;

4745
writeback:
4746
	rc = writeback(ctxt);
4747
	if (rc != X86EMUL_CONTINUE)
4748 4749
		goto done;

4750 4751 4752 4753
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
4754
	ctxt->dst.type = saved_dst_type;
4755

4756
	if ((ctxt->d & SrcMask) == SrcSI)
4757
		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4758

4759
	if ((ctxt->d & DstMask) == DstDI)
4760
		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4761

4762
	if (ctxt->rep_prefix && (ctxt->d & String)) {
4763
		unsigned int count;
4764
		struct read_cache *r = &ctxt->io_read;
4765 4766 4767 4768 4769 4770
		if ((ctxt->d & SrcMask) == SrcSI)
			count = ctxt->src.count;
		else
			count = ctxt->dst.count;
		register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
				-count);
4771

4772 4773 4774 4775 4776
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
4777
			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4778 4779 4780 4781 4782 4783
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
4784
				ctxt->mem_read.end = 0;
4785
				writeback_registers(ctxt);
4786 4787 4788
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
4789
		}
4790
	}
4791

4792
	ctxt->eip = ctxt->_eip;
4793 4794

done:
4795 4796
	if (rc == X86EMUL_PROPAGATE_FAULT)
		ctxt->have_exception = true;
4797 4798 4799
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

4800 4801 4802
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);

4803
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
4804 4805

twobyte_insn:
4806
	switch (ctxt->b) {
4807
	case 0x09:		/* wbinvd */
4808
		(ctxt->ops->wbinvd)(ctxt);
4809 4810
		break;
	case 0x08:		/* invd */
4811 4812 4813 4814
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
		break;
	case 0x20: /* mov cr, reg */
4815
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4816
		break;
A
Avi Kivity 已提交
4817
	case 0x21: /* mov from dr to reg */
4818
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
A
Avi Kivity 已提交
4819 4820
		break;
	case 0x40 ... 0x4f:	/* cmov */
4821 4822 4823
		ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
		if (!test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
4824
		break;
4825
	case 0x80 ... 0x8f: /* jnz rel, etc*/
4826 4827
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
4828
		break;
4829
	case 0x90 ... 0x9f:     /* setcc r/m8 */
4830
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4831
		break;
4832 4833
	case 0xae:              /* clflush */
		break;
A
Avi Kivity 已提交
4834
	case 0xb6 ... 0xb7:	/* movzx */
4835
		ctxt->dst.bytes = ctxt->op_bytes;
4836
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4837
						       : (u16) ctxt->src.val;
A
Avi Kivity 已提交
4838 4839
		break;
	case 0xbe ... 0xbf:	/* movsx */
4840
		ctxt->dst.bytes = ctxt->op_bytes;
4841
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4842
							(s16) ctxt->src.val;
A
Avi Kivity 已提交
4843
		break;
4844
	case 0xc0 ... 0xc1:	/* xadd */
4845
		emulate_2op_SrcV(ctxt, "add");
4846
		/* Write back the register source. */
4847 4848
		ctxt->src.val = ctxt->dst.orig_val;
		write_register_operand(&ctxt->src);
4849
		break;
4850
	case 0xc3:		/* movnti */
4851 4852 4853
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
							(u64) ctxt->src.val;
4854
		break;
4855 4856
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4857
	}
4858 4859 4860 4861

	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
4862 4863 4864
	goto writeback;

cannot_emulate:
4865
	return EMULATION_FAILED;
A
Avi Kivity 已提交
4866
}
4867 4868 4869 4870 4871 4872 4873 4874 4875 4876

void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
{
	invalidate_registers(ctxt);
}

void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
{
	writeback_registers(ctxt);
}