emulate.c 132.7 KB
Newer Older
A
Avi Kivity 已提交
1
/******************************************************************************
2
 * emulate.c
A
Avi Kivity 已提交
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
A
Avi Kivity 已提交
10 11
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
A
Avi Kivity 已提交
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
27
#include <linux/stringify.h>
A
Avi Kivity 已提交
28

29
#include "x86.h"
30
#include "tss.h"
31

32 33 34
/*
 * Operand types
 */
35 36 37 38 39 40 41 42 43
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
44 45 46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
47
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
48 49 50 51 52 53 54
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
55 56 57 58 59 60
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
61
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
62
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
P
Paolo Bonzini 已提交
63
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
64 65
#define OpAccLo           29ull  /* Low part of extended acc (AX/AX/EAX/RAX) */
#define OpAccHi           30ull  /* High part of extended acc (-/DX/EDX/RDX) */
66 67

#define OpBits             5  /* Width of operand field */
68
#define OpMask             ((1ull << OpBits) - 1)
69

A
Avi Kivity 已提交
70 71 72 73 74 75 76 77 78 79
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
80
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
81
/* Destination operand type. */
82 83 84 85 86 87 88
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
89
#define DstMem16    (OpMem16 << DstShift)
90 91
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
92
#define DstAccLo    (OpAccLo << DstShift)
93
#define DstMask     (OpMask << DstShift)
A
Avi Kivity 已提交
94
/* Source operand type. */
95 96 97 98 99 100 101 102 103 104 105 106
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
P
Paolo Bonzini 已提交
107
#define SrcXLat     (OpXLat << SrcShift)
108 109 110 111
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
112
#define SrcImm64    (OpImm64 << SrcShift)
113
#define SrcDX       (OpDX << SrcShift)
114
#define SrcMem8     (OpMem8 << SrcShift)
115
#define SrcAccHi    (OpAccHi << SrcShift)
116
#define SrcMask     (OpMask << SrcShift)
117 118 119 120 121 122 123 124 125
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
126
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
127
#define InstrDual   (6<<15)     /* Alternate instruction decoding of mod == 3 */
128
#define ModeDual    (7<<15)     /* Different instruction for 32/64 bit */
129
#define Sse         (1<<18)     /* SSE Vector instruction */
130 131 132 133
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
134
/* Misc flags */
135
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
136
#define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139
#define Undefined   (1<<25) /* No Such Instruction */
140
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
141
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
142
#define No64	    (1<<28)
143
#define PageTable   (1 << 29)   /* instruction used to write page table */
144
#define NotImpl     (1 << 30)   /* instruction is not implemented */
145
/* Source 2 operand type */
146
#define Src2Shift   (31)
147
#define Src2None    (OpNone << Src2Shift)
148
#define Src2Mem     (OpMem << Src2Shift)
149 150 151 152
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
153 154 155 156 157 158
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
159
#define Src2Mask    (OpMask << Src2Shift)
A
Avi Kivity 已提交
160
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
161 162 163
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
164
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
165
#define NoWrite     ((u64)1 << 45)  /* No writeback */
166
#define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
167
#define NoMod	    ((u64)1 << 47)  /* Mod field is ignored */
168 169
#define Intercept   ((u64)1 << 48)  /* Has valid intercept field */
#define CheckPerm   ((u64)1 << 49)  /* Has valid check_perm field */
170
#define PrivUD      ((u64)1 << 51)  /* #UD instead of #GP on CPL > 0 */
171
#define NearBranch  ((u64)1 << 52)  /* Near branches */
172
#define No16	    ((u64)1 << 53)  /* No 16 bit operand */
173
#define IncSP       ((u64)1 << 54)  /* SP is incremented before ModRM calc */
A
Avi Kivity 已提交
174

175
#define DstXacc     (DstAccLo | SrcAccHi | SrcWrite)
A
Avi Kivity 已提交
176

177 178 179 180 181 182 183 184
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
185

186 187 188 189 190 191
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

/*
 * fastop functions have a special calling convention:
 *
192 193
 * dst:    rax        (in/out)
 * src:    rdx        (in/out)
194 195
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
196
 * ex:     rsi        (in:fastop pointer, out:zero if exception)
197 198 199 200 201 202 203 204 205 206 207
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 *
 * fastop functions are declared as taking a never-defined fastop parameter,
 * so they can't be called from C directly.
 */

struct fastop;

208
struct opcode {
209 210
	u64 flags : 56;
	u64 intercept : 8;
211
	union {
212
		int (*execute)(struct x86_emulate_ctxt *ctxt);
213 214 215
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
216
		const struct escape *esc;
217
		const struct instr_dual *idual;
218
		const struct mode_dual *mdual;
219
		void (*fastop)(struct fastop *fake);
220
	} u;
221
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
222 223 224 225 226
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
227 228
};

229 230 231 232 233 234 235
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

236 237 238 239 240
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

241 242 243 244 245
struct instr_dual {
	struct opcode mod012;
	struct opcode mod3;
};

246 247 248 249 250
struct mode_dual {
	struct opcode mode32;
	struct opcode mode64;
};

251 252
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a

253 254 255 256 257 258 259
enum x86_transfer_type {
	X86_TRANSFER_NONE,
	X86_TRANSFER_CALL_JMP,
	X86_TRANSFER_RET,
	X86_TRANSFER_TASK_SWITCH,
};

260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

A
Avi Kivity 已提交
296 297 298 299
/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
300 301
#define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
		     X86_EFLAGS_PF|X86_EFLAGS_CF)
A
Avi Kivity 已提交
302

303 304 305 306 307 308
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

309 310
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));

311 312 313 314 315 316 317 318 319 320 321 322 323
#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
#define FOP_RET   "ret \n\t"

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
            FOP_ALIGN \
	    "em_" #op ": \n\t"

#define FOP_END \
	    ".popsection")

324 325
#define FOPNOP() FOP_ALIGN FOP_RET

326
#define FOP1E(op,  dst) \
327 328 329 330
	FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET

#define FOP1EEX(op,  dst) \
	FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
331 332 333 334 335 336 337 338 339

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

340 341 342 343 344 345 346 347 348
/* 1-operand, using src2 (for MUL/DIV r/m) */
#define FASTOP1SRC2(op, name) \
	FOP_START(name) \
	FOP1E(op, cl) \
	FOP1E(op, cx) \
	FOP1E(op, ecx) \
	ON64(FOP1E(op, rcx)) \
	FOP_END

349 350 351 352 353 354 355 356 357
/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
#define FASTOP1SRC2EX(op, name) \
	FOP_START(name) \
	FOP1EEX(op, cl) \
	FOP1EEX(op, cx) \
	FOP1EEX(op, ecx) \
	ON64(FOP1EEX(op, rcx)) \
	FOP_END

358 359 360 361 362
#define FOP2E(op,  dst, src)	   \
	FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET

#define FASTOP2(op) \
	FOP_START(op) \
363 364 365 366
	FOP2E(op##b, al, dl) \
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
367 368
	FOP_END

369 370 371 372
/* 2 operand, word only */
#define FASTOP2W(op) \
	FOP_START(op) \
	FOPNOP() \
373 374 375
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
376 377
	FOP_END

378 379 380 381 382 383 384 385 386
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

387 388 389 390 391 392 393 394 395
/* 2 operand, src and dest are reversed */
#define FASTOP2R(op, name) \
	FOP_START(name) \
	FOP2E(op##b, dl, al) \
	FOP2E(op##w, dx, ax) \
	FOP2E(op##l, edx, eax) \
	ON64(FOP2E(op##q, rdx, rax)) \
	FOP_END

396 397 398 399 400 401 402
#define FOP3E(op,  dst, src, src2) \
	FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
403 404 405
	FOP3E(op##w, ax, dx, cl) \
	FOP3E(op##l, eax, edx, cl) \
	ON64(FOP3E(op##q, rax, rdx, cl)) \
406 407
	FOP_END

408 409 410
/* Special case for SETcc - 1 instruction per cc */
#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"

411 412 413
asm(".global kvm_fastop_exception \n"
    "kvm_fastop_exception: xor %esi, %esi; ret");

414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

P
Paolo Bonzini 已提交
433 434 435
FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
FOP_END;

436 437 438 439 440 441
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
442 443 444 445 446
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
447
		.dst_val    = ctxt->dst.val64,
448 449 450
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
451 452 453
		.next_rip   = ctxt->eip,
	};

454
	return ctxt->ops->intercept(ctxt, &info, stage);
455 456
}

A
Avi Kivity 已提交
457 458 459 460 461
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
static void assign_register(unsigned long *reg, u64 val, int bytes)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (bytes) {
	case 1:
		*(u8 *)reg = (u8)val;
		break;
	case 2:
		*(u16 *)reg = (u16)val;
		break;
	case 4:
		*reg = (u32)val;
		break;	/* 64b: zero-extend */
	case 8:
		*reg = val;
		break;
	}
}

481
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
482
{
483
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
484 485
}

A
Avi Kivity 已提交
486 487 488 489 490 491 492 493 494 495 496
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

A
Avi Kivity 已提交
497 498 499 500 501
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

A
Avi Kivity 已提交
502
/* Access/update address held in a register, based on addressing mode. */
503
static inline unsigned long
504
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
505
{
506
	if (ctxt->ad_bytes == sizeof(unsigned long))
507 508
		return reg;
	else
509
		return reg & ad_mask(ctxt);
510 511 512
}

static inline unsigned long
513
register_address(struct x86_emulate_ctxt *ctxt, int reg)
514
{
515
	return address_mask(ctxt, reg_read(ctxt, reg));
516 517
}

518 519 520 521 522
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

523
static inline void
524
register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
525
{
526 527
	ulong mask;

528
	if (ctxt->ad_bytes == sizeof(unsigned long))
529
		mask = ~0UL;
530
	else
531
		mask = ad_mask(ctxt);
532
	masked_increment(reg_rmw(ctxt, reg), mask, inc);
533 534 535 536
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
537
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
538
}
A
Avi Kivity 已提交
539

540 541 542 543 544 545 546
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

547
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
548 549 550 551
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

552
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
553 554
}

555 556
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
557
{
558
	WARN_ON(vec > 0x1f);
559 560 561
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
562
	return X86EMUL_PROPAGATE_FAULT;
563 564
}

565 566 567 568 569
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

570
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
571
{
572
	return emulate_exception(ctxt, GP_VECTOR, err, true);
573 574
}

575 576 577 578 579
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

580
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
581
{
582
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
583 584
}

585
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
586
{
587
	return emulate_exception(ctxt, TS_VECTOR, err, true);
588 589
}

590 591
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
592
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
593 594
}

A
Avi Kivity 已提交
595 596 597 598 599
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.
 */
static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
{
	if (likely(size < 16))
		return false;

	if (ctxt->d & Aligned)
		return true;
	else if (ctxt->d & Unaligned)
		return false;
	else if (ctxt->d & Avx)
		return false;
	else
		return true;
}

643 644 645 646
static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
				       struct segmented_address addr,
				       unsigned *max_size, unsigned size,
				       bool write, bool fetch,
647
				       enum x86emul_mode mode, ulong *linear)
648
{
649 650
	struct desc_struct desc;
	bool usable;
651
	ulong la;
652
	u32 lim;
653
	u16 sel;
654

655
	la = seg_base(ctxt, addr.seg) + addr.ea;
656
	*max_size = 0;
657
	switch (mode) {
658
	case X86EMUL_MODE_PROT64:
659
		if (is_noncanonical_address(la))
660
			goto bad;
661 662 663 664

		*max_size = min_t(u64, ~0u, (1ull << 48) - la);
		if (size > *max_size)
			goto bad;
665 666
		break;
	default:
667 668
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
669 670
		if (!usable)
			goto bad;
671 672 673
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
674 675
			goto bad;
		/* unreadable code segment */
676
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
677 678
			goto bad;
		lim = desc_limit_scaled(&desc);
679
		if (!(desc.type & 8) && (desc.type & 4)) {
G
Guo Chao 已提交
680
			/* expand-down segment */
681
			if (addr.ea <= lim)
682 683 684
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
		}
685 686
		if (addr.ea > lim)
			goto bad;
687 688 689 690 691 692 693
		if (lim == 0xffffffff)
			*max_size = ~0u;
		else {
			*max_size = (u64)lim + 1 - addr.ea;
			if (size > *max_size)
				goto bad;
		}
694
		la &= (u32)-1;
695 696
		break;
	}
697 698
	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
		return emulate_gp(ctxt, 0);
699 700
	*linear = la;
	return X86EMUL_CONTINUE;
701 702
bad:
	if (addr.seg == VCPU_SREG_SS)
703
		return emulate_ss(ctxt, 0);
704
	else
705
		return emulate_gp(ctxt, 0);
706 707
}

708 709 710 711 712
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
713
	unsigned max_size;
714 715
	return __linearize(ctxt, addr, &max_size, size, write, false,
			   ctxt->mode, linear);
716 717
}

718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737
static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
			     enum x86emul_mode mode)
{
	ulong linear;
	int rc;
	unsigned max_size;
	struct segmented_address addr = { .seg = VCPU_SREG_CS,
					   .ea = dst };

	if (ctxt->op_bytes != sizeof(unsigned long))
		addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
	rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
	if (rc == X86EMUL_CONTINUE)
		ctxt->_eip = addr.ea;
	return rc;
}

static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
{
	return assign_eip(ctxt, dst, ctxt->mode);
738 739
}

740 741 742 743
static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
			  const struct desc_struct *cs_desc)
{
	enum x86emul_mode mode = ctxt->mode;
744
	int rc;
745 746

#ifdef CONFIG_X86_64
747 748 749
	if (ctxt->mode >= X86EMUL_MODE_PROT16) {
		if (cs_desc->l) {
			u64 efer = 0;
750

751 752 753 754 755
			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
			if (efer & EFER_LMA)
				mode = X86EMUL_MODE_PROT64;
		} else
			mode = X86EMUL_MODE_PROT32; /* temporary value */
756 757 758 759
	}
#endif
	if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
		mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
760 761 762 763
	rc = assign_eip(ctxt, dst, mode);
	if (rc == X86EMUL_CONTINUE)
		ctxt->mode = mode;
	return rc;
764 765 766 767 768 769
}

static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
{
	return assign_eip_near(ctxt, ctxt->_eip + rel);
}
770

771 772 773 774 775
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
776 777 778
	int rc;
	ulong linear;

779
	rc = linearize(ctxt, addr, size, false, &linear);
780 781
	if (rc != X86EMUL_CONTINUE)
		return rc;
782
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
783 784
}

785
/*
786
 * Prefetch the remaining bytes of the instruction without crossing page
787 788
 * boundary if they are not in fetch_cache yet.
 */
789
static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
790 791
{
	int rc;
792
	unsigned size, max_size;
793
	unsigned long linear;
794
	int cur_size = ctxt->fetch.end - ctxt->fetch.data;
795
	struct segmented_address addr = { .seg = VCPU_SREG_CS,
796 797
					   .ea = ctxt->eip + cur_size };

798 799 800 801 802 803 804 805 806 807
	/*
	 * We do not know exactly how many bytes will be needed, and
	 * __linearize is expensive, so fetch as much as possible.  We
	 * just have to avoid going beyond the 15 byte limit, the end
	 * of the segment, or the end of the page.
	 *
	 * __linearize is called with size 0 so that it does not do any
	 * boundary check itself.  Instead, we use max_size to check
	 * against op_size.
	 */
808 809
	rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
			 &linear);
810 811 812
	if (unlikely(rc != X86EMUL_CONTINUE))
		return rc;

813
	size = min_t(unsigned, 15UL ^ cur_size, max_size);
814
	size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
815 816 817 818 819 820 821 822

	/*
	 * One instruction can only straddle two pages,
	 * and one has been loaded at the beginning of
	 * x86_decode_insn.  So, if not enough bytes
	 * still, we must have hit the 15-byte boundary.
	 */
	if (unlikely(size < op_size))
823 824
		return emulate_gp(ctxt, 0);

825
	rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
826 827 828
			      size, &ctxt->exception);
	if (unlikely(rc != X86EMUL_CONTINUE))
		return rc;
829
	ctxt->fetch.end += size;
830
	return X86EMUL_CONTINUE;
831 832
}

833 834
static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
					       unsigned size)
835
{
836 837 838 839
	unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;

	if (unlikely(done_size < size))
		return __do_insn_fetch_bytes(ctxt, size - done_size);
840 841
	else
		return X86EMUL_CONTINUE;
842 843
}

844
/* Fetch next part of the instruction being emulated. */
845
#define insn_fetch(_type, _ctxt)					\
846 847 848
({	_type _x;							\
									\
	rc = do_insn_fetch_bytes(_ctxt, sizeof(_type));			\
849 850
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
851
	ctxt->_eip += sizeof(_type);					\
852 853
	_x = *(_type __aligned(1) *) ctxt->fetch.ptr;			\
	ctxt->fetch.ptr += sizeof(_type);				\
854
	_x;								\
855 856
})

857
#define insn_fetch_arr(_arr, _size, _ctxt)				\
858 859
({									\
	rc = do_insn_fetch_bytes(_ctxt, _size);				\
860 861
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
862
	ctxt->_eip += (_size);						\
863 864
	memcpy(_arr, ctxt->fetch.ptr, _size);				\
	ctxt->fetch.ptr += (_size);					\
865 866
})

867 868 869 870 871
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
872
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
873
			     int byteop)
A
Avi Kivity 已提交
874 875
{
	void *p;
876
	int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
A
Avi Kivity 已提交
877 878

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
879 880 881
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
A
Avi Kivity 已提交
882 883 884 885
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
886
			   struct segmented_address addr,
A
Avi Kivity 已提交
887 888 889 890 891 892 893
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
894
	rc = segmented_read_std(ctxt, addr, size, 2);
895
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
896
		return rc;
897
	addr.ea += 2;
898
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
899 900 901
	return rc;
}

902 903 904 905 906 907 908 909 910 911
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);

912 913
FASTOP1SRC2(mul, mul_ex);
FASTOP1SRC2(imul, imul_ex);
914 915
FASTOP1SRC2EX(div, div_ex);
FASTOP1SRC2EX(idiv, idiv_ex);
916

917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
FASTOP3WCL(shld);
FASTOP3WCL(shrd);

FASTOP2W(imul);

FASTOP1(not);
FASTOP1(neg);
FASTOP1(inc);
FASTOP1(dec);

FASTOP2CL(rol);
FASTOP2CL(ror);
FASTOP2CL(rcl);
FASTOP2CL(rcr);
FASTOP2CL(shl);
FASTOP2CL(shr);
FASTOP2CL(sar);

FASTOP2W(bsf);
FASTOP2W(bsr);
FASTOP2W(bt);
FASTOP2W(bts);
FASTOP2W(btr);
FASTOP2W(btc);

942 943
FASTOP2(xadd);

944 945
FASTOP2R(cmp, cmp_r);

946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961
static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
{
	/* If src is zero, do not writeback, but update flags */
	if (ctxt->src.val == 0)
		ctxt->dst.type = OP_NONE;
	return fastop(ctxt, em_bsf);
}

static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
{
	/* If src is zero, do not writeback, but update flags */
	if (ctxt->src.val == 0)
		ctxt->dst.type = OP_NONE;
	return fastop(ctxt, em_bsr);
}

962
static u8 test_cc(unsigned int condition, unsigned long flags)
963
{
964 965
	u8 rc;
	void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
966

967
	flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
968
	asm("push %[flags]; popf; call *%[fastop]"
969 970
	    : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
	return rc;
971 972
}

973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

A
Avi Kivity 已提交
991 992 993 994
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
995 996 997 998 999 1000 1001 1002
	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
1003
#ifdef CONFIG_X86_64
1004 1005 1006 1007 1008 1009 1010 1011
	case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
1023 1024 1025 1026 1027 1028 1029 1030
	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
A
Avi Kivity 已提交
1031
#ifdef CONFIG_X86_64
1032 1033 1034 1035 1036 1037 1038 1039
	case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
A
Avi Kivity 已提交
1040 1041 1042 1043 1044 1045
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

A
Avi Kivity 已提交
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
	case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
	case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
	case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
	case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
	case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
	case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
	case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
	case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
	case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
	case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
static int em_fninit(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fninit");
	ctxt->ops->put_fpu(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
{
	u16 fcw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstcw %0": "+m"(fcw));
	ctxt->ops->put_fpu(ctxt);

	ctxt->dst.val = fcw;

	return X86EMUL_CONTINUE;
}

static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
{
	u16 fsw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstsw %0": "+m"(fsw));
	ctxt->ops->put_fpu(ctxt);

	ctxt->dst.val = fsw;

	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1123
static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1124
				    struct operand *op)
1125
{
1126
	unsigned reg = ctxt->modrm_reg;
1127

1128 1129
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
1130

1131
	if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1132 1133 1134 1135 1136 1137
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}
A
Avi Kivity 已提交
1138 1139 1140 1141 1142 1143 1144
	if (ctxt->d & Mmx) {
		reg &= 7;
		op->type = OP_MM;
		op->bytes = 8;
		op->addr.mm = reg;
		return;
	}
A
Avi Kivity 已提交
1145

1146
	op->type = OP_REG;
1147 1148 1149
	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);

1150
	fetch_register_operand(op);
1151 1152 1153
	op->orig_val = op->val;
}

1154 1155 1156 1157 1158 1159
static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
{
	if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
		ctxt->modrm_seg = VCPU_SREG_SS;
}

1160
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1161
			struct operand *op)
1162 1163
{
	u8 sib;
B
Bandan Das 已提交
1164
	int index_reg, base_reg, scale;
1165
	int rc = X86EMUL_CONTINUE;
1166
	ulong modrm_ea = 0;
1167

B
Bandan Das 已提交
1168 1169 1170
	ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
	index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
	base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1171

B
Bandan Das 已提交
1172
	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1173
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
B
Bandan Das 已提交
1174
	ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1175
	ctxt->modrm_seg = VCPU_SREG_DS;
1176

1177
	if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1178
		op->type = OP_REG;
1179
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1180
		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1181
				ctxt->d & ByteOp);
1182
		if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1183 1184
			op->type = OP_XMM;
			op->bytes = 16;
1185 1186
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
A
Avi Kivity 已提交
1187 1188
			return rc;
		}
A
Avi Kivity 已提交
1189 1190 1191
		if (ctxt->d & Mmx) {
			op->type = OP_MM;
			op->bytes = 8;
1192
			op->addr.mm = ctxt->modrm_rm & 7;
A
Avi Kivity 已提交
1193 1194
			return rc;
		}
1195
		fetch_register_operand(op);
1196 1197 1198
		return rc;
	}

1199 1200
	op->type = OP_MEM;

1201
	if (ctxt->ad_bytes == 2) {
1202 1203 1204 1205
		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1206 1207

		/* 16-bit ModR/M decode. */
1208
		switch (ctxt->modrm_mod) {
1209
		case 0:
1210
			if (ctxt->modrm_rm == 6)
1211
				modrm_ea += insn_fetch(u16, ctxt);
1212 1213
			break;
		case 1:
1214
			modrm_ea += insn_fetch(s8, ctxt);
1215 1216
			break;
		case 2:
1217
			modrm_ea += insn_fetch(u16, ctxt);
1218 1219
			break;
		}
1220
		switch (ctxt->modrm_rm) {
1221
		case 0:
1222
			modrm_ea += bx + si;
1223 1224
			break;
		case 1:
1225
			modrm_ea += bx + di;
1226 1227
			break;
		case 2:
1228
			modrm_ea += bp + si;
1229 1230
			break;
		case 3:
1231
			modrm_ea += bp + di;
1232 1233
			break;
		case 4:
1234
			modrm_ea += si;
1235 1236
			break;
		case 5:
1237
			modrm_ea += di;
1238 1239
			break;
		case 6:
1240
			if (ctxt->modrm_mod != 0)
1241
				modrm_ea += bp;
1242 1243
			break;
		case 7:
1244
			modrm_ea += bx;
1245 1246
			break;
		}
1247 1248 1249
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
1250
		modrm_ea = (u16)modrm_ea;
1251 1252
	} else {
		/* 32/64-bit ModR/M decode. */
1253
		if ((ctxt->modrm_rm & 7) == 4) {
1254
			sib = insn_fetch(u8, ctxt);
1255 1256 1257 1258
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

1259
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1260
				modrm_ea += insn_fetch(s32, ctxt);
1261
			else {
1262
				modrm_ea += reg_read(ctxt, base_reg);
1263
				adjust_modrm_seg(ctxt, base_reg);
1264 1265 1266 1267
				/* Increment ESP on POP [ESP] */
				if ((ctxt->d & IncSP) &&
				    base_reg == VCPU_REGS_RSP)
					modrm_ea += ctxt->op_bytes;
1268
			}
1269
			if (index_reg != 4)
1270
				modrm_ea += reg_read(ctxt, index_reg) << scale;
1271
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1272
			modrm_ea += insn_fetch(s32, ctxt);
1273
			if (ctxt->mode == X86EMUL_MODE_PROT64)
1274
				ctxt->rip_relative = 1;
1275 1276
		} else {
			base_reg = ctxt->modrm_rm;
1277
			modrm_ea += reg_read(ctxt, base_reg);
1278 1279
			adjust_modrm_seg(ctxt, base_reg);
		}
1280
		switch (ctxt->modrm_mod) {
1281
		case 1:
1282
			modrm_ea += insn_fetch(s8, ctxt);
1283 1284
			break;
		case 2:
1285
			modrm_ea += insn_fetch(s32, ctxt);
1286 1287 1288
			break;
		}
	}
1289
	op->addr.mem.ea = modrm_ea;
1290 1291 1292
	if (ctxt->ad_bytes != 8)
		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;

1293 1294 1295 1296 1297
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
1298
		      struct operand *op)
1299
{
1300
	int rc = X86EMUL_CONTINUE;
1301

1302
	op->type = OP_MEM;
1303
	switch (ctxt->ad_bytes) {
1304
	case 2:
1305
		op->addr.mem.ea = insn_fetch(u16, ctxt);
1306 1307
		break;
	case 4:
1308
		op->addr.mem.ea = insn_fetch(u32, ctxt);
1309 1310
		break;
	case 8:
1311
		op->addr.mem.ea = insn_fetch(u64, ctxt);
1312 1313 1314 1315 1316 1317
		break;
	}
done:
	return rc;
}

1318
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1319
{
1320
	long sv = 0, mask;
1321

1322
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1323
		mask = ~((long)ctxt->dst.bytes * 8 - 1);
1324

1325 1326 1327 1328
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1329 1330
		else
			sv = (s64)ctxt->src.val & (s64)mask;
1331

1332 1333
		ctxt->dst.addr.mem.ea = address_mask(ctxt,
					   ctxt->dst.addr.mem.ea + (sv >> 3));
1334
	}
1335 1336

	/* only subword offset */
1337
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1338 1339
}

1340 1341
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1342
{
1343
	int rc;
1344
	struct read_cache *mc = &ctxt->mem_read;
A
Avi Kivity 已提交
1345

1346 1347
	if (mc->pos < mc->end)
		goto read_cached;
A
Avi Kivity 已提交
1348

1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
	WARN_ON((mc->end + size) >= sizeof(mc->data));

	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
				      &ctxt->exception);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	mc->end += size;

read_cached:
	memcpy(dest, mc->data + mc->pos, size);
	mc->pos += size;
1361 1362
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1363

1364 1365 1366 1367 1368
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1369 1370 1371
	int rc;
	ulong linear;

1372
	rc = linearize(ctxt, addr, size, false, &linear);
1373 1374
	if (rc != X86EMUL_CONTINUE)
		return rc;
1375
	return read_emulated(ctxt, linear, data, size);
1376 1377 1378 1379 1380 1381 1382
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1383 1384 1385
	int rc;
	ulong linear;

1386
	rc = linearize(ctxt, addr, size, true, &linear);
1387 1388
	if (rc != X86EMUL_CONTINUE)
		return rc;
1389 1390
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1391 1392 1393 1394 1395 1396 1397
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1398 1399 1400
	int rc;
	ulong linear;

1401
	rc = linearize(ctxt, addr, size, true, &linear);
1402 1403
	if (rc != X86EMUL_CONTINUE)
		return rc;
1404 1405
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1406 1407
}

1408 1409 1410 1411
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1412
	struct read_cache *rc = &ctxt->io_read;
1413

1414 1415
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1416
		unsigned int count = ctxt->rep_prefix ?
1417
			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1418
		in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1419 1420
			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1421
		n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1422 1423 1424
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1425
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1426 1427
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1428 1429
	}

1430
	if (ctxt->rep_prefix && (ctxt->d & String) &&
1431
	    !(ctxt->eflags & X86_EFLAGS_DF)) {
1432 1433 1434 1435 1436 1437 1438 1439
		ctxt->dst.data = rc->data + rc->pos;
		ctxt->dst.type = OP_MEM_STR;
		ctxt->dst.count = (rc->end - rc->pos) / size;
		rc->pos = rc->end;
	} else {
		memcpy(dest, rc->data + rc->pos, size);
		rc->pos += size;
	}
1440 1441
	return 1;
}
A
Avi Kivity 已提交
1442

1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458
static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
				     u16 index, struct desc_struct *desc)
{
	struct desc_ptr dt;
	ulong addr;

	ctxt->ops->get_idt(ctxt, &dt);

	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, index << 3 | 0x2);

	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
}

1459 1460 1461
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1462
	const struct x86_emulate_ops *ops = ctxt->ops;
1463
	u32 base3 = 0;
1464

1465 1466
	if (selector & 1 << 2) {
		struct desc_struct desc;
1467 1468
		u16 sel;

1469
		memset (dt, 0, sizeof *dt);
1470 1471
		if (!ops->get_segment(ctxt, &sel, &desc, &base3,
				      VCPU_SREG_LDTR))
1472
			return;
1473

1474
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1475
		dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1476
	} else
1477
		ops->get_gdt(ctxt, dt);
1478
}
1479

1480 1481
static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
			      u16 selector, ulong *desc_addr_p)
1482 1483 1484 1485
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1486

1487
	get_descriptor_table_ptr(ctxt, selector, &dt);
1488

1489 1490
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1491

1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519
	addr = dt.address + index * 8;

#ifdef CONFIG_X86_64
	if (addr >> 32 != 0) {
		u64 efer = 0;

		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (!(efer & EFER_LMA))
			addr &= (u32)-1;
	}
#endif

	*desc_addr_p = addr;
	return X86EMUL_CONTINUE;
}

/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, struct desc_struct *desc,
				   ulong *desc_addr_p)
{
	int rc;

	rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1520
				   &ctxt->exception);
1521
}
1522

1523 1524 1525 1526
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
1527
	int rc;
1528
	ulong addr;
A
Avi Kivity 已提交
1529

1530 1531 1532
	rc = get_descriptor_ptr(ctxt, selector, &addr);
	if (rc != X86EMUL_CONTINUE)
		return rc;
A
Avi Kivity 已提交
1533

1534 1535
	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
				    &ctxt->exception);
1536
}
1537

1538
/* Does not support long mode */
1539
static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1540
				     u16 selector, int seg, u8 cpl,
1541
				     enum x86_transfer_type transfer,
1542
				     struct desc_struct *desc)
1543
{
1544
	struct desc_struct seg_desc, old_desc;
1545
	u8 dpl, rpl;
1546 1547 1548
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1549
	ulong desc_addr;
1550
	int ret;
1551
	u16 dummy;
1552
	u32 base3 = 0;
1553

1554
	memset(&seg_desc, 0, sizeof seg_desc);
1555

1556 1557 1558
	if (ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor (keep limit etc. for
		 * unreal mode) */
1559
		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1560 1561
		set_desc_base(&seg_desc, selector << 4);
		goto load;
1562 1563 1564 1565 1566 1567 1568 1569 1570
	} else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
		/* VM86 needs a clean new segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
		seg_desc.dpl = 3;
		goto load;
1571 1572
	}

1573 1574 1575 1576 1577 1578 1579
	rpl = selector & 3;

	/* NULL selector is not valid for TR, CS and SS (except for long mode) */
	if ((seg == VCPU_SREG_CS
	     || (seg == VCPU_SREG_SS
		 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
	     || seg == VCPU_SREG_TR)
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589
	    && null_selector)
		goto exception;

	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

	if (null_selector) /* for NULL selector skip all following checks */
		goto load;

1590
	ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1591 1592 1593 1594
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
1595 1596
	err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
							   GP_VECTOR;
1597

G
Guo Chao 已提交
1598
	/* can't load system descriptor into segment selector */
1599 1600 1601
	if (seg <= VCPU_SREG_GS && !seg_desc.s) {
		if (transfer == X86_TRANSFER_CALL_JMP)
			return X86EMUL_UNHANDLEABLE;
1602
		goto exception;
1603
	}
1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	dpl = seg_desc.dpl;

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1620
		break;
1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
1634 1635 1636 1637 1638 1639 1640 1641 1642
		/* in long-mode d/b must be clear if l is set */
		if (seg_desc.d && seg_desc.l) {
			u64 efer = 0;

			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
			if (efer & EFER_LMA)
				goto exception;
		}

1643 1644
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1645
		break;
1646 1647 1648
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
1649 1650 1651 1652 1653 1654
		old_desc = seg_desc;
		seg_desc.type |= 2; /* busy */
		ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
						  sizeof(seg_desc), &ctxt->exception);
		if (ret != X86EMUL_CONTINUE)
			return ret;
1655 1656 1657 1658 1659 1660
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1661
		/*
1662 1663 1664
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1665
		 */
1666 1667 1668 1669
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1670
		break;
1671 1672 1673 1674
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
1675 1676 1677 1678 1679 1680 1681
		if (!(seg_desc.type & 1)) {
			seg_desc.type |= 1;
			ret = write_segment_descriptor(ctxt, selector,
						       &seg_desc);
			if (ret != X86EMUL_CONTINUE)
				return ret;
		}
1682 1683 1684 1685 1686
	} else if (ctxt->mode == X86EMUL_MODE_PROT64) {
		ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
				sizeof(base3), &ctxt->exception);
		if (ret != X86EMUL_CONTINUE)
			return ret;
1687 1688 1689
		if (is_noncanonical_address(get_desc_base(&seg_desc) |
					     ((u64)base3 << 32)))
			return emulate_gp(ctxt, 0);
1690 1691
	}
load:
1692
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1693 1694
	if (desc)
		*desc = seg_desc;
1695 1696
	return X86EMUL_CONTINUE;
exception:
1697
	return emulate_exception(ctxt, err_vec, err_code, true);
1698 1699
}

1700 1701 1702 1703
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
	u8 cpl = ctxt->ops->cpl(ctxt);
1704 1705
	return __load_segment_descriptor(ctxt, selector, seg, cpl,
					 X86_TRANSFER_NONE, NULL);
1706 1707
}

1708 1709
static void write_register_operand(struct operand *op)
{
1710
	return assign_register(op->addr.reg, op->val, op->bytes);
1711 1712
}

1713
static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1714
{
1715
	switch (op->type) {
1716
	case OP_REG:
1717
		write_register_operand(op);
A
Avi Kivity 已提交
1718
		break;
1719
	case OP_MEM:
1720
		if (ctxt->lock_prefix)
P
Paolo Bonzini 已提交
1721 1722 1723 1724 1725 1726 1727
			return segmented_cmpxchg(ctxt,
						 op->addr.mem,
						 &op->orig_val,
						 &op->val,
						 op->bytes);
		else
			return segmented_write(ctxt,
1728 1729 1730
					       op->addr.mem,
					       &op->val,
					       op->bytes);
1731
		break;
1732
	case OP_MEM_STR:
P
Paolo Bonzini 已提交
1733 1734 1735 1736
		return segmented_write(ctxt,
				       op->addr.mem,
				       op->data,
				       op->bytes * op->count);
1737
		break;
A
Avi Kivity 已提交
1738
	case OP_XMM:
1739
		write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
A
Avi Kivity 已提交
1740
		break;
A
Avi Kivity 已提交
1741
	case OP_MM:
1742
		write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
A
Avi Kivity 已提交
1743
		break;
1744 1745
	case OP_NONE:
		/* no writeback */
1746
		break;
1747
	default:
1748
		break;
A
Avi Kivity 已提交
1749
	}
1750 1751
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1752

1753
static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1754
{
1755
	struct segmented_address addr;
1756

1757
	rsp_increment(ctxt, -bytes);
1758
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1759 1760
	addr.seg = VCPU_SREG_SS;

1761 1762 1763 1764 1765
	return segmented_write(ctxt, addr, data, bytes);
}

static int em_push(struct x86_emulate_ctxt *ctxt)
{
1766
	/* Disable writeback. */
1767
	ctxt->dst.type = OP_NONE;
1768
	return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1769
}
1770

1771 1772 1773 1774
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1775
	struct segmented_address addr;
1776

1777
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1778
	addr.seg = VCPU_SREG_SS;
1779
	rc = segmented_read(ctxt, addr, dest, len);
1780 1781 1782
	if (rc != X86EMUL_CONTINUE)
		return rc;

1783
	rsp_increment(ctxt, len);
1784
	return rc;
1785 1786
}

1787 1788
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1789
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1790 1791
}

1792
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1793
			void *dest, int len)
1794 1795
{
	int rc;
1796
	unsigned long val, change_mask;
1797
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1798
	int cpl = ctxt->ops->cpl(ctxt);
1799

1800
	rc = emulate_pop(ctxt, &val, len);
1801 1802
	if (rc != X86EMUL_CONTINUE)
		return rc;
1803

1804 1805 1806 1807
	change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
		      X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
		      X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
		      X86_EFLAGS_AC | X86_EFLAGS_ID;
1808

1809 1810 1811 1812 1813
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
1814
			change_mask |= X86_EFLAGS_IOPL;
1815
		if (cpl <= iopl)
1816
			change_mask |= X86_EFLAGS_IF;
1817 1818
		break;
	case X86EMUL_MODE_VM86:
1819 1820
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1821
		change_mask |= X86_EFLAGS_IF;
1822 1823
		break;
	default: /* real mode */
1824
		change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1825
		break;
1826
	}
1827 1828 1829 1830 1831

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1832 1833
}

1834 1835
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1836 1837 1838 1839
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1840 1841
}

A
Avi Kivity 已提交
1842 1843 1844 1845 1846
static int em_enter(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned frame_size = ctxt->src.val;
	unsigned nesting_level = ctxt->src2.val & 31;
1847
	ulong rbp;
A
Avi Kivity 已提交
1848 1849 1850 1851

	if (nesting_level)
		return X86EMUL_UNHANDLEABLE;

1852 1853
	rbp = reg_read(ctxt, VCPU_REGS_RBP);
	rc = push(ctxt, &rbp, stack_size(ctxt));
A
Avi Kivity 已提交
1854 1855
	if (rc != X86EMUL_CONTINUE)
		return rc;
1856
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
A
Avi Kivity 已提交
1857
		      stack_mask(ctxt));
1858 1859
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
A
Avi Kivity 已提交
1860 1861 1862 1863
		      stack_mask(ctxt));
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1864 1865
static int em_leave(struct x86_emulate_ctxt *ctxt)
{
1866
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
A
Avi Kivity 已提交
1867
		      stack_mask(ctxt));
1868
	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
A
Avi Kivity 已提交
1869 1870
}

1871
static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1872
{
1873 1874
	int seg = ctxt->src2.val;

1875
	ctxt->src.val = get_segment_selector(ctxt, seg);
1876 1877 1878 1879
	if (ctxt->op_bytes == 4) {
		rsp_increment(ctxt, -2);
		ctxt->op_bytes = 2;
	}
1880

1881
	return em_push(ctxt);
1882 1883
}

1884
static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1885
{
1886
	int seg = ctxt->src2.val;
1887 1888
	unsigned long selector;
	int rc;
1889

1890
	rc = emulate_pop(ctxt, &selector, 2);
1891 1892 1893
	if (rc != X86EMUL_CONTINUE)
		return rc;

1894 1895
	if (ctxt->modrm_reg == VCPU_SREG_SS)
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1896 1897
	if (ctxt->op_bytes > 2)
		rsp_increment(ctxt, ctxt->op_bytes - 2);
1898

1899
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1900
	return rc;
1901 1902
}

1903
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1904
{
1905
	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1906 1907
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1908

1909 1910
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
1911
		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1912

1913
		rc = em_push(ctxt);
1914 1915
		if (rc != X86EMUL_CONTINUE)
			return rc;
1916

1917
		++reg;
1918 1919
	}

1920
	return rc;
1921 1922
}

1923 1924
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
1925
	ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1926 1927 1928
	return em_push(ctxt);
}

1929
static int em_popa(struct x86_emulate_ctxt *ctxt)
1930
{
1931 1932
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
1933
	u32 val;
1934

1935 1936
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
1937
			rsp_increment(ctxt, ctxt->op_bytes);
1938 1939
			--reg;
		}
1940

1941
		rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1942 1943
		if (rc != X86EMUL_CONTINUE)
			break;
1944
		assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1945
		--reg;
1946
	}
1947
	return rc;
1948 1949
}

1950
static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1951
{
1952
	const struct x86_emulate_ops *ops = ctxt->ops;
1953
	int rc;
1954 1955 1956 1957 1958 1959
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
1960
	ctxt->src.val = ctxt->eflags;
1961
	rc = em_push(ctxt);
1962 1963
	if (rc != X86EMUL_CONTINUE)
		return rc;
1964

1965
	ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
1966

1967
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1968
	rc = em_push(ctxt);
1969 1970
	if (rc != X86EMUL_CONTINUE)
		return rc;
1971

1972
	ctxt->src.val = ctxt->_eip;
1973
	rc = em_push(ctxt);
1974 1975 1976
	if (rc != X86EMUL_CONTINUE)
		return rc;

1977
	ops->get_idt(ctxt, &dt);
1978 1979 1980 1981

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

1982
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1983 1984 1985
	if (rc != X86EMUL_CONTINUE)
		return rc;

1986
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1987 1988 1989
	if (rc != X86EMUL_CONTINUE)
		return rc;

1990
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1991 1992 1993
	if (rc != X86EMUL_CONTINUE)
		return rc;

1994
	ctxt->_eip = eip;
1995 1996 1997 1998

	return rc;
}

1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
{
	int rc;

	invalidate_registers(ctxt);
	rc = __emulate_int_real(ctxt, irq);
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);
	return rc;
}

2010
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2011 2012 2013
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
2014
		return __emulate_int_real(ctxt, irq);
2015 2016 2017 2018 2019 2020 2021 2022 2023 2024
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

2025
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2026
{
2027 2028 2029 2030
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
2031 2032 2033 2034 2035
	unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
			     X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
			     X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
			     X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
			     X86_EFLAGS_AC | X86_EFLAGS_ID |
W
Wanpeng Li 已提交
2036
			     X86_EFLAGS_FIXED;
2037 2038
	unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
				  X86_EFLAGS_VIP;
2039

2040
	/* TODO: Add stack limit check */
2041

2042
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2043

2044 2045
	if (rc != X86EMUL_CONTINUE)
		return rc;
2046

2047 2048
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
2049

2050
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2051

2052 2053
	if (rc != X86EMUL_CONTINUE)
		return rc;
2054

2055
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2056

2057 2058
	if (rc != X86EMUL_CONTINUE)
		return rc;
2059

2060
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2061

2062 2063
	if (rc != X86EMUL_CONTINUE)
		return rc;
2064

2065
	ctxt->_eip = temp_eip;
2066

2067
	if (ctxt->op_bytes == 4)
2068
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2069
	else if (ctxt->op_bytes == 2) {
2070 2071
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
2072
	}
2073 2074

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
W
Wanpeng Li 已提交
2075
	ctxt->eflags |= X86_EFLAGS_FIXED;
2076
	ctxt->ops->set_nmi_mask(ctxt, false);
2077 2078

	return rc;
2079 2080
}

2081
static int em_iret(struct x86_emulate_ctxt *ctxt)
2082
{
2083 2084
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
2085
		return emulate_iret_real(ctxt);
2086 2087 2088 2089
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
2090
	default:
2091 2092
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
2093 2094 2095
	}
}

2096 2097 2098
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
2099 2100 2101 2102 2103 2104 2105 2106 2107
	unsigned short sel, old_sel;
	struct desc_struct old_desc, new_desc;
	const struct x86_emulate_ops *ops = ctxt->ops;
	u8 cpl = ctxt->ops->cpl(ctxt);

	/* Assignment of RIP may only fail in 64-bit mode */
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
				 VCPU_SREG_CS);
2108

2109
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2110

2111 2112
	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
				       X86_TRANSFER_CALL_JMP,
2113
				       &new_desc);
2114 2115 2116
	if (rc != X86EMUL_CONTINUE)
		return rc;

2117
	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2118
	if (rc != X86EMUL_CONTINUE) {
2119
		WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2120 2121 2122 2123 2124
		/* assigning eip failed; restore the old cs */
		ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
		return rc;
	}
	return rc;
2125 2126
}

2127
static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2128
{
2129 2130
	return assign_eip_near(ctxt, ctxt->src.val);
}
2131

2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142
static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	long int old_eip;

	old_eip = ctxt->_eip;
	rc = assign_eip_near(ctxt, ctxt->src.val);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->src.val = old_eip;
	rc = em_push(ctxt);
2143
	return rc;
2144 2145
}

2146
static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2147
{
2148
	u64 old = ctxt->dst.orig_val64;
2149

2150 2151 2152
	if (ctxt->dst.bytes == 16)
		return X86EMUL_UNHANDLEABLE;

2153 2154 2155 2156
	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2157
		ctxt->eflags &= ~X86_EFLAGS_ZF;
2158
	} else {
2159 2160
		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
			(u32) reg_read(ctxt, VCPU_REGS_RBX);
2161

2162
		ctxt->eflags |= X86_EFLAGS_ZF;
2163
	}
2164
	return X86EMUL_CONTINUE;
2165 2166
}

2167 2168
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
2169 2170 2171 2172 2173 2174 2175 2176
	int rc;
	unsigned long eip;

	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	return assign_eip_near(ctxt, eip);
2177 2178
}

2179
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2180 2181
{
	int rc;
2182 2183
	unsigned long eip, cs;
	u16 old_cs;
2184
	int cpl = ctxt->ops->cpl(ctxt);
2185 2186 2187 2188 2189 2190
	struct desc_struct old_desc, new_desc;
	const struct x86_emulate_ops *ops = ctxt->ops;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
				 VCPU_SREG_CS);
2191

2192
	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2193
	if (rc != X86EMUL_CONTINUE)
2194
		return rc;
2195
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2196
	if (rc != X86EMUL_CONTINUE)
2197
		return rc;
2198 2199 2200
	/* Outer-privilege level return is not implemented */
	if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
		return X86EMUL_UNHANDLEABLE;
2201 2202
	rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
				       X86_TRANSFER_RET,
2203 2204 2205
				       &new_desc);
	if (rc != X86EMUL_CONTINUE)
		return rc;
2206
	rc = assign_eip_far(ctxt, eip, &new_desc);
2207
	if (rc != X86EMUL_CONTINUE) {
2208
		WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2209 2210
		ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
	}
2211 2212 2213
	return rc;
}

2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224
static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
{
        int rc;

        rc = em_ret_far(ctxt);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        rsp_increment(ctxt, ctxt->src.val);
        return X86EMUL_CONTINUE;
}

2225 2226 2227
static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
{
	/* Save real source value, then compare EAX against destination. */
2228 2229
	ctxt->dst.orig_val = ctxt->dst.val;
	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2230
	ctxt->src.orig_val = ctxt->src.val;
2231
	ctxt->src.val = ctxt->dst.orig_val;
2232
	fastop(ctxt, em_cmp);
2233

2234
	if (ctxt->eflags & X86_EFLAGS_ZF) {
2235 2236
		/* Success: write back to memory; no update of EAX */
		ctxt->src.type = OP_NONE;
2237 2238 2239
		ctxt->dst.val = ctxt->src.orig_val;
	} else {
		/* Failure: write the value we saw to EAX. */
2240 2241 2242 2243
		ctxt->src.type = OP_REG;
		ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
		ctxt->src.val = ctxt->dst.orig_val;
		/* Create write-cycle to dest by writing the same value */
2244
		ctxt->dst.val = ctxt->dst.orig_val;
2245 2246 2247 2248
	}
	return X86EMUL_CONTINUE;
}

2249
static int em_lseg(struct x86_emulate_ctxt *ctxt)
2250
{
2251
	int seg = ctxt->src2.val;
2252 2253 2254
	unsigned short sel;
	int rc;

2255
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2256

2257
	rc = load_segment_descriptor(ctxt, sel, seg);
2258 2259 2260
	if (rc != X86EMUL_CONTINUE)
		return rc;

2261
	ctxt->dst.val = ctxt->src.val;
2262 2263 2264
	return rc;
}

2265
static void
2266
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2267
			struct desc_struct *cs, struct desc_struct *ss)
2268 2269
{
	cs->l = 0;		/* will be adjusted later */
2270
	set_desc_base(cs, 0);	/* flat segment */
2271
	cs->g = 1;		/* 4kb granularity */
2272
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
2273 2274 2275
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
2276 2277
	cs->p = 1;
	cs->d = 1;
2278
	cs->avl = 0;
2279

2280 2281
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
2282 2283 2284
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
2285
	ss->d = 1;		/* 32bit stack segment */
2286
	ss->dpl = 0;
2287
	ss->p = 1;
2288 2289
	ss->l = 0;
	ss->avl = 0;
2290 2291
}

2292 2293 2294 2295 2296
static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

	eax = ecx = 0;
2297 2298
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2299 2300 2301 2302
		&& ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
		&& edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
}

2303 2304
static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
{
2305
	const struct x86_emulate_ops *ops = ctxt->ops;
2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316
	u32 eax, ebx, ecx, edx;

	/*
	 * syscall should always be enabled in longmode - so only become
	 * vendor specific (cpuid) if other modes are active...
	 */
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return true;

	eax = 0x00000000;
	ecx = 0x00000000;
2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341
	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	/*
	 * Intel ("GenuineIntel")
	 * remark: Intel CPUs only support "syscall" in 64bit
	 * longmode. Also an 64bit guest with a
	 * 32bit compat-app running will #UD !! While this
	 * behaviour can be fixed (by emulating) into AMD
	 * response - CPUs of AMD can't behave like Intel.
	 */
	if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
		return false;

	/* AMD ("AuthenticAMD") */
	if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
		return true;

	/* AMD ("AMDisbetter!") */
	if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
		return true;
2342 2343 2344 2345 2346

	/* default: (not Intel, not AMD), apply Intel's stricter rules... */
	return false;
}

2347
static int em_syscall(struct x86_emulate_ctxt *ctxt)
2348
{
2349
	const struct x86_emulate_ops *ops = ctxt->ops;
2350
	struct desc_struct cs, ss;
2351
	u64 msr_data;
2352
	u16 cs_sel, ss_sel;
2353
	u64 efer = 0;
2354 2355

	/* syscall is not available in real mode */
2356
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2357 2358
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
2359

2360 2361 2362
	if (!(em_syscall_is_enabled(ctxt)))
		return emulate_ud(ctxt);

2363
	ops->get_msr(ctxt, MSR_EFER, &efer);
2364
	setup_syscalls_segments(ctxt, &cs, &ss);
2365

2366 2367 2368
	if (!(efer & EFER_SCE))
		return emulate_ud(ctxt);

2369
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
2370
	msr_data >>= 32;
2371 2372
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
2373

2374
	if (efer & EFER_LMA) {
2375
		cs.d = 0;
2376 2377
		cs.l = 1;
	}
2378 2379
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2380

2381
	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2382
	if (efer & EFER_LMA) {
2383
#ifdef CONFIG_X86_64
2384
		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2385

2386
		ops->get_msr(ctxt,
2387 2388
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
2389
		ctxt->_eip = msr_data;
2390

2391
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2392
		ctxt->eflags &= ~msr_data;
W
Wanpeng Li 已提交
2393
		ctxt->eflags |= X86_EFLAGS_FIXED;
2394 2395 2396
#endif
	} else {
		/* legacy mode */
2397
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
2398
		ctxt->_eip = (u32)msr_data;
2399

2400
		ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2401 2402
	}

2403
	return X86EMUL_CONTINUE;
2404 2405
}

2406
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2407
{
2408
	const struct x86_emulate_ops *ops = ctxt->ops;
2409
	struct desc_struct cs, ss;
2410
	u64 msr_data;
2411
	u16 cs_sel, ss_sel;
2412
	u64 efer = 0;
2413

2414
	ops->get_msr(ctxt, MSR_EFER, &efer);
2415
	/* inject #GP if in real mode */
2416 2417
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
2418

2419 2420 2421 2422
	/*
	 * Not recognized on AMD in compat mode (but is recognized in legacy
	 * mode).
	 */
2423
	if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2424 2425 2426
	    && !vendor_intel(ctxt))
		return emulate_ud(ctxt);

2427
	/* sysenter/sysexit have not been tested in 64bit mode. */
2428
	if (ctxt->mode == X86EMUL_MODE_PROT64)
2429
		return X86EMUL_UNHANDLEABLE;
2430

2431
	setup_syscalls_segments(ctxt, &cs, &ss);
2432

2433
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2434 2435
	if ((msr_data & 0xfffc) == 0x0)
		return emulate_gp(ctxt, 0);
2436

2437
	ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2438
	cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2439
	ss_sel = cs_sel + 8;
2440
	if (efer & EFER_LMA) {
2441
		cs.d = 0;
2442 2443 2444
		cs.l = 1;
	}

2445 2446
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2447

2448
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2449
	ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2450

2451
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2452 2453
	*reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
							      (u32)msr_data;
2454

2455
	return X86EMUL_CONTINUE;
2456 2457
}

2458
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2459
{
2460
	const struct x86_emulate_ops *ops = ctxt->ops;
2461
	struct desc_struct cs, ss;
2462
	u64 msr_data, rcx, rdx;
2463
	int usermode;
X
Xiao Guangrong 已提交
2464
	u16 cs_sel = 0, ss_sel = 0;
2465

2466 2467
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2468 2469
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
2470

2471
	setup_syscalls_segments(ctxt, &cs, &ss);
2472

2473
	if ((ctxt->rex_prefix & 0x8) != 0x0)
2474 2475 2476 2477
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

2478 2479 2480
	rcx = reg_read(ctxt, VCPU_REGS_RCX);
	rdx = reg_read(ctxt, VCPU_REGS_RDX);

2481 2482
	cs.dpl = 3;
	ss.dpl = 3;
2483
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2484 2485
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
2486
		cs_sel = (u16)(msr_data + 16);
2487 2488
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2489
		ss_sel = (u16)(msr_data + 24);
2490 2491
		rcx = (u32)rcx;
		rdx = (u32)rdx;
2492 2493
		break;
	case X86EMUL_MODE_PROT64:
2494
		cs_sel = (u16)(msr_data + 32);
2495 2496
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2497 2498
		ss_sel = cs_sel + 8;
		cs.d = 0;
2499
		cs.l = 1;
2500 2501 2502
		if (is_noncanonical_address(rcx) ||
		    is_noncanonical_address(rdx))
			return emulate_gp(ctxt, 0);
2503 2504
		break;
	}
2505 2506
	cs_sel |= SEGMENT_RPL_MASK;
	ss_sel |= SEGMENT_RPL_MASK;
2507

2508 2509
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2510

2511 2512
	ctxt->_eip = rdx;
	*reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2513

2514
	return X86EMUL_CONTINUE;
2515 2516
}

2517
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2518 2519 2520 2521 2522 2523
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
2524
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2525
	return ctxt->ops->cpl(ctxt) > iopl;
2526 2527 2528 2529 2530
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2531
	const struct x86_emulate_ops *ops = ctxt->ops;
2532
	struct desc_struct tr_seg;
2533
	u32 base3;
2534
	int r;
2535
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2536
	unsigned mask = (1 << len) - 1;
2537
	unsigned long base;
2538

2539
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2540
	if (!tr_seg.p)
2541
		return false;
2542
	if (desc_limit_scaled(&tr_seg) < 103)
2543
		return false;
2544 2545 2546 2547
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2548
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2549 2550
	if (r != X86EMUL_CONTINUE)
		return false;
2551
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2552
		return false;
2553
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2554 2555 2556 2557 2558 2559 2560 2561 2562 2563
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2564 2565 2566
	if (ctxt->perm_ok)
		return true;

2567 2568
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2569
			return false;
2570 2571 2572

	ctxt->perm_ok = true;

2573 2574 2575
	return true;
}

2576 2577 2578
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
2579
	tss->ip = ctxt->_eip;
2580
	tss->flag = ctxt->eflags;
2581 2582 2583 2584 2585 2586 2587 2588
	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->si = reg_read(ctxt, VCPU_REGS_RSI);
	tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2589

2590 2591 2592 2593 2594
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2595 2596 2597 2598 2599 2600
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;
2601
	u8 cpl;
2602

2603
	ctxt->_eip = tss->ip;
2604
	ctxt->eflags = tss->flag | 2;
2605 2606 2607 2608 2609 2610 2611 2612
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2613 2614 2615 2616 2617

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2618 2619 2620 2621 2622
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2623

2624 2625
	cpl = tss->cs & 3;

2626
	/*
G
Guo Chao 已提交
2627
	 * Now load segment descriptors. If fault happens at this stage
2628 2629
	 * it is handled in a context of new task
	 */
2630
	ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2631
					X86_TRANSFER_TASK_SWITCH, NULL);
2632 2633
	if (ret != X86EMUL_CONTINUE)
		return ret;
2634
	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2635
					X86_TRANSFER_TASK_SWITCH, NULL);
2636 2637
	if (ret != X86EMUL_CONTINUE)
		return ret;
2638
	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2639
					X86_TRANSFER_TASK_SWITCH, NULL);
2640 2641
	if (ret != X86EMUL_CONTINUE)
		return ret;
2642
	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2643
					X86_TRANSFER_TASK_SWITCH, NULL);
2644 2645
	if (ret != X86EMUL_CONTINUE)
		return ret;
2646
	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2647
					X86_TRANSFER_TASK_SWITCH, NULL);
2648 2649 2650 2651 2652 2653 2654 2655 2656 2657
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2658
	const struct x86_emulate_ops *ops = ctxt->ops;
2659 2660
	struct tss_segment_16 tss_seg;
	int ret;
2661
	u32 new_tss_base = get_desc_base(new_desc);
2662

2663
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2664
			    &ctxt->exception);
2665
	if (ret != X86EMUL_CONTINUE)
2666 2667
		return ret;

2668
	save_state_to_tss16(ctxt, &tss_seg);
2669

2670
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2671
			     &ctxt->exception);
2672
	if (ret != X86EMUL_CONTINUE)
2673 2674
		return ret;

2675
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2676
			    &ctxt->exception);
2677
	if (ret != X86EMUL_CONTINUE)
2678 2679 2680 2681 2682
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2683
		ret = ops->write_std(ctxt, new_tss_base,
2684 2685
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2686
				     &ctxt->exception);
2687
		if (ret != X86EMUL_CONTINUE)
2688 2689 2690
			return ret;
	}

2691
	return load_state_from_tss16(ctxt, &tss_seg);
2692 2693 2694 2695 2696
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
2697
	/* CR3 and ldt selector are not saved intentionally */
2698
	tss->eip = ctxt->_eip;
2699
	tss->eflags = ctxt->eflags;
2700 2701 2702 2703 2704 2705 2706 2707
	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2708

2709 2710 2711 2712 2713 2714
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2715 2716 2717 2718 2719 2720
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;
2721
	u8 cpl;
2722

2723
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2724
		return emulate_gp(ctxt, 0);
2725
	ctxt->_eip = tss->eip;
2726
	ctxt->eflags = tss->eflags | 2;
2727 2728

	/* General purpose registers */
2729 2730 2731 2732 2733 2734 2735 2736
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2737 2738 2739

	/*
	 * SDM says that segment selectors are loaded before segment
2740 2741
	 * descriptors.  This is important because CPL checks will
	 * use CS.RPL.
2742
	 */
2743 2744 2745 2746 2747 2748 2749
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2750

2751 2752 2753 2754 2755
	/*
	 * If we're switching between Protected Mode and VM86, we need to make
	 * sure to update the mode before loading the segment descriptors so
	 * that the selectors are interpreted correctly.
	 */
2756
	if (ctxt->eflags & X86_EFLAGS_VM) {
2757
		ctxt->mode = X86EMUL_MODE_VM86;
2758 2759
		cpl = 3;
	} else {
2760
		ctxt->mode = X86EMUL_MODE_PROT32;
2761 2762
		cpl = tss->cs & 3;
	}
2763

2764 2765 2766 2767
	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2768
	ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2769
					cpl, X86_TRANSFER_TASK_SWITCH, NULL);
2770 2771
	if (ret != X86EMUL_CONTINUE)
		return ret;
2772
	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2773
					X86_TRANSFER_TASK_SWITCH, NULL);
2774 2775
	if (ret != X86EMUL_CONTINUE)
		return ret;
2776
	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2777
					X86_TRANSFER_TASK_SWITCH, NULL);
2778 2779
	if (ret != X86EMUL_CONTINUE)
		return ret;
2780
	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2781
					X86_TRANSFER_TASK_SWITCH, NULL);
2782 2783
	if (ret != X86EMUL_CONTINUE)
		return ret;
2784
	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2785
					X86_TRANSFER_TASK_SWITCH, NULL);
2786 2787
	if (ret != X86EMUL_CONTINUE)
		return ret;
2788
	ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2789
					X86_TRANSFER_TASK_SWITCH, NULL);
2790 2791
	if (ret != X86EMUL_CONTINUE)
		return ret;
2792
	ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2793
					X86_TRANSFER_TASK_SWITCH, NULL);
2794

2795
	return ret;
2796 2797 2798 2799 2800 2801
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2802
	const struct x86_emulate_ops *ops = ctxt->ops;
2803 2804
	struct tss_segment_32 tss_seg;
	int ret;
2805
	u32 new_tss_base = get_desc_base(new_desc);
2806 2807
	u32 eip_offset = offsetof(struct tss_segment_32, eip);
	u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2808

2809
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2810
			    &ctxt->exception);
2811
	if (ret != X86EMUL_CONTINUE)
2812 2813
		return ret;

2814
	save_state_to_tss32(ctxt, &tss_seg);
2815

2816 2817 2818
	/* Only GP registers and segment selectors are saved */
	ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
			     ldt_sel_offset - eip_offset, &ctxt->exception);
2819
	if (ret != X86EMUL_CONTINUE)
2820 2821
		return ret;

2822
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2823
			    &ctxt->exception);
2824
	if (ret != X86EMUL_CONTINUE)
2825 2826 2827 2828 2829
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2830
		ret = ops->write_std(ctxt, new_tss_base,
2831 2832
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2833
				     &ctxt->exception);
2834
		if (ret != X86EMUL_CONTINUE)
2835 2836 2837
			return ret;
	}

2838
	return load_state_from_tss32(ctxt, &tss_seg);
2839 2840 2841
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2842
				   u16 tss_selector, int idt_index, int reason,
2843
				   bool has_error_code, u32 error_code)
2844
{
2845
	const struct x86_emulate_ops *ops = ctxt->ops;
2846 2847
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
2848
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2849
	ulong old_tss_base =
2850
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2851
	u32 desc_limit;
2852
	ulong desc_addr;
2853 2854 2855

	/* FIXME: old_tss_base == ~0 ? */

2856
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2857 2858
	if (ret != X86EMUL_CONTINUE)
		return ret;
2859
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2860 2861 2862 2863 2864
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

2865 2866 2867 2868 2869
	/*
	 * Check privileges. The three cases are task switch caused by...
	 *
	 * 1. jmp/call/int to task gate: Check against DPL of the task gate
	 * 2. Exception/IRQ/iret: No check is performed
2870 2871
	 * 3. jmp/call to TSS/task-gate: No check is performed since the
	 *    hardware checks it before exiting.
2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887
	 */
	if (reason == TASK_SWITCH_GATE) {
		if (idt_index != -1) {
			/* Software interrupts */
			struct desc_struct task_gate_desc;
			int dpl;

			ret = read_interrupt_descriptor(ctxt, idt_index,
							&task_gate_desc);
			if (ret != X86EMUL_CONTINUE)
				return ret;

			dpl = task_gate_desc.dpl;
			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
				return emulate_gp(ctxt, (idt_index << 3) | 0x2);
		}
2888 2889
	}

2890 2891 2892 2893
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
2894
		return emulate_ts(ctxt, tss_selector & 0xfffc);
2895 2896 2897 2898
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2899
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2900 2901 2902 2903 2904 2905
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
G
Guo Chao 已提交
2906
	   note that old_tss_sel is not used after this point */
2907 2908 2909 2910
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
2911
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2912 2913
				     old_tss_base, &next_tss_desc);
	else
2914
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2915
				     old_tss_base, &next_tss_desc);
2916 2917
	if (ret != X86EMUL_CONTINUE)
		return ret;
2918 2919 2920 2921 2922 2923

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
2924
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2925 2926
	}

2927
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
2928
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2929

2930
	if (has_error_code) {
2931 2932 2933
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
2934
		ret = em_push(ctxt);
2935 2936
	}

2937 2938 2939 2940
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2941
			 u16 tss_selector, int idt_index, int reason,
2942
			 bool has_error_code, u32 error_code)
2943 2944 2945
{
	int rc;

2946
	invalidate_registers(ctxt);
2947 2948
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
2949

2950
	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2951
				     has_error_code, error_code);
2952

2953
	if (rc == X86EMUL_CONTINUE) {
2954
		ctxt->eip = ctxt->_eip;
2955 2956
		writeback_registers(ctxt);
	}
2957

2958
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2959 2960
}

2961 2962
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
		struct operand *op)
2963
{
2964
	int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
2965

2966 2967
	register_address_increment(ctxt, reg, df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, reg);
2968 2969
}

2970 2971 2972 2973 2974 2975
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
2976
	al = ctxt->dst.val;
2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

2994
	ctxt->dst.val = al;
2995
	/* Set PF, ZF, SF */
2996 2997 2998
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
2999
	fastop(ctxt, em_or);
3000 3001 3002 3003 3004 3005 3006 3007
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029
static int em_aam(struct x86_emulate_ctxt *ctxt)
{
	u8 al, ah;

	if (ctxt->src.val == 0)
		return emulate_de(ctxt);

	al = ctxt->dst.val & 0xff;
	ah = al / ctxt->src.val;
	al %= ctxt->src.val;

	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);

	/* Set PF, ZF, SF */
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
	fastop(ctxt, em_or);

	return X86EMUL_CONTINUE;
}

3030 3031 3032 3033 3034 3035 3036 3037 3038
static int em_aad(struct x86_emulate_ctxt *ctxt)
{
	u8 al = ctxt->dst.val & 0xff;
	u8 ah = (ctxt->dst.val >> 8) & 0xff;

	al = (al + (ah * ctxt->src.val)) & 0xff;

	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;

3039 3040 3041 3042 3043
	/* Set PF, ZF, SF */
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
	fastop(ctxt, em_or);
3044 3045 3046 3047

	return X86EMUL_CONTINUE;
}

3048 3049
static int em_call(struct x86_emulate_ctxt *ctxt)
{
3050
	int rc;
3051 3052 3053
	long rel = ctxt->src.val;

	ctxt->src.val = (unsigned long)ctxt->_eip;
3054 3055 3056
	rc = jmp_rel(ctxt, rel);
	if (rc != X86EMUL_CONTINUE)
		return rc;
3057 3058 3059
	return em_push(ctxt);
}

3060 3061 3062 3063 3064
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;
3065 3066 3067
	struct desc_struct old_desc, new_desc;
	const struct x86_emulate_ops *ops = ctxt->ops;
	int cpl = ctxt->ops->cpl(ctxt);
3068
	enum x86emul_mode prev_mode = ctxt->mode;
3069

3070
	old_eip = ctxt->_eip;
3071
	ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3072

3073
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3074 3075
	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
				       X86_TRANSFER_CALL_JMP, &new_desc);
3076
	if (rc != X86EMUL_CONTINUE)
3077
		return rc;
3078

3079
	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3080 3081
	if (rc != X86EMUL_CONTINUE)
		goto fail;
3082

3083
	ctxt->src.val = old_cs;
3084
	rc = em_push(ctxt);
3085
	if (rc != X86EMUL_CONTINUE)
3086
		goto fail;
3087

3088
	ctxt->src.val = old_eip;
3089 3090 3091
	rc = em_push(ctxt);
	/* If we failed, we tainted the memory, but the very least we should
	   restore cs */
3092 3093
	if (rc != X86EMUL_CONTINUE) {
		pr_warn_once("faulting far call emulation tainted memory\n");
3094
		goto fail;
3095
	}
3096 3097 3098
	return rc;
fail:
	ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3099
	ctxt->mode = prev_mode;
3100 3101
	return rc;

3102 3103
}

3104 3105 3106
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;
3107
	unsigned long eip;
3108

3109 3110 3111 3112
	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	rc = assign_eip_near(ctxt, eip);
3113 3114
	if (rc != X86EMUL_CONTINUE)
		return rc;
3115
	rsp_increment(ctxt, ctxt->src.val);
3116 3117 3118
	return X86EMUL_CONTINUE;
}

3119 3120 3121
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
3122 3123
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
3124 3125

	/* Write back the memory destination with implicit LOCK prefix. */
3126 3127
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
3128 3129 3130
	return X86EMUL_CONTINUE;
}

3131 3132
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
3133
	ctxt->dst.val = ctxt->src2.val;
3134
	return fastop(ctxt, em_imul);
3135 3136
}

3137 3138
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
3139 3140
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
3141
	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3142
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3143 3144 3145 3146

	return X86EMUL_CONTINUE;
}

3147 3148 3149 3150
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

3151
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3152 3153
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3154 3155 3156
	return X86EMUL_CONTINUE;
}

3157 3158 3159 3160
static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
{
	u64 pmc;

3161
	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3162
		return emulate_gp(ctxt, 0);
3163 3164
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3165 3166 3167
	return X86EMUL_CONTINUE;
}

3168 3169
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
3170
	memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3171 3172 3173
	return X86EMUL_CONTINUE;
}

B
Borislav Petkov 已提交
3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208
#define FFL(x) bit(X86_FEATURE_##x)

static int em_movbe(struct x86_emulate_ctxt *ctxt)
{
	u32 ebx, ecx, edx, eax = 1;
	u16 tmp;

	/*
	 * Check MOVBE is set in the guest-visible CPUID leaf.
	 */
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	if (!(ecx & FFL(MOVBE)))
		return emulate_ud(ctxt);

	switch (ctxt->op_bytes) {
	case 2:
		/*
		 * From MOVBE definition: "...When the operand size is 16 bits,
		 * the upper word of the destination register remains unchanged
		 * ..."
		 *
		 * Both casting ->valptr and ->val to u16 breaks strict aliasing
		 * rules so we have to do the operation almost per hand.
		 */
		tmp = (u16)ctxt->src.val;
		ctxt->dst.val &= ~0xffffUL;
		ctxt->dst.val |= (unsigned long)swab16(tmp);
		break;
	case 4:
		ctxt->dst.val = swab32((u32)ctxt->src.val);
		break;
	case 8:
		ctxt->dst.val = swab64(ctxt->src.val);
		break;
	default:
3209
		BUG();
B
Borislav Petkov 已提交
3210 3211 3212 3213
	}
	return X86EMUL_CONTINUE;
}

3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241
static int em_cr_write(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_dr_write(struct x86_emulate_ctxt *ctxt)
{
	unsigned long val;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		val = ctxt->src.val & ~0ULL;
	else
		val = ctxt->src.val & ~0U;

	/* #UD condition is already handled. */
	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3242 3243 3244 3245
static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3246 3247 3248
	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
	if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3249 3250 3251 3252 3253 3254 3255 3256 3257
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3258
	if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3259 3260
		return emulate_gp(ctxt, 0);

3261 3262
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
	*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3263 3264 3265
	return X86EMUL_CONTINUE;
}

3266 3267
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
3268
	if (ctxt->modrm_reg > VCPU_SREG_GS)
3269 3270
		return emulate_ud(ctxt);

3271
	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3272 3273
	if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
		ctxt->dst.bytes = 2;
3274 3275 3276 3277 3278
	return X86EMUL_CONTINUE;
}

static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
3279
	u16 sel = ctxt->src.val;
3280

3281
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3282 3283
		return emulate_ud(ctxt);

3284
	if (ctxt->modrm_reg == VCPU_SREG_SS)
3285 3286 3287
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
3288 3289
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3290 3291
}

A
Avi Kivity 已提交
3292 3293 3294 3295 3296 3297 3298 3299 3300
static int em_lldt(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
}

A
Avi Kivity 已提交
3301 3302 3303 3304 3305 3306 3307 3308 3309
static int em_ltr(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
}

3310 3311
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
3312 3313 3314
	int rc;
	ulong linear;

3315
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3316
	if (rc == X86EMUL_CONTINUE)
3317
		ctxt->ops->invlpg(ctxt, linear);
3318
	/* Disable writeback. */
3319
	ctxt->dst.type = OP_NONE;
3320 3321 3322
	return X86EMUL_CONTINUE;
}

3323 3324 3325 3326 3327 3328 3329 3330 3331 3332
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

3333
static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3334
{
3335
	int rc = ctxt->ops->fix_hypercall(ctxt);
3336 3337 3338 3339 3340

	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
3341
	ctxt->_eip = ctxt->eip;
3342
	/* Disable writeback. */
3343
	ctxt->dst.type = OP_NONE;
3344 3345 3346
	return X86EMUL_CONTINUE;
}

3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375
static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
				  void (*get)(struct x86_emulate_ctxt *ctxt,
					      struct desc_ptr *ptr))
{
	struct desc_ptr desc_ptr;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
	get(ctxt, &desc_ptr);
	if (ctxt->op_bytes == 2) {
		ctxt->op_bytes = 4;
		desc_ptr.address &= 0x00ffffff;
	}
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return segmented_write(ctxt, ctxt->dst.addr.mem,
			       &desc_ptr, 2 + ctxt->op_bytes);
}

static int em_sgdt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
}

static int em_sidt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
}

3376
static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3377 3378 3379 3380
{
	struct desc_ptr desc_ptr;
	int rc;

3381 3382
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
3383
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3384
			     &desc_ptr.size, &desc_ptr.address,
3385
			     ctxt->op_bytes);
3386 3387
	if (rc != X86EMUL_CONTINUE)
		return rc;
3388 3389 3390
	if (ctxt->mode == X86EMUL_MODE_PROT64 &&
	    is_noncanonical_address(desc_ptr.address))
		return emulate_gp(ctxt, 0);
3391 3392 3393 3394
	if (lgdt)
		ctxt->ops->set_gdt(ctxt, &desc_ptr);
	else
		ctxt->ops->set_idt(ctxt, &desc_ptr);
3395
	/* Disable writeback. */
3396
	ctxt->dst.type = OP_NONE;
3397 3398 3399
	return X86EMUL_CONTINUE;
}

3400 3401 3402 3403 3404
static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	return em_lgdt_lidt(ctxt, true);
}

3405 3406
static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
3407
	return em_lgdt_lidt(ctxt, false);
3408 3409 3410 3411
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
3412 3413
	if (ctxt->dst.type == OP_MEM)
		ctxt->dst.bytes = 2;
3414
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3415 3416 3417 3418 3419 3420
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3421 3422
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
3423 3424 3425
	return X86EMUL_CONTINUE;
}

3426 3427
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
3428 3429
	int rc = X86EMUL_CONTINUE;

3430
	register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3431
	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3432
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3433
		rc = jmp_rel(ctxt, ctxt->src.val);
3434

3435
	return rc;
3436 3437 3438 3439
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
3440 3441
	int rc = X86EMUL_CONTINUE;

3442
	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3443
		rc = jmp_rel(ctxt, ctxt->src.val);
3444

3445
	return rc;
3446 3447
}

3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465
static int em_in(struct x86_emulate_ctxt *ctxt)
{
	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
			     &ctxt->dst.val))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
}

static int em_out(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				    &ctxt->src.val, 1);
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3485 3486 3487 3488
static int em_cpuid(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

3489 3490
	eax = reg_read(ctxt, VCPU_REGS_RAX);
	ecx = reg_read(ctxt, VCPU_REGS_RCX);
A
Avi Kivity 已提交
3491
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3492 3493 3494 3495
	*reg_write(ctxt, VCPU_REGS_RAX) = eax;
	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;
	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = edx;
A
Avi Kivity 已提交
3496 3497 3498
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3499 3500 3501 3502
static int em_sahf(struct x86_emulate_ctxt *ctxt)
{
	u32 flags;

3503 3504
	flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
		X86_EFLAGS_SF;
P
Paolo Bonzini 已提交
3505 3506 3507 3508 3509 3510 3511
	flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;

	ctxt->eflags &= ~0xffUL;
	ctxt->eflags |= flags | X86_EFLAGS_FIXED;
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3512 3513
static int em_lahf(struct x86_emulate_ctxt *ctxt)
{
3514 3515
	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
A
Avi Kivity 已提交
3516 3517 3518
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533
static int em_bswap(struct x86_emulate_ctxt *ctxt)
{
	switch (ctxt->op_bytes) {
#ifdef CONFIG_X86_64
	case 8:
		asm("bswap %0" : "+r"(ctxt->dst.val));
		break;
#endif
	default:
		asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
		break;
	}
	return X86EMUL_CONTINUE;
}

3534 3535 3536 3537 3538 3539
static int em_clflush(struct x86_emulate_ctxt *ctxt)
{
	/* emulating clflush regardless of cpuid */
	return X86EMUL_CONTINUE;
}

3540 3541 3542 3543 3544 3545
static int em_movsxd(struct x86_emulate_ctxt *ctxt)
{
	ctxt->dst.val = (s32) ctxt->src.val;
	return X86EMUL_CONTINUE;
}

3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
3560
	if (!valid_cr(ctxt->modrm_reg))
3561 3562 3563 3564 3565 3566 3567
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
3568 3569
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
3570
	u64 efer = 0;
3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
3588
		u64 cr4;
3589 3590 3591 3592
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

3593 3594
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3595 3596 3597 3598 3599 3600 3601 3602 3603 3604

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

3605 3606
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
N
Nadav Amit 已提交
3607
			rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3608 3609 3610 3611 3612 3613 3614

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
3615
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

3627 3628 3629 3630
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

3631
	ctxt->ops->get_dr(ctxt, 7, &dr7);
3632 3633 3634 3635 3636 3637 3638

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
3639
	int dr = ctxt->modrm_reg;
3640 3641 3642 3643 3644
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

3645
	cr4 = ctxt->ops->get_cr(ctxt, 4);
3646 3647 3648
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

3649 3650 3651 3652 3653 3654 3655
	if (check_dr7_gd(ctxt)) {
		ulong dr6;

		ctxt->ops->get_dr(ctxt, 6, &dr6);
		dr6 &= ~15;
		dr6 |= DR6_BD | DR6_RTM;
		ctxt->ops->set_dr(ctxt, 6, dr6);
3656
		return emulate_db(ctxt);
3657
	}
3658 3659 3660 3661 3662 3663

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
3664 3665
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
3666 3667 3668 3669 3670 3671 3672

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

3673 3674 3675 3676
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
	u64 efer;

3677
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3678 3679 3680 3681 3682 3683 3684 3685 3686

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
3687
	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3688 3689

	/* Valid physical address? */
3690
	if (rax & 0xffff000000000000ULL)
3691 3692 3693 3694 3695
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

3696 3697
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
3698
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3699

3700
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3701 3702 3703 3704 3705
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

3706 3707
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
3708
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3709
	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3710

3711
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3712
	    ctxt->ops->check_pmc(ctxt, rcx))
3713 3714 3715 3716 3717
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3718 3719
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
3720 3721
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3722 3723 3724 3725 3726 3727 3728
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
3729 3730
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3731 3732 3733 3734 3735
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3736
#define D(_y) { .flags = (_y) }
3737 3738 3739
#define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
#define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
		      .intercept = x86_intercept_##_i, .check_perm = (_p) }
3740
#define N    D(NotImpl)
3741
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3742 3743
#define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
#define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3744
#define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3745
#define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3746
#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3747
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3748
#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3749
#define II(_f, _e, _i) \
3750
	{ .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3751
#define IIP(_f, _e, _i, _p) \
3752 3753
	{ .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
	  .intercept = x86_intercept_##_i, .check_perm = (_p) }
3754
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3755

3756
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
3757
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3758
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
3759
#define F2bv(_f, _e)  F((_f) | ByteOp, _e), F(_f, _e)
3760 3761
#define I2bvIP(_f, _e, _i, _p) \
	IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3762

3763 3764 3765
#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3766

3767 3768
static const struct opcode group7_rm0[] = {
	N,
3769
	I(SrcNone | Priv | EmulateOnUD,	em_hypercall),
3770 3771 3772
	N, N, N, N, N, N,
};

3773
static const struct opcode group7_rm1[] = {
3774 3775
	DI(SrcNone | Priv, monitor),
	DI(SrcNone | Priv, mwait),
3776 3777 3778
	N, N, N, N, N, N,
};

3779
static const struct opcode group7_rm3[] = {
3780
	DIP(SrcNone | Prot | Priv,		vmrun,		check_svme_pa),
3781
	II(SrcNone  | Prot | EmulateOnUD,	em_hypercall,	vmmcall),
3782 3783 3784 3785 3786 3787
	DIP(SrcNone | Prot | Priv,		vmload,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		vmsave,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		stgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		clgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		skinit,		check_svme),
	DIP(SrcNone | Prot | Priv,		invlpga,	check_svme),
3788
};
3789

3790
static const struct opcode group7_rm7[] = {
3791
	N,
3792
	DIP(SrcNone, rdtscp, check_rdtsc),
3793 3794
	N, N, N, N, N, N,
};
3795

3796
static const struct opcode group1[] = {
3797 3798 3799 3800 3801 3802 3803 3804
	F(Lock, em_add),
	F(Lock | PageTable, em_or),
	F(Lock, em_adc),
	F(Lock, em_sbb),
	F(Lock | PageTable, em_and),
	F(Lock, em_sub),
	F(Lock, em_xor),
	F(NoWrite, em_cmp),
3805 3806
};

3807
static const struct opcode group1A[] = {
3808
	I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
3809 3810
};

3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821
static const struct opcode group2[] = {
	F(DstMem | ModRM, em_rol),
	F(DstMem | ModRM, em_ror),
	F(DstMem | ModRM, em_rcl),
	F(DstMem | ModRM, em_rcr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_shr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_sar),
};

3822
static const struct opcode group3[] = {
3823 3824
	F(DstMem | SrcImm | NoWrite, em_test),
	F(DstMem | SrcImm | NoWrite, em_test),
3825 3826
	F(DstMem | SrcNone | Lock, em_not),
	F(DstMem | SrcNone | Lock, em_neg),
3827 3828
	F(DstXacc | Src2Mem, em_mul_ex),
	F(DstXacc | Src2Mem, em_imul_ex),
3829 3830
	F(DstXacc | Src2Mem, em_div_ex),
	F(DstXacc | Src2Mem, em_idiv_ex),
3831 3832
};

3833
static const struct opcode group4[] = {
3834 3835
	F(ByteOp | DstMem | SrcNone | Lock, em_inc),
	F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3836 3837 3838
	N, N, N, N, N, N,
};

3839
static const struct opcode group5[] = {
3840 3841
	F(DstMem | SrcNone | Lock,		em_inc),
	F(DstMem | SrcNone | Lock,		em_dec),
3842
	I(SrcMem | NearBranch,			em_call_near_abs),
3843
	I(SrcMemFAddr | ImplicitOps | Stack,	em_call_far),
3844
	I(SrcMem | NearBranch,			em_jmp_abs),
3845 3846
	I(SrcMemFAddr | ImplicitOps,		em_jmp_far),
	I(SrcMem | Stack,			em_push), D(Undefined),
3847 3848
};

3849
static const struct opcode group6[] = {
3850 3851
	DI(Prot | DstMem,	sldt),
	DI(Prot | DstMem,	str),
A
Avi Kivity 已提交
3852
	II(Prot | Priv | SrcMem16, em_lldt, lldt),
A
Avi Kivity 已提交
3853
	II(Prot | Priv | SrcMem16, em_ltr, ltr),
3854 3855 3856
	N, N, N, N,
};

3857
static const struct group_dual group7 = { {
3858 3859
	II(Mov | DstMem,			em_sgdt, sgdt),
	II(Mov | DstMem,			em_sidt, sidt),
3860 3861 3862 3863 3864
	II(SrcMem | Priv,			em_lgdt, lgdt),
	II(SrcMem | Priv,			em_lidt, lidt),
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	II(SrcMem | ByteOp | Priv | NoAccess,	em_invlpg, invlpg),
3865
}, {
3866
	EXT(0, group7_rm0),
3867
	EXT(0, group7_rm1),
3868
	N, EXT(0, group7_rm3),
3869 3870 3871
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	EXT(0, group7_rm7),
3872 3873
} };

3874
static const struct opcode group8[] = {
3875
	N, N, N, N,
3876 3877 3878 3879
	F(DstMem | SrcImmByte | NoWrite,		em_bt),
	F(DstMem | SrcImmByte | Lock | PageTable,	em_bts),
	F(DstMem | SrcImmByte | Lock,			em_btr),
	F(DstMem | SrcImmByte | Lock | PageTable,	em_btc),
3880 3881
};

3882
static const struct group_dual group9 = { {
3883
	N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3884 3885 3886 3887
}, {
	N, N, N, N, N, N, N, N,
} };

3888
static const struct opcode group11[] = {
3889
	I(DstMem | SrcImm | Mov | PageTable, em_mov),
3890
	X7(D(Undefined)),
3891 3892
};

3893
static const struct gprefix pfx_0f_ae_7 = {
3894
	I(SrcMem | ByteOp, em_clflush), N, N, N,
3895 3896 3897 3898 3899 3900 3901 3902
};

static const struct group_dual group15 = { {
	N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
}, {
	N, N, N, N, N, N, N, N,
} };

3903
static const struct gprefix pfx_0f_6f_0f_7f = {
3904
	I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3905 3906
};

3907 3908 3909 3910
static const struct instr_dual instr_dual_0f_2b = {
	I(0, em_mov), N
};

3911
static const struct gprefix pfx_0f_2b = {
3912
	ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
3913 3914
};

3915
static const struct gprefix pfx_0f_28_0f_29 = {
3916
	I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3917 3918
};

3919 3920 3921 3922
static const struct gprefix pfx_0f_e7 = {
	N, I(Sse, em_mov), N, N,
};

3923
static const struct escape escape_d9 = { {
3924
	N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_db = { {
	N, N, N, N, N, N, N, N,
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_dd = { {
3966
	N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

3986 3987 3988 3989
static const struct instr_dual instr_dual_0f_c3 = {
	I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
};

3990 3991 3992 3993
static const struct mode_dual mode_dual_63 = {
	N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
};

3994
static const struct opcode opcode_table[256] = {
3995
	/* 0x00 - 0x07 */
3996
	F6ALU(Lock, em_add),
3997 3998
	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3999
	/* 0x08 - 0x0F */
4000
	F6ALU(Lock | PageTable, em_or),
4001 4002
	I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
	N,
4003
	/* 0x10 - 0x17 */
4004
	F6ALU(Lock, em_adc),
4005 4006
	I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4007
	/* 0x18 - 0x1F */
4008
	F6ALU(Lock, em_sbb),
4009 4010
	I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4011
	/* 0x20 - 0x27 */
4012
	F6ALU(Lock | PageTable, em_and), N, N,
4013
	/* 0x28 - 0x2F */
4014
	F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4015
	/* 0x30 - 0x37 */
4016
	F6ALU(Lock, em_xor), N, N,
4017
	/* 0x38 - 0x3F */
4018
	F6ALU(NoWrite, em_cmp), N, N,
4019
	/* 0x40 - 0x4F */
4020
	X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4021
	/* 0x50 - 0x57 */
4022
	X8(I(SrcReg | Stack, em_push)),
4023
	/* 0x58 - 0x5F */
4024
	X8(I(DstReg | Stack, em_pop)),
4025
	/* 0x60 - 0x67 */
4026 4027
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
4028
	N, MD(ModRM, &mode_dual_63),
4029 4030
	N, N, N, N,
	/* 0x68 - 0x6F */
4031 4032
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4033 4034
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4035
	I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4036
	I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4037
	/* 0x70 - 0x7F */
4038
	X16(D(SrcImmByte | NearBranch)),
4039
	/* 0x80 - 0x87 */
4040 4041 4042 4043
	G(ByteOp | DstMem | SrcImm, group1),
	G(DstMem | SrcImm, group1),
	G(ByteOp | DstMem | SrcImm | No64, group1),
	G(DstMem | SrcImmByte, group1),
4044
	F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4045
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4046
	/* 0x88 - 0x8F */
4047
	I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4048
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4049
	I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4050 4051 4052
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
4053
	/* 0x90 - 0x97 */
4054
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4055
	/* 0x98 - 0x9F */
4056
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4057
	I(SrcImmFAddr | No64, em_call_far), N,
4058
	II(ImplicitOps | Stack, em_pushf, pushf),
P
Paolo Bonzini 已提交
4059 4060
	II(ImplicitOps | Stack, em_popf, popf),
	I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4061
	/* 0xA0 - 0xA7 */
4062
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4063
	I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4064
	I2bv(SrcSI | DstDI | Mov | String, em_mov),
4065
	F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4066
	/* 0xA8 - 0xAF */
4067
	F2bv(DstAcc | SrcImm | NoWrite, em_test),
4068 4069
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4070
	F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4071
	/* 0xB0 - 0xB7 */
4072
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4073
	/* 0xB8 - 0xBF */
4074
	X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4075
	/* 0xC0 - 0xC7 */
4076
	G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4077 4078
	I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
	I(ImplicitOps | NearBranch, em_ret),
4079 4080
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4081
	G(ByteOp, group11), G(0, group11),
4082
	/* 0xC8 - 0xCF */
A
Avi Kivity 已提交
4083
	I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4084 4085
	I(ImplicitOps | SrcImmU16, em_ret_far_imm),
	I(ImplicitOps, em_ret_far),
4086
	D(ImplicitOps), DI(SrcImmByte, intn),
4087
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4088
	/* 0xD0 - 0xD7 */
4089 4090
	G(Src2One | ByteOp, group2), G(Src2One, group2),
	G(Src2CL | ByteOp, group2), G(Src2CL, group2),
P
Paolo Bonzini 已提交
4091
	I(DstAcc | SrcImmUByte | No64, em_aam),
P
Paolo Bonzini 已提交
4092 4093
	I(DstAcc | SrcImmUByte | No64, em_aad),
	F(DstAcc | ByteOp | No64, em_salc),
P
Paolo Bonzini 已提交
4094
	I(DstAcc | SrcXLat | ByteOp, em_mov),
4095
	/* 0xD8 - 0xDF */
4096
	N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4097
	/* 0xE0 - 0xE7 */
4098 4099
	X3(I(SrcImmByte | NearBranch, em_loop)),
	I(SrcImmByte | NearBranch, em_jcxz),
4100 4101
	I2bvIP(SrcImmUByte | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4102
	/* 0xE8 - 0xEF */
4103 4104 4105
	I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
	I(SrcImmFAddr | No64, em_jmp_far),
	D(SrcImmByte | ImplicitOps | NearBranch),
4106 4107
	I2bvIP(SrcDX | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4108
	/* 0xF0 - 0xF7 */
4109
	N, DI(ImplicitOps, icebp), N, N,
4110 4111
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
4112
	/* 0xF8 - 0xFF */
4113 4114
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4115 4116 4117
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

4118
static const struct opcode twobyte_table[256] = {
4119
	/* 0x00 - 0x0F */
4120
	G(0, group6), GD(0, &group7), N, N,
4121
	N, I(ImplicitOps | EmulateOnUD, em_syscall),
4122
	II(ImplicitOps | Priv, em_clts, clts), N,
4123
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4124
	N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4125
	/* 0x10 - 0x1F */
P
Paolo Bonzini 已提交
4126
	N, N, N, N, N, N, N, N,
4127 4128
	D(ImplicitOps | ModRM | SrcMem | NoAccess),
	N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4129
	/* 0x20 - 0x2F */
4130 4131 4132 4133 4134 4135
	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
						check_cr_write),
	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
						check_dr_write),
4136
	N, N, N, N,
4137 4138
	GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
	GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4139
	N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4140
	N, N, N, N,
4141
	/* 0x30 - 0x3F */
4142
	II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4143
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4144
	II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4145
	IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4146 4147
	I(ImplicitOps | EmulateOnUD, em_sysenter),
	I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4148
	N, N,
4149 4150
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
4151
	X16(D(DstReg | SrcMem | ModRM)),
4152 4153 4154
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
4155 4156 4157 4158
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4159
	/* 0x70 - 0x7F */
4160 4161 4162 4163
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4164
	/* 0x80 - 0x8F */
4165
	X16(D(SrcImm | NearBranch)),
4166
	/* 0x90 - 0x9F */
4167
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4168
	/* 0xA0 - 0xA7 */
4169
	I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4170 4171
	II(ImplicitOps, em_cpuid, cpuid),
	F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4172 4173
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4174
	/* 0xA8 - 0xAF */
4175
	I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4176
	DI(ImplicitOps, rsm),
4177
	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4178 4179
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4180
	GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4181
	/* 0xB0 - 0xB7 */
4182
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4183
	I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4184
	F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4185 4186
	I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4187
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4188 4189
	/* 0xB8 - 0xBF */
	N, N,
4190
	G(BitOp, group8),
4191
	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4192 4193
	I(DstReg | SrcMem | ModRM, em_bsf_c),
	I(DstReg | SrcMem | ModRM, em_bsr_c),
4194
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
A
Avi Kivity 已提交
4195
	/* 0xC0 - 0xC7 */
4196
	F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4197
	N, ID(0, &instr_dual_0f_c3),
4198
	N, N, N, GD(0, &group9),
A
Avi Kivity 已提交
4199 4200
	/* 0xC8 - 0xCF */
	X8(I(DstReg, em_bswap)),
4201 4202 4203
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
4204 4205
	N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
	N, N, N, N, N, N, N, N,
4206 4207 4208 4209
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

4210 4211 4212 4213 4214 4215 4216 4217
static const struct instr_dual instr_dual_0f_38_f0 = {
	I(DstReg | SrcMem | Mov, em_movbe), N
};

static const struct instr_dual instr_dual_0f_38_f1 = {
	I(DstMem | SrcReg | Mov, em_movbe), N
};

4218
static const struct gprefix three_byte_0f_38_f0 = {
4219
	ID(0, &instr_dual_0f_38_f0), N, N, N
4220 4221 4222
};

static const struct gprefix three_byte_0f_38_f1 = {
4223
	ID(0, &instr_dual_0f_38_f1), N, N, N
4224 4225 4226 4227 4228 4229 4230 4231 4232
};

/*
 * Insns below are selected by the prefix which indexed by the third opcode
 * byte.
 */
static const struct opcode opcode_map_0f_38[256] = {
	/* 0x00 - 0x7f */
	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
B
Borislav Petkov 已提交
4233 4234 4235
	/* 0x80 - 0xef */
	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
	/* 0xf0 - 0xf1 */
4236 4237
	GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
	GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
B
Borislav Petkov 已提交
4238 4239
	/* 0xf2 - 0xff */
	N, N, X4(N), X8(N)
4240 4241
};

4242 4243 4244 4245 4246
#undef D
#undef N
#undef G
#undef GD
#undef I
4247
#undef GP
4248
#undef EXT
4249
#undef MD
N
Nadav Amit 已提交
4250
#undef ID
4251

4252
#undef D2bv
4253
#undef D2bvIP
4254
#undef I2bv
4255
#undef I2bvIP
4256
#undef I6ALU
4257

4258
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4259 4260 4261
{
	unsigned size;

4262
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
4275
	op->addr.mem.ea = ctxt->_eip;
4276 4277 4278
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
4279
		op->val = insn_fetch(s8, ctxt);
4280 4281
		break;
	case 2:
4282
		op->val = insn_fetch(s16, ctxt);
4283 4284
		break;
	case 4:
4285
		op->val = insn_fetch(s32, ctxt);
4286
		break;
4287 4288 4289
	case 8:
		op->val = insn_fetch(s64, ctxt);
		break;
4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

4308 4309 4310 4311 4312 4313 4314
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
			  unsigned d)
{
	int rc = X86EMUL_CONTINUE;

	switch (d) {
	case OpReg:
4315
		decode_register_operand(ctxt, op);
4316 4317
		break;
	case OpImmUByte:
4318
		rc = decode_imm(ctxt, op, 1, false);
4319 4320
		break;
	case OpMem:
4321
		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4322 4323 4324
	mem_common:
		*op = ctxt->memop;
		ctxt->memopp = op;
4325
		if (ctxt->d & BitOp)
4326 4327 4328
			fetch_bit_operand(ctxt);
		op->orig_val = op->val;
		break;
4329
	case OpMem64:
4330
		ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4331
		goto mem_common;
4332 4333 4334
	case OpAcc:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4335
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4336 4337 4338
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356
	case OpAccLo:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
	case OpAccHi:
		if (ctxt->d & ByteOp) {
			op->type = OP_NONE;
			break;
		}
		op->type = OP_REG;
		op->bytes = ctxt->op_bytes;
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
4357 4358 4359 4360
	case OpDI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4361
			register_address(ctxt, VCPU_REGS_RDI);
4362 4363
		op->addr.mem.seg = VCPU_SREG_ES;
		op->val = 0;
4364
		op->count = 1;
4365 4366 4367 4368
		break;
	case OpDX:
		op->type = OP_REG;
		op->bytes = 2;
4369
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4370 4371
		fetch_register_operand(op);
		break;
4372
	case OpCL:
4373
		op->type = OP_IMM;
4374
		op->bytes = 1;
4375
		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4376 4377 4378 4379 4380
		break;
	case OpImmByte:
		rc = decode_imm(ctxt, op, 1, true);
		break;
	case OpOne:
4381
		op->type = OP_IMM;
4382 4383 4384 4385 4386 4387
		op->bytes = 1;
		op->val = 1;
		break;
	case OpImm:
		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
		break;
4388 4389 4390
	case OpImm64:
		rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
		break;
4391 4392
	case OpMem8:
		ctxt->memop.bytes = 1;
4393
		if (ctxt->memop.type == OP_REG) {
4394 4395
			ctxt->memop.addr.reg = decode_register(ctxt,
					ctxt->modrm_rm, true);
4396 4397
			fetch_register_operand(&ctxt->memop);
		}
4398
		goto mem_common;
4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414
	case OpMem16:
		ctxt->memop.bytes = 2;
		goto mem_common;
	case OpMem32:
		ctxt->memop.bytes = 4;
		goto mem_common;
	case OpImmU16:
		rc = decode_imm(ctxt, op, 2, false);
		break;
	case OpImmU:
		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
		break;
	case OpSI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4415
			register_address(ctxt, VCPU_REGS_RSI);
B
Bandan Das 已提交
4416
		op->addr.mem.seg = ctxt->seg_override;
4417
		op->val = 0;
4418
		op->count = 1;
4419
		break;
P
Paolo Bonzini 已提交
4420 4421 4422 4423
	case OpXLat:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4424
			address_mask(ctxt,
P
Paolo Bonzini 已提交
4425 4426
				reg_read(ctxt, VCPU_REGS_RBX) +
				(reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
B
Bandan Das 已提交
4427
		op->addr.mem.seg = ctxt->seg_override;
P
Paolo Bonzini 已提交
4428 4429
		op->val = 0;
		break;
4430 4431 4432 4433 4434 4435 4436 4437 4438
	case OpImmFAddr:
		op->type = OP_IMM;
		op->addr.mem.ea = ctxt->_eip;
		op->bytes = ctxt->op_bytes + 2;
		insn_fetch_arr(op->valptr, op->bytes, ctxt);
		break;
	case OpMemFAddr:
		ctxt->memop.bytes = ctxt->op_bytes + 2;
		goto mem_common;
4439
	case OpES:
4440
		op->type = OP_IMM;
4441 4442 4443
		op->val = VCPU_SREG_ES;
		break;
	case OpCS:
4444
		op->type = OP_IMM;
4445 4446 4447
		op->val = VCPU_SREG_CS;
		break;
	case OpSS:
4448
		op->type = OP_IMM;
4449 4450 4451
		op->val = VCPU_SREG_SS;
		break;
	case OpDS:
4452
		op->type = OP_IMM;
4453 4454 4455
		op->val = VCPU_SREG_DS;
		break;
	case OpFS:
4456
		op->type = OP_IMM;
4457 4458 4459
		op->val = VCPU_SREG_FS;
		break;
	case OpGS:
4460
		op->type = OP_IMM;
4461 4462
		op->val = VCPU_SREG_GS;
		break;
4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473
	case OpImplicit:
		/* Special instructions do their own operand decoding. */
	default:
		op->type = OP_NONE; /* Disable writeback. */
		break;
	}

done:
	return rc;
}

4474
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4475 4476 4477
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
4478
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4479
	bool op_prefix = false;
B
Bandan Das 已提交
4480
	bool has_seg_override = false;
4481
	struct opcode opcode;
4482

4483 4484
	ctxt->memop.type = OP_NONE;
	ctxt->memopp = NULL;
4485
	ctxt->_eip = ctxt->eip;
4486 4487
	ctxt->fetch.ptr = ctxt->fetch.data;
	ctxt->fetch.end = ctxt->fetch.data + insn_len;
B
Borislav Petkov 已提交
4488
	ctxt->opcode_len = 1;
4489
	if (insn_len > 0)
4490
		memcpy(ctxt->fetch.data, insn, insn_len);
4491
	else {
4492
		rc = __do_insn_fetch_bytes(ctxt, 1);
4493 4494 4495
		if (rc != X86EMUL_CONTINUE)
			return rc;
	}
4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
4513
		return EMULATION_FAILED;
4514 4515
	}

4516 4517
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
4518 4519 4520

	/* Legacy prefixes. */
	for (;;) {
4521
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
4522
		case 0x66:	/* operand-size override */
4523
			op_prefix = true;
4524
			/* switch between 2/4 bytes */
4525
			ctxt->op_bytes = def_op_bytes ^ 6;
4526 4527 4528 4529
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
4530
				ctxt->ad_bytes = def_ad_bytes ^ 12;
4531 4532
			else
				/* switch between 2/4 bytes */
4533
				ctxt->ad_bytes = def_ad_bytes ^ 6;
4534 4535 4536 4537 4538
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
B
Bandan Das 已提交
4539 4540
			has_seg_override = true;
			ctxt->seg_override = (ctxt->b >> 3) & 3;
4541 4542 4543
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
B
Bandan Das 已提交
4544 4545
			has_seg_override = true;
			ctxt->seg_override = ctxt->b & 7;
4546 4547 4548 4549
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
4550
			ctxt->rex_prefix = ctxt->b;
4551 4552
			continue;
		case 0xf0:	/* LOCK */
4553
			ctxt->lock_prefix = 1;
4554 4555 4556
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
4557
			ctxt->rep_prefix = ctxt->b;
4558 4559 4560 4561 4562 4563 4564
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

4565
		ctxt->rex_prefix = 0;
4566 4567 4568 4569 4570
	}

done_prefixes:

	/* REX prefix. */
4571 4572
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
4573 4574

	/* Opcode byte(s). */
4575
	opcode = opcode_table[ctxt->b];
4576
	/* Two-byte opcode? */
4577
	if (ctxt->b == 0x0f) {
B
Borislav Petkov 已提交
4578
		ctxt->opcode_len = 2;
4579
		ctxt->b = insn_fetch(u8, ctxt);
4580
		opcode = twobyte_table[ctxt->b];
4581 4582 4583 4584 4585 4586 4587

		/* 0F_38 opcode map */
		if (ctxt->b == 0x38) {
			ctxt->opcode_len = 3;
			ctxt->b = insn_fetch(u8, ctxt);
			opcode = opcode_map_0f_38[ctxt->b];
		}
4588
	}
4589
	ctxt->d = opcode.flags;
4590

4591 4592 4593
	if (ctxt->d & ModRM)
		ctxt->modrm = insn_fetch(u8, ctxt);

4594 4595
	/* vex-prefix instructions are not implemented */
	if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4596
	    (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4597 4598 4599
		ctxt->d = NotImpl;
	}

4600 4601
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
4602
		case Group:
4603
			goffset = (ctxt->modrm >> 3) & 7;
4604 4605 4606
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
4607 4608
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
4609 4610 4611 4612 4613
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
4614
			goffset = ctxt->modrm & 7;
4615
			opcode = opcode.u.group[goffset];
4616 4617
			break;
		case Prefix:
4618
			if (ctxt->rep_prefix && op_prefix)
4619
				return EMULATION_FAILED;
4620
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4621 4622 4623 4624 4625 4626 4627
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
4628 4629 4630 4631 4632 4633
		case Escape:
			if (ctxt->modrm > 0xbf)
				opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
			else
				opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
			break;
4634 4635 4636 4637 4638 4639
		case InstrDual:
			if ((ctxt->modrm >> 6) == 3)
				opcode = opcode.u.idual->mod3;
			else
				opcode = opcode.u.idual->mod012;
			break;
4640 4641 4642 4643 4644 4645
		case ModeDual:
			if (ctxt->mode == X86EMUL_MODE_PROT64)
				opcode = opcode.u.mdual->mode64;
			else
				opcode = opcode.u.mdual->mode32;
			break;
4646
		default:
4647
			return EMULATION_FAILED;
4648
		}
4649

4650
		ctxt->d &= ~(u64)GroupMask;
4651
		ctxt->d |= opcode.flags;
4652 4653
	}

4654 4655 4656 4657
	/* Unrecognised? */
	if (ctxt->d == 0)
		return EMULATION_FAILED;

4658
	ctxt->execute = opcode.u.execute;
4659

4660 4661 4662
	if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
		return EMULATION_FAILED;

4663
	if (unlikely(ctxt->d &
4664 4665
	    (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
	     No16))) {
4666 4667 4668 4669 4670 4671
		/*
		 * These are copied unconditionally here, and checked unconditionally
		 * in x86_emulate_insn.
		 */
		ctxt->check_perm = opcode.check_perm;
		ctxt->intercept = opcode.intercept;
4672

4673 4674
		if (ctxt->d & NotImpl)
			return EMULATION_FAILED;
4675

4676 4677 4678 4679 4680 4681
		if (mode == X86EMUL_MODE_PROT64) {
			if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
				ctxt->op_bytes = 8;
			else if (ctxt->d & NearBranch)
				ctxt->op_bytes = 8;
		}
4682

4683 4684 4685 4686 4687 4688 4689
		if (ctxt->d & Op3264) {
			if (mode == X86EMUL_MODE_PROT64)
				ctxt->op_bytes = 8;
			else
				ctxt->op_bytes = 4;
		}

4690 4691 4692
		if ((ctxt->d & No16) && ctxt->op_bytes == 2)
			ctxt->op_bytes = 4;

4693 4694 4695 4696 4697
		if (ctxt->d & Sse)
			ctxt->op_bytes = 16;
		else if (ctxt->d & Mmx)
			ctxt->op_bytes = 8;
	}
A
Avi Kivity 已提交
4698

4699
	/* ModRM and SIB bytes. */
4700
	if (ctxt->d & ModRM) {
4701
		rc = decode_modrm(ctxt, &ctxt->memop);
B
Bandan Das 已提交
4702 4703 4704 4705
		if (!has_seg_override) {
			has_seg_override = true;
			ctxt->seg_override = ctxt->modrm_seg;
		}
4706
	} else if (ctxt->d & MemAbs)
4707
		rc = decode_abs(ctxt, &ctxt->memop);
4708 4709 4710
	if (rc != X86EMUL_CONTINUE)
		goto done;

B
Bandan Das 已提交
4711 4712
	if (!has_seg_override)
		ctxt->seg_override = VCPU_SREG_DS;
4713

B
Bandan Das 已提交
4714
	ctxt->memop.addr.mem.seg = ctxt->seg_override;
4715 4716 4717 4718 4719

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
4720
	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4721 4722 4723
	if (rc != X86EMUL_CONTINUE)
		goto done;

4724 4725 4726 4727
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
4728
	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4729 4730 4731
	if (rc != X86EMUL_CONTINUE)
		goto done;

4732
	/* Decode and fetch the destination operand: register or memory. */
4733
	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4734

4735
	if (ctxt->rip_relative)
4736 4737
		ctxt->memopp->addr.mem.ea = address_mask(ctxt,
					ctxt->memopp->addr.mem.ea + ctxt->_eip);
4738

4739
done:
4740
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4741 4742
}

4743 4744 4745 4746 4747
bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
{
	return ctxt->d & PageTable;
}

4748 4749 4750 4751 4752 4753 4754 4755 4756
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
4757 4758 4759
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
4760
		 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
4761
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
4762
		    ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
4763 4764 4765 4766 4767
		return true;

	return false;
}

A
Avi Kivity 已提交
4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780
static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
{
	bool fault = false;

	ctxt->ops->get_fpu(ctxt);
	asm volatile("1: fwait \n\t"
		     "2: \n\t"
		     ".pushsection .fixup,\"ax\" \n\t"
		     "3: \n\t"
		     "movb $1, %[fault] \n\t"
		     "jmp 2b \n\t"
		     ".popsection \n\t"
		     _ASM_EXTABLE(1b, 3b)
4781
		     : [fault]"+qm"(fault));
A
Avi Kivity 已提交
4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796
	ctxt->ops->put_fpu(ctxt);

	if (unlikely(fault))
		return emulate_exception(ctxt, MF_VECTOR, 0, false);

	return X86EMUL_CONTINUE;
}

static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
				       struct operand *op)
{
	if (op->type == OP_MM)
		read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
}

4797 4798 4799
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
{
	ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4800 4801
	if (!(ctxt->d & ByteOp))
		fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4802
	asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4803 4804 4805
	    : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
	      [fastop]"+S"(fop)
	    : "c"(ctxt->src2.val));
4806
	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4807 4808
	if (!fop) /* exception is returned in fop variable */
		return emulate_de(ctxt);
4809 4810
	return X86EMUL_CONTINUE;
}
4811

4812 4813
void init_decode_cache(struct x86_emulate_ctxt *ctxt)
{
B
Bandan Das 已提交
4814 4815
	memset(&ctxt->rip_relative, 0,
	       (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4816 4817 4818 4819 4820 4821

	ctxt->io_read.pos = 0;
	ctxt->io_read.end = 0;
	ctxt->mem_read.end = 0;
}

4822
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4823
{
4824
	const struct x86_emulate_ops *ops = ctxt->ops;
4825
	int rc = X86EMUL_CONTINUE;
4826
	int saved_dst_type = ctxt->dst.type;
4827

4828
	ctxt->mem_read.pos = 0;
4829

4830 4831
	/* LOCK prefix is allowed only with some instructions */
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4832
		rc = emulate_ud(ctxt);
4833 4834 4835
		goto done;
	}

4836
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4837
		rc = emulate_ud(ctxt);
4838 4839 4840
		goto done;
	}

4841 4842 4843 4844 4845 4846 4847
	if (unlikely(ctxt->d &
		     (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
		if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
				(ctxt->d & Undefined)) {
			rc = emulate_ud(ctxt);
			goto done;
		}
A
Avi Kivity 已提交
4848

4849 4850 4851
		if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
		    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
			rc = emulate_ud(ctxt);
A
Avi Kivity 已提交
4852
			goto done;
4853
		}
A
Avi Kivity 已提交
4854

4855 4856
		if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
			rc = emulate_nm(ctxt);
4857
			goto done;
4858
		}
4859

4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872
		if (ctxt->d & Mmx) {
			rc = flush_pending_x87_faults(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			/*
			 * Now that we know the fpu is exception safe, we can fetch
			 * operands from it.
			 */
			fetch_possible_mmx_operand(ctxt, &ctxt->src);
			fetch_possible_mmx_operand(ctxt, &ctxt->src2);
			if (!(ctxt->d & Mov))
				fetch_possible_mmx_operand(ctxt, &ctxt->dst);
		}
4873

4874
		if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4875 4876 4877 4878 4879
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_PRE_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}
4880

4881 4882 4883 4884 4885 4886
		/* Instruction can only be executed in protected mode */
		if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
			rc = emulate_ud(ctxt);
			goto done;
		}

4887 4888
		/* Privileged instruction can be executed only in CPL=0 */
		if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4889 4890 4891 4892
			if (ctxt->d & PrivUD)
				rc = emulate_ud(ctxt);
			else
				rc = emulate_gp(ctxt, 0);
4893
			goto done;
4894
		}
4895

4896
		/* Do instruction specific permission checks */
4897
		if (ctxt->d & CheckPerm) {
4898 4899 4900 4901 4902
			rc = ctxt->check_perm(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

4903
		if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4904 4905 4906 4907 4908 4909 4910 4911 4912 4913
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_POST_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

		if (ctxt->rep_prefix && (ctxt->d & String)) {
			/* All REP prefixes have the same first termination condition */
			if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
				ctxt->eip = ctxt->_eip;
4914
				ctxt->eflags &= ~X86_EFLAGS_RF;
4915 4916
				goto done;
			}
4917 4918 4919
		}
	}

4920 4921 4922
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
4923
		if (rc != X86EMUL_CONTINUE)
4924
			goto done;
4925
		ctxt->src.orig_val64 = ctxt->src.val64;
4926 4927
	}

4928 4929 4930
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
4931 4932 4933 4934
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4935
	if ((ctxt->d & DstMask) == ImplicitOps)
4936 4937 4938
		goto special_insn;


4939
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4940
		/* optimisation - avoid slow emulated read if Mov */
4941 4942
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
4943
		if (rc != X86EMUL_CONTINUE) {
4944 4945
			if (!(ctxt->d & NoWrite) &&
			    rc == X86EMUL_PROPAGATE_FAULT &&
4946 4947
			    ctxt->exception.vector == PF_VECTOR)
				ctxt->exception.error_code |= PFERR_WRITE_MASK;
4948
			goto done;
4949
		}
4950
	}
4951 4952
	/* Copy full 64-bit value for CMPXCHG8B.  */
	ctxt->dst.orig_val64 = ctxt->dst.val64;
4953

4954 4955
special_insn:

4956
	if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4957
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
4958
					      X86_ICPT_POST_MEMACCESS);
4959 4960 4961 4962
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4963
	if (ctxt->rep_prefix && (ctxt->d & String))
4964
		ctxt->eflags |= X86_EFLAGS_RF;
4965
	else
4966
		ctxt->eflags &= ~X86_EFLAGS_RF;
4967

4968
	if (ctxt->execute) {
4969 4970 4971 4972 4973 4974 4975
		if (ctxt->d & Fastop) {
			void (*fop)(struct fastop *) = (void *)ctxt->execute;
			rc = fastop(ctxt, fop);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			goto writeback;
		}
4976
		rc = ctxt->execute(ctxt);
4977 4978 4979 4980 4981
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

B
Borislav Petkov 已提交
4982
	if (ctxt->opcode_len == 2)
A
Avi Kivity 已提交
4983
		goto twobyte_insn;
4984 4985
	else if (ctxt->opcode_len == 3)
		goto threebyte_insn;
A
Avi Kivity 已提交
4986

4987
	switch (ctxt->b) {
4988
	case 0x70 ... 0x7f: /* jcc (short) */
4989
		if (test_cc(ctxt->b, ctxt->eflags))
4990
			rc = jmp_rel(ctxt, ctxt->src.val);
4991
		break;
N
Nitin A Kamble 已提交
4992
	case 0x8d: /* lea r16/r32, m */
4993
		ctxt->dst.val = ctxt->src.addr.mem.ea;
N
Nitin A Kamble 已提交
4994
		break;
4995
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
4996
		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4997 4998 4999
			ctxt->dst.type = OP_NONE;
		else
			rc = em_xchg(ctxt);
5000
		break;
5001
	case 0x98: /* cbw/cwde/cdqe */
5002 5003 5004 5005
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5006 5007
		}
		break;
5008
	case 0xcc:		/* int3 */
5009 5010
		rc = emulate_int(ctxt, 3);
		break;
5011
	case 0xcd:		/* int n */
5012
		rc = emulate_int(ctxt, ctxt->src.val);
5013 5014
		break;
	case 0xce:		/* into */
5015
		if (ctxt->eflags & X86_EFLAGS_OF)
5016
			rc = emulate_int(ctxt, 4);
5017
		break;
5018
	case 0xe9: /* jmp rel */
5019
	case 0xeb: /* jmp rel short */
5020
		rc = jmp_rel(ctxt, ctxt->src.val);
5021
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
5022
		break;
5023
	case 0xf4:              /* hlt */
5024
		ctxt->ops->halt(ctxt);
5025
		break;
5026 5027
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
5028
		ctxt->eflags ^= X86_EFLAGS_CF;
5029 5030
		break;
	case 0xf8: /* clc */
5031
		ctxt->eflags &= ~X86_EFLAGS_CF;
5032
		break;
5033
	case 0xf9: /* stc */
5034
		ctxt->eflags |= X86_EFLAGS_CF;
5035
		break;
5036
	case 0xfc: /* cld */
5037
		ctxt->eflags &= ~X86_EFLAGS_DF;
5038 5039
		break;
	case 0xfd: /* std */
5040
		ctxt->eflags |= X86_EFLAGS_DF;
5041
		break;
5042 5043
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
5044
	}
5045

5046 5047 5048
	if (rc != X86EMUL_CONTINUE)
		goto done;

5049
writeback:
5050 5051 5052 5053 5054 5055
	if (ctxt->d & SrcWrite) {
		BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
		rc = writeback(ctxt, &ctxt->src);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
5056 5057 5058 5059 5060
	if (!(ctxt->d & NoWrite)) {
		rc = writeback(ctxt, &ctxt->dst);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
5061

5062 5063 5064 5065
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
5066
	ctxt->dst.type = saved_dst_type;
5067

5068
	if ((ctxt->d & SrcMask) == SrcSI)
5069
		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5070

5071
	if ((ctxt->d & DstMask) == DstDI)
5072
		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5073

5074
	if (ctxt->rep_prefix && (ctxt->d & String)) {
5075
		unsigned int count;
5076
		struct read_cache *r = &ctxt->io_read;
5077 5078 5079 5080
		if ((ctxt->d & SrcMask) == SrcSI)
			count = ctxt->src.count;
		else
			count = ctxt->dst.count;
5081
		register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5082

5083 5084 5085 5086 5087
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
5088
			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5089 5090 5091 5092 5093 5094
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
5095
				ctxt->mem_read.end = 0;
5096
				writeback_registers(ctxt);
5097 5098 5099
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
5100
		}
5101
		ctxt->eflags &= ~X86_EFLAGS_RF;
5102
	}
5103

5104
	ctxt->eip = ctxt->_eip;
5105 5106

done:
5107 5108
	if (rc == X86EMUL_PROPAGATE_FAULT) {
		WARN_ON(ctxt->exception.vector > 0x1f);
5109
		ctxt->have_exception = true;
5110
	}
5111 5112 5113
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

5114 5115 5116
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);

5117
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
5118 5119

twobyte_insn:
5120
	switch (ctxt->b) {
5121
	case 0x09:		/* wbinvd */
5122
		(ctxt->ops->wbinvd)(ctxt);
5123 5124
		break;
	case 0x08:		/* invd */
5125 5126
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
P
Paolo Bonzini 已提交
5127
	case 0x1f:		/* nop */
5128 5129
		break;
	case 0x20: /* mov cr, reg */
5130
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5131
		break;
A
Avi Kivity 已提交
5132
	case 0x21: /* mov from dr to reg */
5133
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
A
Avi Kivity 已提交
5134 5135
		break;
	case 0x40 ... 0x4f:	/* cmov */
5136 5137
		if (test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.val = ctxt->src.val;
5138
		else if (ctxt->op_bytes != 4)
5139
			ctxt->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
5140
		break;
5141
	case 0x80 ... 0x8f: /* jnz rel, etc*/
5142
		if (test_cc(ctxt->b, ctxt->eflags))
5143
			rc = jmp_rel(ctxt, ctxt->src.val);
5144
		break;
5145
	case 0x90 ... 0x9f:     /* setcc r/m8 */
5146
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5147
		break;
A
Avi Kivity 已提交
5148
	case 0xb6 ... 0xb7:	/* movzx */
5149
		ctxt->dst.bytes = ctxt->op_bytes;
5150
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5151
						       : (u16) ctxt->src.val;
A
Avi Kivity 已提交
5152 5153
		break;
	case 0xbe ... 0xbf:	/* movsx */
5154
		ctxt->dst.bytes = ctxt->op_bytes;
5155
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5156
							(s16) ctxt->src.val;
A
Avi Kivity 已提交
5157
		break;
5158 5159
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
5160
	}
5161

5162 5163
threebyte_insn:

5164 5165 5166
	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
5167 5168 5169
	goto writeback;

cannot_emulate:
5170
	return EMULATION_FAILED;
A
Avi Kivity 已提交
5171
}
5172 5173 5174 5175 5176 5177 5178 5179 5180 5181

void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
{
	invalidate_registers(ctxt);
}

void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
{
	writeback_registers(ctxt);
}