emulate.c 147.1 KB
Newer Older
A
Avi Kivity 已提交
1
/******************************************************************************
2
 * emulate.c
A
Avi Kivity 已提交
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
A
Avi Kivity 已提交
10 11
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
25
#include <asm/kvm_emulate.h>
26
#include <linux/stringify.h>
27
#include <asm/debugreg.h>
28
#include <asm/nospec-branch.h>
A
Avi Kivity 已提交
29

30
#include "x86.h"
31
#include "tss.h"
32
#include "mmu.h"
33

34 35 36
/*
 * Operand types
 */
37 38 39 40 41 42 43 44 45
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
46 47 48
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
49
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
50 51 52 53 54 55 56
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
57 58 59 60 61 62
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
63
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
64
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
P
Paolo Bonzini 已提交
65
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
66 67
#define OpAccLo           29ull  /* Low part of extended acc (AX/AX/EAX/RAX) */
#define OpAccHi           30ull  /* High part of extended acc (-/DX/EDX/RDX) */
68 69

#define OpBits             5  /* Width of operand field */
70
#define OpMask             ((1ull << OpBits) - 1)
71

A
Avi Kivity 已提交
72 73 74 75 76 77 78 79 80 81
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
82
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
83
/* Destination operand type. */
84 85 86 87 88 89 90
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
91
#define DstMem16    (OpMem16 << DstShift)
92 93
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
94
#define DstAccLo    (OpAccLo << DstShift)
95
#define DstMask     (OpMask << DstShift)
A
Avi Kivity 已提交
96
/* Source operand type. */
97 98 99 100 101 102 103 104 105 106 107 108
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
P
Paolo Bonzini 已提交
109
#define SrcXLat     (OpXLat << SrcShift)
110 111 112 113
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
114
#define SrcImm64    (OpImm64 << SrcShift)
115
#define SrcDX       (OpDX << SrcShift)
116
#define SrcMem8     (OpMem8 << SrcShift)
117
#define SrcAccHi    (OpAccHi << SrcShift)
118
#define SrcMask     (OpMask << SrcShift)
119 120 121 122 123 124 125 126 127
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
128
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
129
#define InstrDual   (6<<15)     /* Alternate instruction decoding of mod == 3 */
130
#define ModeDual    (7<<15)     /* Different instruction for 32/64 bit */
131
#define Sse         (1<<18)     /* SSE Vector instruction */
132 133 134 135
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
136
/* Misc flags */
137
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
138
#define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141
#define Undefined   (1<<25) /* No Such Instruction */
142
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
143
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
144
#define No64	    (1<<28)
145
#define PageTable   (1 << 29)   /* instruction used to write page table */
146
#define NotImpl     (1 << 30)   /* instruction is not implemented */
147
/* Source 2 operand type */
148
#define Src2Shift   (31)
149
#define Src2None    (OpNone << Src2Shift)
150
#define Src2Mem     (OpMem << Src2Shift)
151 152 153 154
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
155 156 157 158 159 160
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
161
#define Src2Mask    (OpMask << Src2Shift)
A
Avi Kivity 已提交
162
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
163
#define AlignMask   ((u64)7 << 41)
164
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
165 166 167
#define Unaligned   ((u64)2 << 41)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)3 << 41)  /* Advanced Vector Extensions */
#define Aligned16   ((u64)4 << 41)  /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
169
#define NoWrite     ((u64)1 << 45)  /* No writeback */
170
#define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
171
#define NoMod	    ((u64)1 << 47)  /* Mod field is ignored */
172 173
#define Intercept   ((u64)1 << 48)  /* Has valid intercept field */
#define CheckPerm   ((u64)1 << 49)  /* Has valid check_perm field */
174
#define PrivUD      ((u64)1 << 51)  /* #UD instead of #GP on CPL > 0 */
175
#define NearBranch  ((u64)1 << 52)  /* Near branches */
176
#define No16	    ((u64)1 << 53)  /* No 16 bit operand */
177
#define IncSP       ((u64)1 << 54)  /* SP is incremented before ModRM calc */
178
#define TwoMemOp    ((u64)1 << 55)  /* Instruction has two memory operand */
A
Avi Kivity 已提交
179

180
#define DstXacc     (DstAccLo | SrcAccHi | SrcWrite)
A
Avi Kivity 已提交
181

182 183 184 185 186 187 188 189
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
190

191 192 193 194 195 196
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

/*
 * fastop functions have a special calling convention:
 *
197 198
 * dst:    rax        (in/out)
 * src:    rdx        (in/out)
199 200
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
201
 * ex:     rsi        (in:fastop pointer, out:zero if exception)
202 203 204 205 206 207 208 209 210 211 212
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 *
 * fastop functions are declared as taking a never-defined fastop parameter,
 * so they can't be called from C directly.
 */

struct fastop;

213
struct opcode {
214 215
	u64 flags : 56;
	u64 intercept : 8;
216
	union {
217
		int (*execute)(struct x86_emulate_ctxt *ctxt);
218 219 220
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
221
		const struct escape *esc;
222
		const struct instr_dual *idual;
223
		const struct mode_dual *mdual;
224
		void (*fastop)(struct fastop *fake);
225
	} u;
226
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
227 228 229 230 231
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
232 233
};

234 235 236 237 238 239 240
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

241 242 243 244 245
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

246 247 248 249 250
struct instr_dual {
	struct opcode mod012;
	struct opcode mod3;
};

251 252 253 254 255
struct mode_dual {
	struct opcode mode32;
	struct opcode mode64;
};

256 257
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a

258 259 260 261 262 263 264
enum x86_transfer_type {
	X86_TRANSFER_NONE,
	X86_TRANSFER_CALL_JMP,
	X86_TRANSFER_RET,
	X86_TRANSFER_TASK_SWITCH,
};

265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

A
Avi Kivity 已提交
301 302 303 304
/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
305 306
#define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
		     X86_EFLAGS_PF|X86_EFLAGS_CF)
A
Avi Kivity 已提交
307

308 309 310 311 312 313
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

314 315
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));

316 317 318 319 320
#define FOP_FUNC(name) \
	".align " __stringify(FASTOP_SIZE) " \n\t" \
	".type " name ", @function \n\t" \
	name ":\n\t"

321 322 323 324 325 326
#define FOP_RET   "ret \n\t"

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
327
	    FOP_FUNC("em_" #op)
328 329 330 331

#define FOP_END \
	    ".popsection")

332 333 334
#define FOPNOP() \
	FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
	FOP_RET
335

336
#define FOP1E(op,  dst) \
337 338
	FOP_FUNC(#op "_" #dst) \
	"10: " #op " %" #dst " \n\t" FOP_RET
339 340 341

#define FOP1EEX(op,  dst) \
	FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
342 343 344 345 346 347 348 349 350

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

351 352 353 354 355 356 357 358 359
/* 1-operand, using src2 (for MUL/DIV r/m) */
#define FASTOP1SRC2(op, name) \
	FOP_START(name) \
	FOP1E(op, cl) \
	FOP1E(op, cx) \
	FOP1E(op, ecx) \
	ON64(FOP1E(op, rcx)) \
	FOP_END

360 361 362 363 364 365 366 367 368
/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
#define FASTOP1SRC2EX(op, name) \
	FOP_START(name) \
	FOP1EEX(op, cl) \
	FOP1EEX(op, cx) \
	FOP1EEX(op, ecx) \
	ON64(FOP1EEX(op, rcx)) \
	FOP_END

369
#define FOP2E(op,  dst, src)	   \
370 371
	FOP_FUNC(#op "_" #dst "_" #src) \
	#op " %" #src ", %" #dst " \n\t" FOP_RET
372 373 374

#define FASTOP2(op) \
	FOP_START(op) \
375 376 377 378
	FOP2E(op##b, al, dl) \
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
379 380
	FOP_END

381 382 383 384
/* 2 operand, word only */
#define FASTOP2W(op) \
	FOP_START(op) \
	FOPNOP() \
385 386 387
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
388 389
	FOP_END

390 391 392 393 394 395 396 397 398
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

399 400 401 402 403 404 405 406 407
/* 2 operand, src and dest are reversed */
#define FASTOP2R(op, name) \
	FOP_START(name) \
	FOP2E(op##b, dl, al) \
	FOP2E(op##w, dx, ax) \
	FOP2E(op##l, edx, eax) \
	ON64(FOP2E(op##q, rdx, rax)) \
	FOP_END

408
#define FOP3E(op,  dst, src, src2) \
409 410
	FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
	#op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
411 412 413 414 415

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
416 417 418
	FOP3E(op##w, ax, dx, cl) \
	FOP3E(op##l, eax, edx, cl) \
	ON64(FOP3E(op##q, rax, rdx, cl)) \
419 420
	FOP_END

421
/* Special case for SETcc - 1 instruction per cc */
422 423 424 425 426 427
#define FOP_SETCC(op) \
	".align 4 \n\t" \
	".type " #op ", @function \n\t" \
	#op ": \n\t" \
	#op " %al \n\t" \
	FOP_RET
428

429 430 431 432
asm(".pushsection .fixup, \"ax\"\n"
    ".global kvm_fastop_exception \n"
    "kvm_fastop_exception: xor %esi, %esi; ret\n"
    ".popsection");
433

434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

P
Paolo Bonzini 已提交
453 454 455
FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
FOP_END;

R
Radim Krčmář 已提交
456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
/*
 * XXX: inoutclob user must know where the argument is being expanded.
 *      Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
 */
#define asm_safe(insn, inoutclob...) \
({ \
	int _fault = 0; \
 \
	asm volatile("1:" insn "\n" \
	             "2:\n" \
	             ".pushsection .fixup, \"ax\"\n" \
	             "3: movl $1, %[_fault]\n" \
	             "   jmp  2b\n" \
	             ".popsection\n" \
	             _ASM_EXTABLE(1b, 3b) \
	             : [_fault] "+qm"(_fault) inoutclob ); \
 \
	_fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
})

476 477 478 479 480 481
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
482 483 484 485 486
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
487
		.dst_val    = ctxt->dst.val64,
488 489 490
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
491 492 493
		.next_rip   = ctxt->eip,
	};

494
	return ctxt->ops->intercept(ctxt, &info, stage);
495 496
}

A
Avi Kivity 已提交
497 498 499 500 501
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520
static void assign_register(unsigned long *reg, u64 val, int bytes)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (bytes) {
	case 1:
		*(u8 *)reg = (u8)val;
		break;
	case 2:
		*(u16 *)reg = (u16)val;
		break;
	case 4:
		*reg = (u32)val;
		break;	/* 64b: zero-extend */
	case 8:
		*reg = val;
		break;
	}
}

521
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
522
{
523
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
524 525
}

A
Avi Kivity 已提交
526 527 528 529 530 531 532 533 534 535 536
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

A
Avi Kivity 已提交
537 538 539 540 541
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

A
Avi Kivity 已提交
542
/* Access/update address held in a register, based on addressing mode. */
543
static inline unsigned long
544
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
545
{
546
	if (ctxt->ad_bytes == sizeof(unsigned long))
547 548
		return reg;
	else
549
		return reg & ad_mask(ctxt);
550 551 552
}

static inline unsigned long
553
register_address(struct x86_emulate_ctxt *ctxt, int reg)
554
{
555
	return address_mask(ctxt, reg_read(ctxt, reg));
556 557
}

558 559 560 561 562
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

563
static inline void
564
register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
565
{
566
	ulong *preg = reg_rmw(ctxt, reg);
567

568
	assign_register(preg, *preg + inc, ctxt->ad_bytes);
569 570 571 572
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
573
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
574
}
A
Avi Kivity 已提交
575

576 577 578 579 580 581 582
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

583
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
584 585 586 587
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

588
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
589 590
}

591 592
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
593
{
594
	WARN_ON(vec > 0x1f);
595 596 597
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
598
	return X86EMUL_PROPAGATE_FAULT;
599 600
}

601 602 603 604 605
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

606
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
607
{
608
	return emulate_exception(ctxt, GP_VECTOR, err, true);
609 610
}

611 612 613 614 615
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

616
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
617
{
618
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
619 620
}

621
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
622
{
623
	return emulate_exception(ctxt, TS_VECTOR, err, true);
624 625
}

626 627
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
628
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
629 630
}

A
Avi Kivity 已提交
631 632 633 634 635
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

656 657 658 659 660 661
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
662 663
 * subject to the same check.  FXSAVE and FXRSTOR are checked here too as their
 * 512 bytes of data must be aligned to a 16 byte boundary.
664
 */
665
static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
666
{
667
	u64 alignment = ctxt->d & AlignMask;
668 669

	if (likely(size < 16))
670
		return 1;
671

672 673 674
	switch (alignment) {
	case Unaligned:
	case Avx:
675
		return 1;
676
	case Aligned16:
677
		return 16;
678 679
	case Aligned:
	default:
680
		return size;
681
	}
682 683
}

684 685 686 687
static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
				       struct segmented_address addr,
				       unsigned *max_size, unsigned size,
				       bool write, bool fetch,
688
				       enum x86emul_mode mode, ulong *linear)
689
{
690 691
	struct desc_struct desc;
	bool usable;
692
	ulong la;
693
	u32 lim;
694
	u16 sel;
695
	u8  va_bits;
696

697
	la = seg_base(ctxt, addr.seg) + addr.ea;
698
	*max_size = 0;
699
	switch (mode) {
700
	case X86EMUL_MODE_PROT64:
701
		*linear = la;
702 703
		va_bits = ctxt_virt_addr_bits(ctxt);
		if (get_canonical(la, va_bits) != la)
704
			goto bad;
705

706
		*max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
707 708
		if (size > *max_size)
			goto bad;
709 710
		break;
	default:
711
		*linear = la = (u32)la;
712 713
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
714 715
		if (!usable)
			goto bad;
716 717 718
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
719 720
			goto bad;
		/* unreadable code segment */
721
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
722 723
			goto bad;
		lim = desc_limit_scaled(&desc);
724
		if (!(desc.type & 8) && (desc.type & 4)) {
G
Guo Chao 已提交
725
			/* expand-down segment */
726
			if (addr.ea <= lim)
727 728 729
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
		}
730 731
		if (addr.ea > lim)
			goto bad;
732 733 734 735 736 737 738
		if (lim == 0xffffffff)
			*max_size = ~0u;
		else {
			*max_size = (u64)lim + 1 - addr.ea;
			if (size > *max_size)
				goto bad;
		}
739 740
		break;
	}
741
	if (la & (insn_alignment(ctxt, size) - 1))
742
		return emulate_gp(ctxt, 0);
743
	return X86EMUL_CONTINUE;
744 745
bad:
	if (addr.seg == VCPU_SREG_SS)
746
		return emulate_ss(ctxt, 0);
747
	else
748
		return emulate_gp(ctxt, 0);
749 750
}

751 752 753 754 755
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
756
	unsigned max_size;
757 758
	return __linearize(ctxt, addr, &max_size, size, write, false,
			   ctxt->mode, linear);
759 760
}

761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780
static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
			     enum x86emul_mode mode)
{
	ulong linear;
	int rc;
	unsigned max_size;
	struct segmented_address addr = { .seg = VCPU_SREG_CS,
					   .ea = dst };

	if (ctxt->op_bytes != sizeof(unsigned long))
		addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
	rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
	if (rc == X86EMUL_CONTINUE)
		ctxt->_eip = addr.ea;
	return rc;
}

static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
{
	return assign_eip(ctxt, dst, ctxt->mode);
781 782
}

783 784 785 786
static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
			  const struct desc_struct *cs_desc)
{
	enum x86emul_mode mode = ctxt->mode;
787
	int rc;
788 789

#ifdef CONFIG_X86_64
790 791 792
	if (ctxt->mode >= X86EMUL_MODE_PROT16) {
		if (cs_desc->l) {
			u64 efer = 0;
793

794 795 796 797 798
			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
			if (efer & EFER_LMA)
				mode = X86EMUL_MODE_PROT64;
		} else
			mode = X86EMUL_MODE_PROT32; /* temporary value */
799 800 801 802
	}
#endif
	if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
		mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
803 804 805 806
	rc = assign_eip(ctxt, dst, mode);
	if (rc == X86EMUL_CONTINUE)
		ctxt->mode = mode;
	return rc;
807 808 809 810 811 812
}

static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
{
	return assign_eip_near(ctxt, ctxt->_eip + rel);
}
813

814 815 816 817 818
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
819 820 821
	int rc;
	ulong linear;

822
	rc = linearize(ctxt, addr, size, false, &linear);
823 824
	if (rc != X86EMUL_CONTINUE)
		return rc;
825
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
826 827
}

828 829 830 831 832 833 834 835 836 837 838 839 840 841
static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
			       struct segmented_address addr,
			       void *data,
			       unsigned int size)
{
	int rc;
	ulong linear;

	rc = linearize(ctxt, addr, size, true, &linear);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception);
}

842
/*
843
 * Prefetch the remaining bytes of the instruction without crossing page
844 845
 * boundary if they are not in fetch_cache yet.
 */
846
static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
847 848
{
	int rc;
849
	unsigned size, max_size;
850
	unsigned long linear;
851
	int cur_size = ctxt->fetch.end - ctxt->fetch.data;
852
	struct segmented_address addr = { .seg = VCPU_SREG_CS,
853 854
					   .ea = ctxt->eip + cur_size };

855 856 857 858 859 860 861 862 863 864
	/*
	 * We do not know exactly how many bytes will be needed, and
	 * __linearize is expensive, so fetch as much as possible.  We
	 * just have to avoid going beyond the 15 byte limit, the end
	 * of the segment, or the end of the page.
	 *
	 * __linearize is called with size 0 so that it does not do any
	 * boundary check itself.  Instead, we use max_size to check
	 * against op_size.
	 */
865 866
	rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
			 &linear);
867 868 869
	if (unlikely(rc != X86EMUL_CONTINUE))
		return rc;

870
	size = min_t(unsigned, 15UL ^ cur_size, max_size);
871
	size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
872 873 874 875 876 877 878 879

	/*
	 * One instruction can only straddle two pages,
	 * and one has been loaded at the beginning of
	 * x86_decode_insn.  So, if not enough bytes
	 * still, we must have hit the 15-byte boundary.
	 */
	if (unlikely(size < op_size))
880 881
		return emulate_gp(ctxt, 0);

882
	rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
883 884 885
			      size, &ctxt->exception);
	if (unlikely(rc != X86EMUL_CONTINUE))
		return rc;
886
	ctxt->fetch.end += size;
887
	return X86EMUL_CONTINUE;
888 889
}

890 891
static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
					       unsigned size)
892
{
893 894 895 896
	unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;

	if (unlikely(done_size < size))
		return __do_insn_fetch_bytes(ctxt, size - done_size);
897 898
	else
		return X86EMUL_CONTINUE;
899 900
}

901
/* Fetch next part of the instruction being emulated. */
902
#define insn_fetch(_type, _ctxt)					\
903 904 905
({	_type _x;							\
									\
	rc = do_insn_fetch_bytes(_ctxt, sizeof(_type));			\
906 907
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
908
	ctxt->_eip += sizeof(_type);					\
909
	memcpy(&_x, ctxt->fetch.ptr, sizeof(_type));			\
910
	ctxt->fetch.ptr += sizeof(_type);				\
911
	_x;								\
912 913
})

914
#define insn_fetch_arr(_arr, _size, _ctxt)				\
915 916
({									\
	rc = do_insn_fetch_bytes(_ctxt, _size);				\
917 918
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
919
	ctxt->_eip += (_size);						\
920 921
	memcpy(_arr, ctxt->fetch.ptr, _size);				\
	ctxt->fetch.ptr += (_size);					\
922 923
})

924 925 926 927 928
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
929
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
930
			     int byteop)
A
Avi Kivity 已提交
931 932
{
	void *p;
933
	int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
A
Avi Kivity 已提交
934 935

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
936 937 938
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
A
Avi Kivity 已提交
939 940 941 942
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
943
			   struct segmented_address addr,
A
Avi Kivity 已提交
944 945 946 947 948 949 950
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
951
	rc = segmented_read_std(ctxt, addr, size, 2);
952
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
953
		return rc;
954
	addr.ea += 2;
955
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
956 957 958
	return rc;
}

959 960 961 962 963 964 965 966 967 968
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);

969 970
FASTOP1SRC2(mul, mul_ex);
FASTOP1SRC2(imul, imul_ex);
971 972
FASTOP1SRC2EX(div, div_ex);
FASTOP1SRC2EX(idiv, idiv_ex);
973

974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
FASTOP3WCL(shld);
FASTOP3WCL(shrd);

FASTOP2W(imul);

FASTOP1(not);
FASTOP1(neg);
FASTOP1(inc);
FASTOP1(dec);

FASTOP2CL(rol);
FASTOP2CL(ror);
FASTOP2CL(rcl);
FASTOP2CL(rcr);
FASTOP2CL(shl);
FASTOP2CL(shr);
FASTOP2CL(sar);

FASTOP2W(bsf);
FASTOP2W(bsr);
FASTOP2W(bt);
FASTOP2W(bts);
FASTOP2W(btr);
FASTOP2W(btc);

999 1000
FASTOP2(xadd);

1001 1002
FASTOP2R(cmp, cmp_r);

1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
{
	/* If src is zero, do not writeback, but update flags */
	if (ctxt->src.val == 0)
		ctxt->dst.type = OP_NONE;
	return fastop(ctxt, em_bsf);
}

static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
{
	/* If src is zero, do not writeback, but update flags */
	if (ctxt->src.val == 0)
		ctxt->dst.type = OP_NONE;
	return fastop(ctxt, em_bsr);
}

1019
static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1020
{
1021 1022
	u8 rc;
	void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1023

1024
	flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1025 1026
	asm("push %[flags]; popf; " CALL_NOSPEC
	    : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1027
	return rc;
1028 1029
}

1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

A
Avi Kivity 已提交
1048 1049 1050 1051
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
1052 1053 1054 1055 1056 1057 1058 1059
	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
1060
#ifdef CONFIG_X86_64
1061 1062 1063 1064 1065 1066 1067 1068
	case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
1080 1081 1082 1083 1084 1085 1086 1087
	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
A
Avi Kivity 已提交
1088
#ifdef CONFIG_X86_64
1089 1090 1091 1092 1093 1094 1095 1096
	case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
A
Avi Kivity 已提交
1097 1098 1099 1100 1101 1102
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

A
Avi Kivity 已提交
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
	case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
	case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
	case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
	case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
	case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
	case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
	case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
	case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
	case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
	case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
static int em_fninit(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fninit");
	ctxt->ops->put_fpu(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
{
	u16 fcw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstcw %0": "+m"(fcw));
	ctxt->ops->put_fpu(ctxt);

	ctxt->dst.val = fcw;

	return X86EMUL_CONTINUE;
}

static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
{
	u16 fsw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstsw %0": "+m"(fsw));
	ctxt->ops->put_fpu(ctxt);

	ctxt->dst.val = fsw;

	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1180
static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1181
				    struct operand *op)
1182
{
1183
	unsigned reg = ctxt->modrm_reg;
1184

1185 1186
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
1187

1188
	if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1189 1190 1191 1192 1193 1194
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}
A
Avi Kivity 已提交
1195 1196 1197 1198 1199 1200 1201
	if (ctxt->d & Mmx) {
		reg &= 7;
		op->type = OP_MM;
		op->bytes = 8;
		op->addr.mm = reg;
		return;
	}
A
Avi Kivity 已提交
1202

1203
	op->type = OP_REG;
1204 1205 1206
	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);

1207
	fetch_register_operand(op);
1208 1209 1210
	op->orig_val = op->val;
}

1211 1212 1213 1214 1215 1216
static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
{
	if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
		ctxt->modrm_seg = VCPU_SREG_SS;
}

1217
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1218
			struct operand *op)
1219 1220
{
	u8 sib;
B
Bandan Das 已提交
1221
	int index_reg, base_reg, scale;
1222
	int rc = X86EMUL_CONTINUE;
1223
	ulong modrm_ea = 0;
1224

B
Bandan Das 已提交
1225 1226 1227
	ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
	index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
	base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1228

B
Bandan Das 已提交
1229
	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1230
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
B
Bandan Das 已提交
1231
	ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1232
	ctxt->modrm_seg = VCPU_SREG_DS;
1233

1234
	if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1235
		op->type = OP_REG;
1236
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1237
		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1238
				ctxt->d & ByteOp);
1239
		if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1240 1241
			op->type = OP_XMM;
			op->bytes = 16;
1242 1243
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
A
Avi Kivity 已提交
1244 1245
			return rc;
		}
A
Avi Kivity 已提交
1246 1247 1248
		if (ctxt->d & Mmx) {
			op->type = OP_MM;
			op->bytes = 8;
1249
			op->addr.mm = ctxt->modrm_rm & 7;
A
Avi Kivity 已提交
1250 1251
			return rc;
		}
1252
		fetch_register_operand(op);
1253 1254 1255
		return rc;
	}

1256 1257
	op->type = OP_MEM;

1258
	if (ctxt->ad_bytes == 2) {
1259 1260 1261 1262
		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1263 1264

		/* 16-bit ModR/M decode. */
1265
		switch (ctxt->modrm_mod) {
1266
		case 0:
1267
			if (ctxt->modrm_rm == 6)
1268
				modrm_ea += insn_fetch(u16, ctxt);
1269 1270
			break;
		case 1:
1271
			modrm_ea += insn_fetch(s8, ctxt);
1272 1273
			break;
		case 2:
1274
			modrm_ea += insn_fetch(u16, ctxt);
1275 1276
			break;
		}
1277
		switch (ctxt->modrm_rm) {
1278
		case 0:
1279
			modrm_ea += bx + si;
1280 1281
			break;
		case 1:
1282
			modrm_ea += bx + di;
1283 1284
			break;
		case 2:
1285
			modrm_ea += bp + si;
1286 1287
			break;
		case 3:
1288
			modrm_ea += bp + di;
1289 1290
			break;
		case 4:
1291
			modrm_ea += si;
1292 1293
			break;
		case 5:
1294
			modrm_ea += di;
1295 1296
			break;
		case 6:
1297
			if (ctxt->modrm_mod != 0)
1298
				modrm_ea += bp;
1299 1300
			break;
		case 7:
1301
			modrm_ea += bx;
1302 1303
			break;
		}
1304 1305 1306
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
1307
		modrm_ea = (u16)modrm_ea;
1308 1309
	} else {
		/* 32/64-bit ModR/M decode. */
1310
		if ((ctxt->modrm_rm & 7) == 4) {
1311
			sib = insn_fetch(u8, ctxt);
1312 1313 1314 1315
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

1316
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1317
				modrm_ea += insn_fetch(s32, ctxt);
1318
			else {
1319
				modrm_ea += reg_read(ctxt, base_reg);
1320
				adjust_modrm_seg(ctxt, base_reg);
1321 1322 1323 1324
				/* Increment ESP on POP [ESP] */
				if ((ctxt->d & IncSP) &&
				    base_reg == VCPU_REGS_RSP)
					modrm_ea += ctxt->op_bytes;
1325
			}
1326
			if (index_reg != 4)
1327
				modrm_ea += reg_read(ctxt, index_reg) << scale;
1328
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1329
			modrm_ea += insn_fetch(s32, ctxt);
1330
			if (ctxt->mode == X86EMUL_MODE_PROT64)
1331
				ctxt->rip_relative = 1;
1332 1333
		} else {
			base_reg = ctxt->modrm_rm;
1334
			modrm_ea += reg_read(ctxt, base_reg);
1335 1336
			adjust_modrm_seg(ctxt, base_reg);
		}
1337
		switch (ctxt->modrm_mod) {
1338
		case 1:
1339
			modrm_ea += insn_fetch(s8, ctxt);
1340 1341
			break;
		case 2:
1342
			modrm_ea += insn_fetch(s32, ctxt);
1343 1344 1345
			break;
		}
	}
1346
	op->addr.mem.ea = modrm_ea;
1347 1348 1349
	if (ctxt->ad_bytes != 8)
		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;

1350 1351 1352 1353 1354
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
1355
		      struct operand *op)
1356
{
1357
	int rc = X86EMUL_CONTINUE;
1358

1359
	op->type = OP_MEM;
1360
	switch (ctxt->ad_bytes) {
1361
	case 2:
1362
		op->addr.mem.ea = insn_fetch(u16, ctxt);
1363 1364
		break;
	case 4:
1365
		op->addr.mem.ea = insn_fetch(u32, ctxt);
1366 1367
		break;
	case 8:
1368
		op->addr.mem.ea = insn_fetch(u64, ctxt);
1369 1370 1371 1372 1373 1374
		break;
	}
done:
	return rc;
}

1375
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1376
{
1377
	long sv = 0, mask;
1378

1379
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1380
		mask = ~((long)ctxt->dst.bytes * 8 - 1);
1381

1382 1383 1384 1385
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1386 1387
		else
			sv = (s64)ctxt->src.val & (s64)mask;
1388

1389 1390
		ctxt->dst.addr.mem.ea = address_mask(ctxt,
					   ctxt->dst.addr.mem.ea + (sv >> 3));
1391
	}
1392 1393

	/* only subword offset */
1394
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1395 1396
}

1397 1398
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1399
{
1400
	int rc;
1401
	struct read_cache *mc = &ctxt->mem_read;
A
Avi Kivity 已提交
1402

1403 1404
	if (mc->pos < mc->end)
		goto read_cached;
A
Avi Kivity 已提交
1405

1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
	WARN_ON((mc->end + size) >= sizeof(mc->data));

	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
				      &ctxt->exception);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	mc->end += size;

read_cached:
	memcpy(dest, mc->data + mc->pos, size);
	mc->pos += size;
1418 1419
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1420

1421 1422 1423 1424 1425
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1426 1427 1428
	int rc;
	ulong linear;

1429
	rc = linearize(ctxt, addr, size, false, &linear);
1430 1431
	if (rc != X86EMUL_CONTINUE)
		return rc;
1432
	return read_emulated(ctxt, linear, data, size);
1433 1434 1435 1436 1437 1438 1439
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1440 1441 1442
	int rc;
	ulong linear;

1443
	rc = linearize(ctxt, addr, size, true, &linear);
1444 1445
	if (rc != X86EMUL_CONTINUE)
		return rc;
1446 1447
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1448 1449 1450 1451 1452 1453 1454
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1455 1456 1457
	int rc;
	ulong linear;

1458
	rc = linearize(ctxt, addr, size, true, &linear);
1459 1460
	if (rc != X86EMUL_CONTINUE)
		return rc;
1461 1462
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1463 1464
}

1465 1466 1467 1468
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1469
	struct read_cache *rc = &ctxt->io_read;
1470

1471 1472
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1473
		unsigned int count = ctxt->rep_prefix ?
1474
			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1475
		in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1476 1477
			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1478
		n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1479 1480 1481
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1482
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1483 1484
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1485 1486
	}

1487
	if (ctxt->rep_prefix && (ctxt->d & String) &&
1488
	    !(ctxt->eflags & X86_EFLAGS_DF)) {
1489 1490 1491 1492 1493 1494 1495 1496
		ctxt->dst.data = rc->data + rc->pos;
		ctxt->dst.type = OP_MEM_STR;
		ctxt->dst.count = (rc->end - rc->pos) / size;
		rc->pos = rc->end;
	} else {
		memcpy(dest, rc->data + rc->pos, size);
		rc->pos += size;
	}
1497 1498
	return 1;
}
A
Avi Kivity 已提交
1499

1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
				     u16 index, struct desc_struct *desc)
{
	struct desc_ptr dt;
	ulong addr;

	ctxt->ops->get_idt(ctxt, &dt);

	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, index << 3 | 0x2);

	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
}

1516 1517 1518
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1519
	const struct x86_emulate_ops *ops = ctxt->ops;
1520
	u32 base3 = 0;
1521

1522 1523
	if (selector & 1 << 2) {
		struct desc_struct desc;
1524 1525
		u16 sel;

1526
		memset (dt, 0, sizeof *dt);
1527 1528
		if (!ops->get_segment(ctxt, &sel, &desc, &base3,
				      VCPU_SREG_LDTR))
1529
			return;
1530

1531
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1532
		dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1533
	} else
1534
		ops->get_gdt(ctxt, dt);
1535
}
1536

1537 1538
static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
			      u16 selector, ulong *desc_addr_p)
1539 1540 1541 1542
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1543

1544
	get_descriptor_table_ptr(ctxt, selector, &dt);
1545

1546 1547
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1548

1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576
	addr = dt.address + index * 8;

#ifdef CONFIG_X86_64
	if (addr >> 32 != 0) {
		u64 efer = 0;

		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (!(efer & EFER_LMA))
			addr &= (u32)-1;
	}
#endif

	*desc_addr_p = addr;
	return X86EMUL_CONTINUE;
}

/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, struct desc_struct *desc,
				   ulong *desc_addr_p)
{
	int rc;

	rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1577
				   &ctxt->exception);
1578
}
1579

1580 1581 1582 1583
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
1584
	int rc;
1585
	ulong addr;
A
Avi Kivity 已提交
1586

1587 1588 1589
	rc = get_descriptor_ptr(ctxt, selector, &addr);
	if (rc != X86EMUL_CONTINUE)
		return rc;
A
Avi Kivity 已提交
1590

1591 1592
	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
				    &ctxt->exception);
1593
}
1594

1595
static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1596
				     u16 selector, int seg, u8 cpl,
1597
				     enum x86_transfer_type transfer,
1598
				     struct desc_struct *desc)
1599
{
1600
	struct desc_struct seg_desc, old_desc;
1601
	u8 dpl, rpl;
1602 1603 1604
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1605
	ulong desc_addr;
1606
	int ret;
1607
	u16 dummy;
1608
	u32 base3 = 0;
1609

1610
	memset(&seg_desc, 0, sizeof seg_desc);
1611

1612 1613 1614
	if (ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor (keep limit etc. for
		 * unreal mode) */
1615
		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1616 1617
		set_desc_base(&seg_desc, selector << 4);
		goto load;
1618 1619 1620 1621 1622 1623 1624 1625 1626
	} else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
		/* VM86 needs a clean new segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
		seg_desc.dpl = 3;
		goto load;
1627 1628
	}

1629 1630
	rpl = selector & 3;

1631 1632 1633 1634
	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656
	/* NULL selector is not valid for TR, CS and (except for long mode) SS */
	if (null_selector) {
		if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
			goto exception;

		if (seg == VCPU_SREG_SS) {
			if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
				goto exception;

			/*
			 * ctxt->ops->set_segment expects the CPL to be in
			 * SS.DPL, so fake an expand-up 32-bit data segment.
			 */
			seg_desc.type = 3;
			seg_desc.p = 1;
			seg_desc.s = 1;
			seg_desc.dpl = cpl;
			seg_desc.d = 1;
			seg_desc.g = 1;
		}

		/* Skip all following checks */
1657
		goto load;
1658
	}
1659

1660
	ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1661 1662 1663 1664
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
1665 1666
	err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
							   GP_VECTOR;
1667

G
Guo Chao 已提交
1668
	/* can't load system descriptor into segment selector */
1669 1670 1671
	if (seg <= VCPU_SREG_GS && !seg_desc.s) {
		if (transfer == X86_TRANSFER_CALL_JMP)
			return X86EMUL_UNHANDLEABLE;
1672
		goto exception;
1673
	}
1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	dpl = seg_desc.dpl;

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1690
		break;
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
1704 1705 1706 1707 1708 1709 1710 1711 1712
		/* in long-mode d/b must be clear if l is set */
		if (seg_desc.d && seg_desc.l) {
			u64 efer = 0;

			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
			if (efer & EFER_LMA)
				goto exception;
		}

1713 1714
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1715
		break;
1716 1717 1718
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
1719 1720 1721 1722 1723 1724
		old_desc = seg_desc;
		seg_desc.type |= 2; /* busy */
		ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
						  sizeof(seg_desc), &ctxt->exception);
		if (ret != X86EMUL_CONTINUE)
			return ret;
1725 1726 1727 1728 1729 1730
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1731
		/*
1732 1733 1734
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1735
		 */
1736 1737 1738 1739
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1740
		break;
1741 1742 1743 1744
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
1745 1746 1747 1748 1749 1750 1751
		if (!(seg_desc.type & 1)) {
			seg_desc.type |= 1;
			ret = write_segment_descriptor(ctxt, selector,
						       &seg_desc);
			if (ret != X86EMUL_CONTINUE)
				return ret;
		}
1752 1753 1754 1755 1756
	} else if (ctxt->mode == X86EMUL_MODE_PROT64) {
		ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
				sizeof(base3), &ctxt->exception);
		if (ret != X86EMUL_CONTINUE)
			return ret;
1757 1758
		if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
				((u64)base3 << 32), ctxt))
1759
			return emulate_gp(ctxt, 0);
1760 1761
	}
load:
1762
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1763 1764
	if (desc)
		*desc = seg_desc;
1765 1766
	return X86EMUL_CONTINUE;
exception:
1767
	return emulate_exception(ctxt, err_vec, err_code, true);
1768 1769
}

1770 1771 1772 1773
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
	u8 cpl = ctxt->ops->cpl(ctxt);
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788

	/*
	 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
	 * they can load it at CPL<3 (Intel's manual says only LSS can,
	 * but it's wrong).
	 *
	 * However, the Intel manual says that putting IST=1/DPL=3 in
	 * an interrupt gate will result in SS=3 (the AMD manual instead
	 * says it doesn't), so allow SS=3 in __load_segment_descriptor
	 * and only forbid it here.
	 */
	if (seg == VCPU_SREG_SS && selector == 3 &&
	    ctxt->mode == X86EMUL_MODE_PROT64)
		return emulate_exception(ctxt, GP_VECTOR, 0, true);

1789 1790
	return __load_segment_descriptor(ctxt, selector, seg, cpl,
					 X86_TRANSFER_NONE, NULL);
1791 1792
}

1793 1794
static void write_register_operand(struct operand *op)
{
1795
	return assign_register(op->addr.reg, op->val, op->bytes);
1796 1797
}

1798
static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1799
{
1800
	switch (op->type) {
1801
	case OP_REG:
1802
		write_register_operand(op);
A
Avi Kivity 已提交
1803
		break;
1804
	case OP_MEM:
1805
		if (ctxt->lock_prefix)
P
Paolo Bonzini 已提交
1806 1807 1808 1809 1810 1811 1812
			return segmented_cmpxchg(ctxt,
						 op->addr.mem,
						 &op->orig_val,
						 &op->val,
						 op->bytes);
		else
			return segmented_write(ctxt,
1813 1814 1815
					       op->addr.mem,
					       &op->val,
					       op->bytes);
1816
		break;
1817
	case OP_MEM_STR:
P
Paolo Bonzini 已提交
1818 1819 1820 1821
		return segmented_write(ctxt,
				       op->addr.mem,
				       op->data,
				       op->bytes * op->count);
1822
		break;
A
Avi Kivity 已提交
1823
	case OP_XMM:
1824
		write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
A
Avi Kivity 已提交
1825
		break;
A
Avi Kivity 已提交
1826
	case OP_MM:
1827
		write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
A
Avi Kivity 已提交
1828
		break;
1829 1830
	case OP_NONE:
		/* no writeback */
1831
		break;
1832
	default:
1833
		break;
A
Avi Kivity 已提交
1834
	}
1835 1836
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1837

1838
static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1839
{
1840
	struct segmented_address addr;
1841

1842
	rsp_increment(ctxt, -bytes);
1843
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1844 1845
	addr.seg = VCPU_SREG_SS;

1846 1847 1848 1849 1850
	return segmented_write(ctxt, addr, data, bytes);
}

static int em_push(struct x86_emulate_ctxt *ctxt)
{
1851
	/* Disable writeback. */
1852
	ctxt->dst.type = OP_NONE;
1853
	return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1854
}
1855

1856 1857 1858 1859
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1860
	struct segmented_address addr;
1861

1862
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1863
	addr.seg = VCPU_SREG_SS;
1864
	rc = segmented_read(ctxt, addr, dest, len);
1865 1866 1867
	if (rc != X86EMUL_CONTINUE)
		return rc;

1868
	rsp_increment(ctxt, len);
1869
	return rc;
1870 1871
}

1872 1873
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1874
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1875 1876
}

1877
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1878
			void *dest, int len)
1879 1880
{
	int rc;
1881
	unsigned long val, change_mask;
1882
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1883
	int cpl = ctxt->ops->cpl(ctxt);
1884

1885
	rc = emulate_pop(ctxt, &val, len);
1886 1887
	if (rc != X86EMUL_CONTINUE)
		return rc;
1888

1889 1890 1891 1892
	change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
		      X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
		      X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
		      X86_EFLAGS_AC | X86_EFLAGS_ID;
1893

1894 1895 1896 1897 1898
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
1899
			change_mask |= X86_EFLAGS_IOPL;
1900
		if (cpl <= iopl)
1901
			change_mask |= X86_EFLAGS_IF;
1902 1903
		break;
	case X86EMUL_MODE_VM86:
1904 1905
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1906
		change_mask |= X86_EFLAGS_IF;
1907 1908
		break;
	default: /* real mode */
1909
		change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1910
		break;
1911
	}
1912 1913 1914 1915 1916

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1917 1918
}

1919 1920
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1921 1922 1923 1924
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1925 1926
}

A
Avi Kivity 已提交
1927 1928 1929 1930 1931
static int em_enter(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned frame_size = ctxt->src.val;
	unsigned nesting_level = ctxt->src2.val & 31;
1932
	ulong rbp;
A
Avi Kivity 已提交
1933 1934 1935 1936

	if (nesting_level)
		return X86EMUL_UNHANDLEABLE;

1937 1938
	rbp = reg_read(ctxt, VCPU_REGS_RBP);
	rc = push(ctxt, &rbp, stack_size(ctxt));
A
Avi Kivity 已提交
1939 1940
	if (rc != X86EMUL_CONTINUE)
		return rc;
1941
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
A
Avi Kivity 已提交
1942
		      stack_mask(ctxt));
1943 1944
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
A
Avi Kivity 已提交
1945 1946 1947 1948
		      stack_mask(ctxt));
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1949 1950
static int em_leave(struct x86_emulate_ctxt *ctxt)
{
1951
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
A
Avi Kivity 已提交
1952
		      stack_mask(ctxt));
1953
	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
A
Avi Kivity 已提交
1954 1955
}

1956
static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1957
{
1958 1959
	int seg = ctxt->src2.val;

1960
	ctxt->src.val = get_segment_selector(ctxt, seg);
1961 1962 1963 1964
	if (ctxt->op_bytes == 4) {
		rsp_increment(ctxt, -2);
		ctxt->op_bytes = 2;
	}
1965

1966
	return em_push(ctxt);
1967 1968
}

1969
static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1970
{
1971
	int seg = ctxt->src2.val;
1972 1973
	unsigned long selector;
	int rc;
1974

1975
	rc = emulate_pop(ctxt, &selector, 2);
1976 1977 1978
	if (rc != X86EMUL_CONTINUE)
		return rc;

1979 1980
	if (ctxt->modrm_reg == VCPU_SREG_SS)
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1981 1982
	if (ctxt->op_bytes > 2)
		rsp_increment(ctxt, ctxt->op_bytes - 2);
1983

1984
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1985
	return rc;
1986 1987
}

1988
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1989
{
1990
	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1991 1992
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1993

1994 1995
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
1996
		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1997

1998
		rc = em_push(ctxt);
1999 2000
		if (rc != X86EMUL_CONTINUE)
			return rc;
2001

2002
		++reg;
2003 2004
	}

2005
	return rc;
2006 2007
}

2008 2009
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
2010
	ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2011 2012 2013
	return em_push(ctxt);
}

2014
static int em_popa(struct x86_emulate_ctxt *ctxt)
2015
{
2016 2017
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
2018
	u32 val;
2019

2020 2021
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
2022
			rsp_increment(ctxt, ctxt->op_bytes);
2023 2024
			--reg;
		}
2025

2026
		rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2027 2028
		if (rc != X86EMUL_CONTINUE)
			break;
2029
		assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2030
		--reg;
2031
	}
2032
	return rc;
2033 2034
}

2035
static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2036
{
2037
	const struct x86_emulate_ops *ops = ctxt->ops;
2038
	int rc;
2039 2040 2041 2042 2043 2044
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
2045
	ctxt->src.val = ctxt->eflags;
2046
	rc = em_push(ctxt);
2047 2048
	if (rc != X86EMUL_CONTINUE)
		return rc;
2049

2050
	ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2051

2052
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2053
	rc = em_push(ctxt);
2054 2055
	if (rc != X86EMUL_CONTINUE)
		return rc;
2056

2057
	ctxt->src.val = ctxt->_eip;
2058
	rc = em_push(ctxt);
2059 2060 2061
	if (rc != X86EMUL_CONTINUE)
		return rc;

2062
	ops->get_idt(ctxt, &dt);
2063 2064 2065 2066

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

2067
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
2068 2069 2070
	if (rc != X86EMUL_CONTINUE)
		return rc;

2071
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
2072 2073 2074
	if (rc != X86EMUL_CONTINUE)
		return rc;

2075
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2076 2077 2078
	if (rc != X86EMUL_CONTINUE)
		return rc;

2079
	ctxt->_eip = eip;
2080 2081 2082 2083

	return rc;
}

2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
{
	int rc;

	invalidate_registers(ctxt);
	rc = __emulate_int_real(ctxt, irq);
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);
	return rc;
}

2095
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2096 2097 2098
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
2099
		return __emulate_int_real(ctxt, irq);
2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

2110
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2111
{
2112 2113 2114 2115
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
2116 2117 2118 2119 2120
	unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
			     X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
			     X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
			     X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
			     X86_EFLAGS_AC | X86_EFLAGS_ID |
W
Wanpeng Li 已提交
2121
			     X86_EFLAGS_FIXED;
2122 2123
	unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
				  X86_EFLAGS_VIP;
2124

2125
	/* TODO: Add stack limit check */
2126

2127
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2128

2129 2130
	if (rc != X86EMUL_CONTINUE)
		return rc;
2131

2132 2133
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
2134

2135
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2136

2137 2138
	if (rc != X86EMUL_CONTINUE)
		return rc;
2139

2140
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2141

2142 2143
	if (rc != X86EMUL_CONTINUE)
		return rc;
2144

2145
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2146

2147 2148
	if (rc != X86EMUL_CONTINUE)
		return rc;
2149

2150
	ctxt->_eip = temp_eip;
2151

2152
	if (ctxt->op_bytes == 4)
2153
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2154
	else if (ctxt->op_bytes == 2) {
2155 2156
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
2157
	}
2158 2159

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
W
Wanpeng Li 已提交
2160
	ctxt->eflags |= X86_EFLAGS_FIXED;
2161
	ctxt->ops->set_nmi_mask(ctxt, false);
2162 2163

	return rc;
2164 2165
}

2166
static int em_iret(struct x86_emulate_ctxt *ctxt)
2167
{
2168 2169
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
2170
		return emulate_iret_real(ctxt);
2171 2172 2173 2174
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
2175
	default:
2176 2177
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
2178 2179 2180
	}
}

2181 2182 2183
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
2184 2185
	unsigned short sel;
	struct desc_struct new_desc;
2186 2187
	u8 cpl = ctxt->ops->cpl(ctxt);

2188
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2189

2190 2191
	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
				       X86_TRANSFER_CALL_JMP,
2192
				       &new_desc);
2193 2194 2195
	if (rc != X86EMUL_CONTINUE)
		return rc;

2196
	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2197 2198 2199 2200
	/* Error handling is not implemented. */
	if (rc != X86EMUL_CONTINUE)
		return X86EMUL_UNHANDLEABLE;

2201
	return rc;
2202 2203
}

2204
static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2205
{
2206 2207
	return assign_eip_near(ctxt, ctxt->src.val);
}
2208

2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219
static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	long int old_eip;

	old_eip = ctxt->_eip;
	rc = assign_eip_near(ctxt, ctxt->src.val);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->src.val = old_eip;
	rc = em_push(ctxt);
2220
	return rc;
2221 2222
}

2223
static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2224
{
2225
	u64 old = ctxt->dst.orig_val64;
2226

2227 2228 2229
	if (ctxt->dst.bytes == 16)
		return X86EMUL_UNHANDLEABLE;

2230 2231 2232 2233
	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2234
		ctxt->eflags &= ~X86_EFLAGS_ZF;
2235
	} else {
2236 2237
		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
			(u32) reg_read(ctxt, VCPU_REGS_RBX);
2238

2239
		ctxt->eflags |= X86_EFLAGS_ZF;
2240
	}
2241
	return X86EMUL_CONTINUE;
2242 2243
}

2244 2245
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
2246 2247 2248 2249 2250 2251 2252 2253
	int rc;
	unsigned long eip;

	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	return assign_eip_near(ctxt, eip);
2254 2255
}

2256
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2257 2258
{
	int rc;
2259
	unsigned long eip, cs;
2260
	int cpl = ctxt->ops->cpl(ctxt);
2261
	struct desc_struct new_desc;
2262

2263
	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2264
	if (rc != X86EMUL_CONTINUE)
2265
		return rc;
2266
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2267
	if (rc != X86EMUL_CONTINUE)
2268
		return rc;
2269 2270 2271
	/* Outer-privilege level return is not implemented */
	if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
		return X86EMUL_UNHANDLEABLE;
2272 2273
	rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
				       X86_TRANSFER_RET,
2274 2275 2276
				       &new_desc);
	if (rc != X86EMUL_CONTINUE)
		return rc;
2277
	rc = assign_eip_far(ctxt, eip, &new_desc);
2278 2279 2280 2281
	/* Error handling is not implemented. */
	if (rc != X86EMUL_CONTINUE)
		return X86EMUL_UNHANDLEABLE;

2282 2283 2284
	return rc;
}

2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295
static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
{
        int rc;

        rc = em_ret_far(ctxt);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        rsp_increment(ctxt, ctxt->src.val);
        return X86EMUL_CONTINUE;
}

2296 2297 2298
static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
{
	/* Save real source value, then compare EAX against destination. */
2299 2300
	ctxt->dst.orig_val = ctxt->dst.val;
	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2301
	ctxt->src.orig_val = ctxt->src.val;
2302
	ctxt->src.val = ctxt->dst.orig_val;
2303
	fastop(ctxt, em_cmp);
2304

2305
	if (ctxt->eflags & X86_EFLAGS_ZF) {
2306 2307
		/* Success: write back to memory; no update of EAX */
		ctxt->src.type = OP_NONE;
2308 2309 2310
		ctxt->dst.val = ctxt->src.orig_val;
	} else {
		/* Failure: write the value we saw to EAX. */
2311 2312 2313 2314
		ctxt->src.type = OP_REG;
		ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
		ctxt->src.val = ctxt->dst.orig_val;
		/* Create write-cycle to dest by writing the same value */
2315
		ctxt->dst.val = ctxt->dst.orig_val;
2316 2317 2318 2319
	}
	return X86EMUL_CONTINUE;
}

2320
static int em_lseg(struct x86_emulate_ctxt *ctxt)
2321
{
2322
	int seg = ctxt->src2.val;
2323 2324 2325
	unsigned short sel;
	int rc;

2326
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2327

2328
	rc = load_segment_descriptor(ctxt, sel, seg);
2329 2330 2331
	if (rc != X86EMUL_CONTINUE)
		return rc;

2332
	ctxt->dst.val = ctxt->src.val;
2333 2334 2335
	return rc;
}

2336 2337 2338 2339 2340 2341
static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

	eax = 0x80000001;
	ecx = 0;
2342
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2343 2344 2345 2346 2347 2348
	return edx & bit(X86_FEATURE_LM);
}

#define GET_SMSTATE(type, smbase, offset)				  \
	({								  \
	 type __val;							  \
2349 2350
	 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val,      \
				      sizeof(__val));			  \
2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494
	 if (r != X86EMUL_CONTINUE)					  \
		 return X86EMUL_UNHANDLEABLE;				  \
	 __val;								  \
	})

static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
{
	desc->g    = (flags >> 23) & 1;
	desc->d    = (flags >> 22) & 1;
	desc->l    = (flags >> 21) & 1;
	desc->avl  = (flags >> 20) & 1;
	desc->p    = (flags >> 15) & 1;
	desc->dpl  = (flags >> 13) & 3;
	desc->s    = (flags >> 12) & 1;
	desc->type = (flags >>  8) & 15;
}

static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
{
	struct desc_struct desc;
	int offset;
	u16 selector;

	selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);

	if (n < 3)
		offset = 0x7f84 + n * 12;
	else
		offset = 0x7f2c + (n - 3) * 12;

	set_desc_base(&desc,      GET_SMSTATE(u32, smbase, offset + 8));
	set_desc_limit(&desc,     GET_SMSTATE(u32, smbase, offset + 4));
	rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
	ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
	return X86EMUL_CONTINUE;
}

static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
{
	struct desc_struct desc;
	int offset;
	u16 selector;
	u32 base3;

	offset = 0x7e00 + n * 16;

	selector =                GET_SMSTATE(u16, smbase, offset);
	rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
	set_desc_limit(&desc,     GET_SMSTATE(u32, smbase, offset + 4));
	set_desc_base(&desc,      GET_SMSTATE(u32, smbase, offset + 8));
	base3 =                   GET_SMSTATE(u32, smbase, offset + 12);

	ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
	return X86EMUL_CONTINUE;
}

static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
				     u64 cr0, u64 cr4)
{
	int bad;

	/*
	 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
	 * Then enable protected mode.	However, PCID cannot be enabled
	 * if EFER.LMA=0, so set it separately.
	 */
	bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
	if (bad)
		return X86EMUL_UNHANDLEABLE;

	bad = ctxt->ops->set_cr(ctxt, 0, cr0);
	if (bad)
		return X86EMUL_UNHANDLEABLE;

	if (cr4 & X86_CR4_PCIDE) {
		bad = ctxt->ops->set_cr(ctxt, 4, cr4);
		if (bad)
			return X86EMUL_UNHANDLEABLE;
	}

	return X86EMUL_CONTINUE;
}

static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
{
	struct desc_struct desc;
	struct desc_ptr dt;
	u16 selector;
	u32 val, cr0, cr4;
	int i;

	cr0 =                      GET_SMSTATE(u32, smbase, 0x7ffc);
	ctxt->ops->set_cr(ctxt, 3, GET_SMSTATE(u32, smbase, 0x7ff8));
	ctxt->eflags =             GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
	ctxt->_eip =               GET_SMSTATE(u32, smbase, 0x7ff0);

	for (i = 0; i < 8; i++)
		*reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);

	val = GET_SMSTATE(u32, smbase, 0x7fcc);
	ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
	val = GET_SMSTATE(u32, smbase, 0x7fc8);
	ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);

	selector =                 GET_SMSTATE(u32, smbase, 0x7fc4);
	set_desc_base(&desc,       GET_SMSTATE(u32, smbase, 0x7f64));
	set_desc_limit(&desc,      GET_SMSTATE(u32, smbase, 0x7f60));
	rsm_set_desc_flags(&desc,  GET_SMSTATE(u32, smbase, 0x7f5c));
	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);

	selector =                 GET_SMSTATE(u32, smbase, 0x7fc0);
	set_desc_base(&desc,       GET_SMSTATE(u32, smbase, 0x7f80));
	set_desc_limit(&desc,      GET_SMSTATE(u32, smbase, 0x7f7c));
	rsm_set_desc_flags(&desc,  GET_SMSTATE(u32, smbase, 0x7f78));
	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);

	dt.address =               GET_SMSTATE(u32, smbase, 0x7f74);
	dt.size =                  GET_SMSTATE(u32, smbase, 0x7f70);
	ctxt->ops->set_gdt(ctxt, &dt);

	dt.address =               GET_SMSTATE(u32, smbase, 0x7f58);
	dt.size =                  GET_SMSTATE(u32, smbase, 0x7f54);
	ctxt->ops->set_idt(ctxt, &dt);

	for (i = 0; i < 6; i++) {
		int r = rsm_load_seg_32(ctxt, smbase, i);
		if (r != X86EMUL_CONTINUE)
			return r;
	}

	cr4 = GET_SMSTATE(u32, smbase, 0x7f14);

	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));

	return rsm_enter_protected_mode(ctxt, cr0, cr4);
}

static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
{
	struct desc_struct desc;
	struct desc_ptr dt;
	u64 val, cr0, cr4;
	u32 base3;
	u16 selector;
2495
	int i, r;
2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536

	for (i = 0; i < 16; i++)
		*reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);

	ctxt->_eip   = GET_SMSTATE(u64, smbase, 0x7f78);
	ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;

	val = GET_SMSTATE(u32, smbase, 0x7f68);
	ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
	val = GET_SMSTATE(u32, smbase, 0x7f60);
	ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);

	cr0 =                       GET_SMSTATE(u64, smbase, 0x7f58);
	ctxt->ops->set_cr(ctxt, 3,  GET_SMSTATE(u64, smbase, 0x7f50));
	cr4 =                       GET_SMSTATE(u64, smbase, 0x7f48);
	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
	val =                       GET_SMSTATE(u64, smbase, 0x7ed0);
	ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);

	selector =                  GET_SMSTATE(u32, smbase, 0x7e90);
	rsm_set_desc_flags(&desc,   GET_SMSTATE(u32, smbase, 0x7e92) << 8);
	set_desc_limit(&desc,       GET_SMSTATE(u32, smbase, 0x7e94));
	set_desc_base(&desc,        GET_SMSTATE(u32, smbase, 0x7e98));
	base3 =                     GET_SMSTATE(u32, smbase, 0x7e9c);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);

	dt.size =                   GET_SMSTATE(u32, smbase, 0x7e84);
	dt.address =                GET_SMSTATE(u64, smbase, 0x7e88);
	ctxt->ops->set_idt(ctxt, &dt);

	selector =                  GET_SMSTATE(u32, smbase, 0x7e70);
	rsm_set_desc_flags(&desc,   GET_SMSTATE(u32, smbase, 0x7e72) << 8);
	set_desc_limit(&desc,       GET_SMSTATE(u32, smbase, 0x7e74));
	set_desc_base(&desc,        GET_SMSTATE(u32, smbase, 0x7e78));
	base3 =                     GET_SMSTATE(u32, smbase, 0x7e7c);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);

	dt.size =                   GET_SMSTATE(u32, smbase, 0x7e64);
	dt.address =                GET_SMSTATE(u64, smbase, 0x7e68);
	ctxt->ops->set_gdt(ctxt, &dt);

2537 2538 2539 2540
	r = rsm_enter_protected_mode(ctxt, cr0, cr4);
	if (r != X86EMUL_CONTINUE)
		return r;

2541
	for (i = 0; i < 6; i++) {
2542
		r = rsm_load_seg_64(ctxt, smbase, i);
2543 2544 2545 2546
		if (r != X86EMUL_CONTINUE)
			return r;
	}

2547
	return X86EMUL_CONTINUE;
2548 2549
}

P
Paolo Bonzini 已提交
2550 2551
static int em_rsm(struct x86_emulate_ctxt *ctxt)
{
2552 2553 2554 2555
	unsigned long cr0, cr4, efer;
	u64 smbase;
	int ret;

2556
	if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
P
Paolo Bonzini 已提交
2557 2558
		return emulate_ud(ctxt);

2559 2560
	/*
	 * Get back to real mode, to prepare a safe state in which to load
2561 2562
	 * CR0/CR3/CR4/EFER.  It's all a bit more complicated if the vCPU
	 * supports long mode.
2563
	 */
2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581
	cr4 = ctxt->ops->get_cr(ctxt, 4);
	if (emulator_has_longmode(ctxt)) {
		struct desc_struct cs_desc;

		/* Zero CR4.PCIDE before CR0.PG.  */
		if (cr4 & X86_CR4_PCIDE) {
			ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
			cr4 &= ~X86_CR4_PCIDE;
		}

		/* A 32-bit code segment is required to clear EFER.LMA.  */
		memset(&cs_desc, 0, sizeof(cs_desc));
		cs_desc.type = 0xb;
		cs_desc.s = cs_desc.g = cs_desc.p = 1;
		ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
	}

	/* For the 64-bit case, this will clear EFER.LMA.  */
2582 2583 2584
	cr0 = ctxt->ops->get_cr(ctxt, 0);
	if (cr0 & X86_CR0_PE)
		ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2585 2586

	/* Now clear CR4.PAE (which must be done before clearing EFER.LME).  */
2587 2588
	if (cr4 & X86_CR4_PAE)
		ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2589 2590

	/* And finally go back to 32-bit mode.  */
2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604
	efer = 0;
	ctxt->ops->set_msr(ctxt, MSR_EFER, efer);

	smbase = ctxt->ops->get_smbase(ctxt);
	if (emulator_has_longmode(ctxt))
		ret = rsm_load_state_64(ctxt, smbase + 0x8000);
	else
		ret = rsm_load_state_32(ctxt, smbase + 0x8000);

	if (ret != X86EMUL_CONTINUE) {
		/* FIXME: should triple fault */
		return X86EMUL_UNHANDLEABLE;
	}

2605
	if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2606 2607
		ctxt->ops->set_nmi_mask(ctxt, false);

2608 2609
	ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
		~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2610
	return X86EMUL_CONTINUE;
P
Paolo Bonzini 已提交
2611 2612
}

2613
static void
2614
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2615
			struct desc_struct *cs, struct desc_struct *ss)
2616 2617
{
	cs->l = 0;		/* will be adjusted later */
2618
	set_desc_base(cs, 0);	/* flat segment */
2619
	cs->g = 1;		/* 4kb granularity */
2620
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
2621 2622 2623
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
2624 2625
	cs->p = 1;
	cs->d = 1;
2626
	cs->avl = 0;
2627

2628 2629
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
2630 2631 2632
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
2633
	ss->d = 1;		/* 32bit stack segment */
2634
	ss->dpl = 0;
2635
	ss->p = 1;
2636 2637
	ss->l = 0;
	ss->avl = 0;
2638 2639
}

2640 2641 2642 2643 2644
static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

	eax = ecx = 0;
2645
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2646
	return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2647 2648 2649 2650
		&& ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
		&& edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
}

2651 2652
static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
{
2653
	const struct x86_emulate_ops *ops = ctxt->ops;
2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664
	u32 eax, ebx, ecx, edx;

	/*
	 * syscall should always be enabled in longmode - so only become
	 * vendor specific (cpuid) if other modes are active...
	 */
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return true;

	eax = 0x00000000;
	ecx = 0x00000000;
2665
	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689
	/*
	 * Intel ("GenuineIntel")
	 * remark: Intel CPUs only support "syscall" in 64bit
	 * longmode. Also an 64bit guest with a
	 * 32bit compat-app running will #UD !! While this
	 * behaviour can be fixed (by emulating) into AMD
	 * response - CPUs of AMD can't behave like Intel.
	 */
	if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
		return false;

	/* AMD ("AuthenticAMD") */
	if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
		return true;

	/* AMD ("AMDisbetter!") */
	if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
		return true;
2690 2691 2692 2693 2694

	/* default: (not Intel, not AMD), apply Intel's stricter rules... */
	return false;
}

2695
static int em_syscall(struct x86_emulate_ctxt *ctxt)
2696
{
2697
	const struct x86_emulate_ops *ops = ctxt->ops;
2698
	struct desc_struct cs, ss;
2699
	u64 msr_data;
2700
	u16 cs_sel, ss_sel;
2701
	u64 efer = 0;
2702 2703

	/* syscall is not available in real mode */
2704
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2705 2706
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
2707

2708 2709 2710
	if (!(em_syscall_is_enabled(ctxt)))
		return emulate_ud(ctxt);

2711
	ops->get_msr(ctxt, MSR_EFER, &efer);
2712
	setup_syscalls_segments(ctxt, &cs, &ss);
2713

2714 2715 2716
	if (!(efer & EFER_SCE))
		return emulate_ud(ctxt);

2717
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
2718
	msr_data >>= 32;
2719 2720
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
2721

2722
	if (efer & EFER_LMA) {
2723
		cs.d = 0;
2724 2725
		cs.l = 1;
	}
2726 2727
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2728

2729
	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2730
	if (efer & EFER_LMA) {
2731
#ifdef CONFIG_X86_64
2732
		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2733

2734
		ops->get_msr(ctxt,
2735 2736
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
2737
		ctxt->_eip = msr_data;
2738

2739
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2740
		ctxt->eflags &= ~msr_data;
W
Wanpeng Li 已提交
2741
		ctxt->eflags |= X86_EFLAGS_FIXED;
2742 2743 2744
#endif
	} else {
		/* legacy mode */
2745
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
2746
		ctxt->_eip = (u32)msr_data;
2747

2748
		ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2749 2750
	}

2751
	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2752
	return X86EMUL_CONTINUE;
2753 2754
}

2755
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2756
{
2757
	const struct x86_emulate_ops *ops = ctxt->ops;
2758
	struct desc_struct cs, ss;
2759
	u64 msr_data;
2760
	u16 cs_sel, ss_sel;
2761
	u64 efer = 0;
2762

2763
	ops->get_msr(ctxt, MSR_EFER, &efer);
2764
	/* inject #GP if in real mode */
2765 2766
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
2767

2768 2769 2770 2771
	/*
	 * Not recognized on AMD in compat mode (but is recognized in legacy
	 * mode).
	 */
2772
	if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2773 2774 2775
	    && !vendor_intel(ctxt))
		return emulate_ud(ctxt);

2776
	/* sysenter/sysexit have not been tested in 64bit mode. */
2777
	if (ctxt->mode == X86EMUL_MODE_PROT64)
2778
		return X86EMUL_UNHANDLEABLE;
2779

2780
	setup_syscalls_segments(ctxt, &cs, &ss);
2781

2782
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2783 2784
	if ((msr_data & 0xfffc) == 0x0)
		return emulate_gp(ctxt, 0);
2785

2786
	ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2787
	cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2788
	ss_sel = cs_sel + 8;
2789
	if (efer & EFER_LMA) {
2790
		cs.d = 0;
2791 2792 2793
		cs.l = 1;
	}

2794 2795
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2796

2797
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2798
	ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2799

2800
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2801 2802
	*reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
							      (u32)msr_data;
2803

2804
	return X86EMUL_CONTINUE;
2805 2806
}

2807
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2808
{
2809
	const struct x86_emulate_ops *ops = ctxt->ops;
2810
	struct desc_struct cs, ss;
2811
	u64 msr_data, rcx, rdx;
2812
	int usermode;
X
Xiao Guangrong 已提交
2813
	u16 cs_sel = 0, ss_sel = 0;
2814

2815 2816
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2817 2818
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
2819

2820
	setup_syscalls_segments(ctxt, &cs, &ss);
2821

2822
	if ((ctxt->rex_prefix & 0x8) != 0x0)
2823 2824 2825 2826
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

2827 2828 2829
	rcx = reg_read(ctxt, VCPU_REGS_RCX);
	rdx = reg_read(ctxt, VCPU_REGS_RDX);

2830 2831
	cs.dpl = 3;
	ss.dpl = 3;
2832
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2833 2834
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
2835
		cs_sel = (u16)(msr_data + 16);
2836 2837
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2838
		ss_sel = (u16)(msr_data + 24);
2839 2840
		rcx = (u32)rcx;
		rdx = (u32)rdx;
2841 2842
		break;
	case X86EMUL_MODE_PROT64:
2843
		cs_sel = (u16)(msr_data + 32);
2844 2845
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2846 2847
		ss_sel = cs_sel + 8;
		cs.d = 0;
2848
		cs.l = 1;
2849 2850
		if (emul_is_noncanonical_address(rcx, ctxt) ||
		    emul_is_noncanonical_address(rdx, ctxt))
2851
			return emulate_gp(ctxt, 0);
2852 2853
		break;
	}
2854 2855
	cs_sel |= SEGMENT_RPL_MASK;
	ss_sel |= SEGMENT_RPL_MASK;
2856

2857 2858
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2859

2860 2861
	ctxt->_eip = rdx;
	*reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2862

2863
	return X86EMUL_CONTINUE;
2864 2865
}

2866
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2867 2868 2869 2870 2871 2872
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
2873
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2874
	return ctxt->ops->cpl(ctxt) > iopl;
2875 2876 2877 2878 2879
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2880
	const struct x86_emulate_ops *ops = ctxt->ops;
2881
	struct desc_struct tr_seg;
2882
	u32 base3;
2883
	int r;
2884
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2885
	unsigned mask = (1 << len) - 1;
2886
	unsigned long base;
2887

2888
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2889
	if (!tr_seg.p)
2890
		return false;
2891
	if (desc_limit_scaled(&tr_seg) < 103)
2892
		return false;
2893 2894 2895 2896
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2897
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2898 2899
	if (r != X86EMUL_CONTINUE)
		return false;
2900
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2901
		return false;
2902
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2903 2904 2905 2906 2907 2908 2909 2910 2911 2912
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2913 2914 2915
	if (ctxt->perm_ok)
		return true;

2916 2917
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2918
			return false;
2919 2920 2921

	ctxt->perm_ok = true;

2922 2923 2924
	return true;
}

2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948
static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
{
	/*
	 * Intel CPUs mask the counter and pointers in quite strange
	 * manner when ECX is zero due to REP-string optimizations.
	 */
#ifdef CONFIG_X86_64
	if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
		return;

	*reg_write(ctxt, VCPU_REGS_RCX) = 0;

	switch (ctxt->b) {
	case 0xa4:	/* movsb */
	case 0xa5:	/* movsd/w */
		*reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
		/* fall through */
	case 0xaa:	/* stosb */
	case 0xab:	/* stosd/w */
		*reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
	}
#endif
}

2949 2950 2951
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
2952
	tss->ip = ctxt->_eip;
2953
	tss->flag = ctxt->eflags;
2954 2955 2956 2957 2958 2959 2960 2961
	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->si = reg_read(ctxt, VCPU_REGS_RSI);
	tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2962

2963 2964 2965 2966 2967
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2968 2969 2970 2971 2972 2973
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;
2974
	u8 cpl;
2975

2976
	ctxt->_eip = tss->ip;
2977
	ctxt->eflags = tss->flag | 2;
2978 2979 2980 2981 2982 2983 2984 2985
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2986 2987 2988 2989 2990

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2991 2992 2993 2994 2995
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2996

2997 2998
	cpl = tss->cs & 3;

2999
	/*
G
Guo Chao 已提交
3000
	 * Now load segment descriptors. If fault happens at this stage
3001 3002
	 * it is handled in a context of new task
	 */
3003
	ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3004
					X86_TRANSFER_TASK_SWITCH, NULL);
3005 3006
	if (ret != X86EMUL_CONTINUE)
		return ret;
3007
	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3008
					X86_TRANSFER_TASK_SWITCH, NULL);
3009 3010
	if (ret != X86EMUL_CONTINUE)
		return ret;
3011
	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3012
					X86_TRANSFER_TASK_SWITCH, NULL);
3013 3014
	if (ret != X86EMUL_CONTINUE)
		return ret;
3015
	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3016
					X86_TRANSFER_TASK_SWITCH, NULL);
3017 3018
	if (ret != X86EMUL_CONTINUE)
		return ret;
3019
	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3020
					X86_TRANSFER_TASK_SWITCH, NULL);
3021 3022 3023 3024 3025 3026 3027 3028 3029 3030
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
3031
	const struct x86_emulate_ops *ops = ctxt->ops;
3032 3033
	struct tss_segment_16 tss_seg;
	int ret;
3034
	u32 new_tss_base = get_desc_base(new_desc);
3035

3036
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
3037
			    &ctxt->exception);
3038
	if (ret != X86EMUL_CONTINUE)
3039 3040
		return ret;

3041
	save_state_to_tss16(ctxt, &tss_seg);
3042

3043
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
3044
			     &ctxt->exception);
3045
	if (ret != X86EMUL_CONTINUE)
3046 3047
		return ret;

3048
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
3049
			    &ctxt->exception);
3050
	if (ret != X86EMUL_CONTINUE)
3051 3052 3053 3054 3055
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

3056
		ret = ops->write_std(ctxt, new_tss_base,
3057 3058
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
3059
				     &ctxt->exception);
3060
		if (ret != X86EMUL_CONTINUE)
3061 3062 3063
			return ret;
	}

3064
	return load_state_from_tss16(ctxt, &tss_seg);
3065 3066 3067 3068 3069
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
3070
	/* CR3 and ldt selector are not saved intentionally */
3071
	tss->eip = ctxt->_eip;
3072
	tss->eflags = ctxt->eflags;
3073 3074 3075 3076 3077 3078 3079 3080
	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3081

3082 3083 3084 3085 3086 3087
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3088 3089 3090 3091 3092 3093
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;
3094
	u8 cpl;
3095

3096
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3097
		return emulate_gp(ctxt, 0);
3098
	ctxt->_eip = tss->eip;
3099
	ctxt->eflags = tss->eflags | 2;
3100 3101

	/* General purpose registers */
3102 3103 3104 3105 3106 3107 3108 3109
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3110 3111 3112

	/*
	 * SDM says that segment selectors are loaded before segment
3113 3114
	 * descriptors.  This is important because CPL checks will
	 * use CS.RPL.
3115
	 */
3116 3117 3118 3119 3120 3121 3122
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3123

3124 3125 3126 3127 3128
	/*
	 * If we're switching between Protected Mode and VM86, we need to make
	 * sure to update the mode before loading the segment descriptors so
	 * that the selectors are interpreted correctly.
	 */
3129
	if (ctxt->eflags & X86_EFLAGS_VM) {
3130
		ctxt->mode = X86EMUL_MODE_VM86;
3131 3132
		cpl = 3;
	} else {
3133
		ctxt->mode = X86EMUL_MODE_PROT32;
3134 3135
		cpl = tss->cs & 3;
	}
3136

3137 3138 3139 3140
	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
3141
	ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3142
					cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3143 3144
	if (ret != X86EMUL_CONTINUE)
		return ret;
3145
	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3146
					X86_TRANSFER_TASK_SWITCH, NULL);
3147 3148
	if (ret != X86EMUL_CONTINUE)
		return ret;
3149
	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3150
					X86_TRANSFER_TASK_SWITCH, NULL);
3151 3152
	if (ret != X86EMUL_CONTINUE)
		return ret;
3153
	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3154
					X86_TRANSFER_TASK_SWITCH, NULL);
3155 3156
	if (ret != X86EMUL_CONTINUE)
		return ret;
3157
	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3158
					X86_TRANSFER_TASK_SWITCH, NULL);
3159 3160
	if (ret != X86EMUL_CONTINUE)
		return ret;
3161
	ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3162
					X86_TRANSFER_TASK_SWITCH, NULL);
3163 3164
	if (ret != X86EMUL_CONTINUE)
		return ret;
3165
	ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3166
					X86_TRANSFER_TASK_SWITCH, NULL);
3167

3168
	return ret;
3169 3170 3171 3172 3173 3174
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
3175
	const struct x86_emulate_ops *ops = ctxt->ops;
3176 3177
	struct tss_segment_32 tss_seg;
	int ret;
3178
	u32 new_tss_base = get_desc_base(new_desc);
3179 3180
	u32 eip_offset = offsetof(struct tss_segment_32, eip);
	u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3181

3182
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
3183
			    &ctxt->exception);
3184
	if (ret != X86EMUL_CONTINUE)
3185 3186
		return ret;

3187
	save_state_to_tss32(ctxt, &tss_seg);
3188

3189 3190 3191
	/* Only GP registers and segment selectors are saved */
	ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
			     ldt_sel_offset - eip_offset, &ctxt->exception);
3192
	if (ret != X86EMUL_CONTINUE)
3193 3194
		return ret;

3195
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
3196
			    &ctxt->exception);
3197
	if (ret != X86EMUL_CONTINUE)
3198 3199 3200 3201 3202
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

3203
		ret = ops->write_std(ctxt, new_tss_base,
3204 3205
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
3206
				     &ctxt->exception);
3207
		if (ret != X86EMUL_CONTINUE)
3208 3209 3210
			return ret;
	}

3211
	return load_state_from_tss32(ctxt, &tss_seg);
3212 3213 3214
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3215
				   u16 tss_selector, int idt_index, int reason,
3216
				   bool has_error_code, u32 error_code)
3217
{
3218
	const struct x86_emulate_ops *ops = ctxt->ops;
3219 3220
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
3221
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3222
	ulong old_tss_base =
3223
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3224
	u32 desc_limit;
3225
	ulong desc_addr, dr7;
3226 3227 3228

	/* FIXME: old_tss_base == ~0 ? */

3229
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3230 3231
	if (ret != X86EMUL_CONTINUE)
		return ret;
3232
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3233 3234 3235 3236 3237
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

3238 3239 3240 3241 3242
	/*
	 * Check privileges. The three cases are task switch caused by...
	 *
	 * 1. jmp/call/int to task gate: Check against DPL of the task gate
	 * 2. Exception/IRQ/iret: No check is performed
3243 3244
	 * 3. jmp/call to TSS/task-gate: No check is performed since the
	 *    hardware checks it before exiting.
3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260
	 */
	if (reason == TASK_SWITCH_GATE) {
		if (idt_index != -1) {
			/* Software interrupts */
			struct desc_struct task_gate_desc;
			int dpl;

			ret = read_interrupt_descriptor(ctxt, idt_index,
							&task_gate_desc);
			if (ret != X86EMUL_CONTINUE)
				return ret;

			dpl = task_gate_desc.dpl;
			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
				return emulate_gp(ctxt, (idt_index << 3) | 0x2);
		}
3261 3262
	}

3263 3264 3265 3266
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
3267
		return emulate_ts(ctxt, tss_selector & 0xfffc);
3268 3269 3270 3271
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3272
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3273 3274 3275 3276 3277 3278
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
G
Guo Chao 已提交
3279
	   note that old_tss_sel is not used after this point */
3280 3281 3282 3283
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
3284
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3285 3286
				     old_tss_base, &next_tss_desc);
	else
3287
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3288
				     old_tss_base, &next_tss_desc);
3289 3290
	if (ret != X86EMUL_CONTINUE)
		return ret;
3291 3292 3293 3294 3295 3296

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
3297
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3298 3299
	}

3300
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
3301
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3302

3303
	if (has_error_code) {
3304 3305 3306
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
3307
		ret = em_push(ctxt);
3308 3309
	}

3310 3311 3312
	ops->get_dr(ctxt, 7, &dr7);
	ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));

3313 3314 3315 3316
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3317
			 u16 tss_selector, int idt_index, int reason,
3318
			 bool has_error_code, u32 error_code)
3319 3320 3321
{
	int rc;

3322
	invalidate_registers(ctxt);
3323 3324
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
3325

3326
	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3327
				     has_error_code, error_code);
3328

3329
	if (rc == X86EMUL_CONTINUE) {
3330
		ctxt->eip = ctxt->_eip;
3331 3332
		writeback_registers(ctxt);
	}
3333

3334
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3335 3336
}

3337 3338
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
		struct operand *op)
3339
{
3340
	int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3341

3342 3343
	register_address_increment(ctxt, reg, df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, reg);
3344 3345
}

3346 3347 3348 3349 3350 3351
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
3352
	al = ctxt->dst.val;
3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

3370
	ctxt->dst.val = al;
3371
	/* Set PF, ZF, SF */
3372 3373 3374
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
3375
	fastop(ctxt, em_or);
3376 3377 3378 3379 3380 3381 3382 3383
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405
static int em_aam(struct x86_emulate_ctxt *ctxt)
{
	u8 al, ah;

	if (ctxt->src.val == 0)
		return emulate_de(ctxt);

	al = ctxt->dst.val & 0xff;
	ah = al / ctxt->src.val;
	al %= ctxt->src.val;

	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);

	/* Set PF, ZF, SF */
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
	fastop(ctxt, em_or);

	return X86EMUL_CONTINUE;
}

3406 3407 3408 3409 3410 3411 3412 3413 3414
static int em_aad(struct x86_emulate_ctxt *ctxt)
{
	u8 al = ctxt->dst.val & 0xff;
	u8 ah = (ctxt->dst.val >> 8) & 0xff;

	al = (al + (ah * ctxt->src.val)) & 0xff;

	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;

3415 3416 3417 3418 3419
	/* Set PF, ZF, SF */
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
	fastop(ctxt, em_or);
3420 3421 3422 3423

	return X86EMUL_CONTINUE;
}

3424 3425
static int em_call(struct x86_emulate_ctxt *ctxt)
{
3426
	int rc;
3427 3428 3429
	long rel = ctxt->src.val;

	ctxt->src.val = (unsigned long)ctxt->_eip;
3430 3431 3432
	rc = jmp_rel(ctxt, rel);
	if (rc != X86EMUL_CONTINUE)
		return rc;
3433 3434 3435
	return em_push(ctxt);
}

3436 3437 3438 3439 3440
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;
3441 3442 3443
	struct desc_struct old_desc, new_desc;
	const struct x86_emulate_ops *ops = ctxt->ops;
	int cpl = ctxt->ops->cpl(ctxt);
3444
	enum x86emul_mode prev_mode = ctxt->mode;
3445

3446
	old_eip = ctxt->_eip;
3447
	ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3448

3449
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3450 3451
	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
				       X86_TRANSFER_CALL_JMP, &new_desc);
3452
	if (rc != X86EMUL_CONTINUE)
3453
		return rc;
3454

3455
	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3456 3457
	if (rc != X86EMUL_CONTINUE)
		goto fail;
3458

3459
	ctxt->src.val = old_cs;
3460
	rc = em_push(ctxt);
3461
	if (rc != X86EMUL_CONTINUE)
3462
		goto fail;
3463

3464
	ctxt->src.val = old_eip;
3465 3466 3467
	rc = em_push(ctxt);
	/* If we failed, we tainted the memory, but the very least we should
	   restore cs */
3468 3469
	if (rc != X86EMUL_CONTINUE) {
		pr_warn_once("faulting far call emulation tainted memory\n");
3470
		goto fail;
3471
	}
3472 3473 3474
	return rc;
fail:
	ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3475
	ctxt->mode = prev_mode;
3476 3477
	return rc;

3478 3479
}

3480 3481 3482
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;
3483
	unsigned long eip;
3484

3485 3486 3487 3488
	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	rc = assign_eip_near(ctxt, eip);
3489 3490
	if (rc != X86EMUL_CONTINUE)
		return rc;
3491
	rsp_increment(ctxt, ctxt->src.val);
3492 3493 3494
	return X86EMUL_CONTINUE;
}

3495 3496 3497
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
3498 3499
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
3500 3501

	/* Write back the memory destination with implicit LOCK prefix. */
3502 3503
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
3504 3505 3506
	return X86EMUL_CONTINUE;
}

3507 3508
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
3509
	ctxt->dst.val = ctxt->src2.val;
3510
	return fastop(ctxt, em_imul);
3511 3512
}

3513 3514
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
3515 3516
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
3517
	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3518
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3519 3520 3521 3522

	return X86EMUL_CONTINUE;
}

3523 3524 3525 3526
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

3527
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3528 3529
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3530 3531 3532
	return X86EMUL_CONTINUE;
}

3533 3534 3535 3536
static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
{
	u64 pmc;

3537
	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3538
		return emulate_gp(ctxt, 0);
3539 3540
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3541 3542 3543
	return X86EMUL_CONTINUE;
}

3544 3545
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
3546
	memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3547 3548 3549
	return X86EMUL_CONTINUE;
}

B
Borislav Petkov 已提交
3550 3551 3552 3553 3554 3555 3556 3557 3558 3559
#define FFL(x) bit(X86_FEATURE_##x)

static int em_movbe(struct x86_emulate_ctxt *ctxt)
{
	u32 ebx, ecx, edx, eax = 1;
	u16 tmp;

	/*
	 * Check MOVBE is set in the guest-visible CPUID leaf.
	 */
3560
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
B
Borislav Petkov 已提交
3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584
	if (!(ecx & FFL(MOVBE)))
		return emulate_ud(ctxt);

	switch (ctxt->op_bytes) {
	case 2:
		/*
		 * From MOVBE definition: "...When the operand size is 16 bits,
		 * the upper word of the destination register remains unchanged
		 * ..."
		 *
		 * Both casting ->valptr and ->val to u16 breaks strict aliasing
		 * rules so we have to do the operation almost per hand.
		 */
		tmp = (u16)ctxt->src.val;
		ctxt->dst.val &= ~0xffffUL;
		ctxt->dst.val |= (unsigned long)swab16(tmp);
		break;
	case 4:
		ctxt->dst.val = swab32((u32)ctxt->src.val);
		break;
	case 8:
		ctxt->dst.val = swab64(ctxt->src.val);
		break;
	default:
3585
		BUG();
B
Borislav Petkov 已提交
3586 3587 3588 3589
	}
	return X86EMUL_CONTINUE;
}

3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617
static int em_cr_write(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_dr_write(struct x86_emulate_ctxt *ctxt)
{
	unsigned long val;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		val = ctxt->src.val & ~0ULL;
	else
		val = ctxt->src.val & ~0U;

	/* #UD condition is already handled. */
	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3618 3619 3620 3621
static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3622 3623 3624
	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
	if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3625 3626 3627 3628 3629 3630 3631 3632 3633
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3634
	if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3635 3636
		return emulate_gp(ctxt, 0);

3637 3638
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
	*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3639 3640 3641
	return X86EMUL_CONTINUE;
}

3642 3643
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
3644
	if (ctxt->modrm_reg > VCPU_SREG_GS)
3645 3646
		return emulate_ud(ctxt);

3647
	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3648 3649
	if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
		ctxt->dst.bytes = 2;
3650 3651 3652 3653 3654
	return X86EMUL_CONTINUE;
}

static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
3655
	u16 sel = ctxt->src.val;
3656

3657
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3658 3659
		return emulate_ud(ctxt);

3660
	if (ctxt->modrm_reg == VCPU_SREG_SS)
3661 3662 3663
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
3664 3665
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3666 3667
}

A
Avi Kivity 已提交
3668 3669 3670 3671 3672 3673 3674 3675 3676
static int em_lldt(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
}

A
Avi Kivity 已提交
3677 3678 3679 3680 3681 3682 3683 3684 3685
static int em_ltr(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
}

3686 3687
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
3688 3689 3690
	int rc;
	ulong linear;

3691
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3692
	if (rc == X86EMUL_CONTINUE)
3693
		ctxt->ops->invlpg(ctxt, linear);
3694
	/* Disable writeback. */
3695
	ctxt->dst.type = OP_NONE;
3696 3697 3698
	return X86EMUL_CONTINUE;
}

3699 3700 3701 3702 3703 3704 3705 3706 3707 3708
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

3709
static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3710
{
3711
	int rc = ctxt->ops->fix_hypercall(ctxt);
3712 3713 3714 3715 3716

	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
3717
	ctxt->_eip = ctxt->eip;
3718
	/* Disable writeback. */
3719
	ctxt->dst.type = OP_NONE;
3720 3721 3722
	return X86EMUL_CONTINUE;
}

3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737
static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
				  void (*get)(struct x86_emulate_ctxt *ctxt,
					      struct desc_ptr *ptr))
{
	struct desc_ptr desc_ptr;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
	get(ctxt, &desc_ptr);
	if (ctxt->op_bytes == 2) {
		ctxt->op_bytes = 4;
		desc_ptr.address &= 0x00ffffff;
	}
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
3738 3739
	return segmented_write_std(ctxt, ctxt->dst.addr.mem,
				   &desc_ptr, 2 + ctxt->op_bytes);
3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751
}

static int em_sgdt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
}

static int em_sidt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
}

3752
static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3753 3754 3755 3756
{
	struct desc_ptr desc_ptr;
	int rc;

3757 3758
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
3759
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3760
			     &desc_ptr.size, &desc_ptr.address,
3761
			     ctxt->op_bytes);
3762 3763
	if (rc != X86EMUL_CONTINUE)
		return rc;
3764
	if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3765
	    emul_is_noncanonical_address(desc_ptr.address, ctxt))
3766
		return emulate_gp(ctxt, 0);
3767 3768 3769 3770
	if (lgdt)
		ctxt->ops->set_gdt(ctxt, &desc_ptr);
	else
		ctxt->ops->set_idt(ctxt, &desc_ptr);
3771
	/* Disable writeback. */
3772
	ctxt->dst.type = OP_NONE;
3773 3774 3775
	return X86EMUL_CONTINUE;
}

3776 3777 3778 3779 3780
static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	return em_lgdt_lidt(ctxt, true);
}

3781 3782
static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
3783
	return em_lgdt_lidt(ctxt, false);
3784 3785 3786 3787
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
3788 3789
	if (ctxt->dst.type == OP_MEM)
		ctxt->dst.bytes = 2;
3790
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3791 3792 3793 3794 3795 3796
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3797 3798
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
3799 3800 3801
	return X86EMUL_CONTINUE;
}

3802 3803
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
3804 3805
	int rc = X86EMUL_CONTINUE;

3806
	register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3807
	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3808
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3809
		rc = jmp_rel(ctxt, ctxt->src.val);
3810

3811
	return rc;
3812 3813 3814 3815
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
3816 3817
	int rc = X86EMUL_CONTINUE;

3818
	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3819
		rc = jmp_rel(ctxt, ctxt->src.val);
3820

3821
	return rc;
3822 3823
}

3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841
static int em_in(struct x86_emulate_ctxt *ctxt)
{
	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
			     &ctxt->dst.val))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
}

static int em_out(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				    &ctxt->src.val, 1);
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3861 3862 3863
static int em_cpuid(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;
K
Kyle Huey 已提交
3864 3865 3866 3867 3868 3869 3870
	u64 msr = 0;

	ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
	if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
	    ctxt->ops->cpl(ctxt)) {
		return emulate_gp(ctxt, 0);
	}
A
Avi Kivity 已提交
3871

3872 3873
	eax = reg_read(ctxt, VCPU_REGS_RAX);
	ecx = reg_read(ctxt, VCPU_REGS_RCX);
3874
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
3875 3876 3877 3878
	*reg_write(ctxt, VCPU_REGS_RAX) = eax;
	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;
	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = edx;
A
Avi Kivity 已提交
3879 3880 3881
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3882 3883 3884 3885
static int em_sahf(struct x86_emulate_ctxt *ctxt)
{
	u32 flags;

3886 3887
	flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
		X86_EFLAGS_SF;
P
Paolo Bonzini 已提交
3888 3889 3890 3891 3892 3893 3894
	flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;

	ctxt->eflags &= ~0xffUL;
	ctxt->eflags |= flags | X86_EFLAGS_FIXED;
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3895 3896
static int em_lahf(struct x86_emulate_ctxt *ctxt)
{
3897 3898
	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
A
Avi Kivity 已提交
3899 3900 3901
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916
static int em_bswap(struct x86_emulate_ctxt *ctxt)
{
	switch (ctxt->op_bytes) {
#ifdef CONFIG_X86_64
	case 8:
		asm("bswap %0" : "+r"(ctxt->dst.val));
		break;
#endif
	default:
		asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
		break;
	}
	return X86EMUL_CONTINUE;
}

3917 3918 3919 3920 3921 3922
static int em_clflush(struct x86_emulate_ctxt *ctxt)
{
	/* emulating clflush regardless of cpuid */
	return X86EMUL_CONTINUE;
}

3923 3924 3925 3926 3927 3928
static int em_movsxd(struct x86_emulate_ctxt *ctxt)
{
	ctxt->dst.val = (s32) ctxt->src.val;
	return X86EMUL_CONTINUE;
}

3929 3930 3931 3932
static int check_fxsr(struct x86_emulate_ctxt *ctxt)
{
	u32 eax = 1, ebx, ecx = 0, edx;

3933
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949
	if (!(edx & FFL(FXSR)))
		return emulate_ud(ctxt);

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	/*
	 * Don't emulate a case that should never be hit, instead of working
	 * around a lack of fxsave64/fxrstor64 on old compilers.
	 */
	if (ctxt->mode >= X86EMUL_MODE_PROT64)
		return X86EMUL_UNHANDLEABLE;

	return X86EMUL_CONTINUE;
}

3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968
/*
 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
 * and restore MXCSR.
 */
static size_t __fxstate_size(int nregs)
{
	return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
}

static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
{
	bool cr4_osfxsr;
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return __fxstate_size(16);

	cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
	return __fxstate_size(cr4_osfxsr ? 8 : 0);
}

3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004
/*
 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
 *  1) 16 bit mode
 *  2) 32 bit mode
 *     - like (1), but FIP and FDP (foo) are only 16 bit.  At least Intel CPUs
 *       preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
 *       save and restore
 *  3) 64-bit mode with REX.W prefix
 *     - like (2), but XMM 8-15 are being saved and restored
 *  4) 64-bit mode without REX.W prefix
 *     - like (3), but FIP and FDP are 64 bit
 *
 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
 * desired result.  (4) is not emulated.
 *
 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
 * and FPU DS) should match.
 */
static int em_fxsave(struct x86_emulate_ctxt *ctxt)
{
	struct fxregs_state fx_state;
	int rc;

	rc = check_fxsr(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	ctxt->ops->get_fpu(ctxt);

	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));

	ctxt->ops->put_fpu(ctxt);

	if (rc != X86EMUL_CONTINUE)
		return rc;

4005 4006
	return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
		                   fxstate_size(ctxt));
4007 4008 4009 4010 4011 4012
}

static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
{
	struct fxregs_state fx_state;
	int rc;
4013
	size_t size;
4014 4015 4016 4017 4018

	rc = check_fxsr(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

4019
	ctxt->ops->get_fpu(ctxt);
4020

4021 4022 4023 4024 4025 4026
	size = fxstate_size(ctxt);
	if (size < __fxstate_size(16)) {
		rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
		if (rc != X86EMUL_CONTINUE)
			goto out;
	}
4027

4028 4029 4030
	rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
	if (rc != X86EMUL_CONTINUE)
		goto out;
4031

4032 4033 4034 4035
	if (fx_state.mxcsr >> 16) {
		rc = emulate_gp(ctxt, 0);
		goto out;
	}
4036 4037 4038 4039

	if (rc == X86EMUL_CONTINUE)
		rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));

4040
out:
4041 4042 4043 4044 4045
	ctxt->ops->put_fpu(ctxt);

	return rc;
}

4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
4060
	if (!valid_cr(ctxt->modrm_reg))
4061 4062 4063 4064 4065 4066 4067
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
4068 4069
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
4070
	u64 efer = 0;
4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
4088
		u64 cr4;
4089 4090 4091 4092
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

4093 4094
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4095 4096 4097 4098 4099 4100 4101 4102 4103 4104

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

4105
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4106 4107
		if (efer & EFER_LMA) {
			u64 maxphyaddr;
4108
			u32 eax, ebx, ecx, edx;
4109

4110 4111 4112 4113
			eax = 0x80000008;
			ecx = 0;
			if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
						 &edx, false))
4114 4115 4116 4117 4118
				maxphyaddr = eax & 0xff;
			else
				maxphyaddr = 36;
			rsvd = rsvd_bits(maxphyaddr, 62);
		}
4119 4120 4121 4122 4123 4124 4125

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
4126
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

4138 4139 4140 4141
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

4142
	ctxt->ops->get_dr(ctxt, 7, &dr7);
4143 4144 4145 4146 4147 4148 4149

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
4150
	int dr = ctxt->modrm_reg;
4151 4152 4153 4154 4155
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

4156
	cr4 = ctxt->ops->get_cr(ctxt, 4);
4157 4158 4159
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

4160 4161 4162 4163 4164 4165 4166
	if (check_dr7_gd(ctxt)) {
		ulong dr6;

		ctxt->ops->get_dr(ctxt, 6, &dr6);
		dr6 &= ~15;
		dr6 |= DR6_BD | DR6_RTM;
		ctxt->ops->set_dr(ctxt, 6, dr6);
4167
		return emulate_db(ctxt);
4168
	}
4169 4170 4171 4172 4173 4174

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
4175 4176
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
4177 4178 4179 4180 4181 4182 4183

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

4184 4185
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
4186
	u64 efer = 0;
4187

4188
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4189 4190 4191 4192 4193 4194 4195 4196 4197

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
4198
	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4199 4200

	/* Valid physical address? */
4201
	if (rax & 0xffff000000000000ULL)
4202 4203 4204 4205 4206
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

4207 4208
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
4209
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4210

4211
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4212 4213 4214 4215 4216
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

4217 4218
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
4219
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4220
	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4221

4222
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4223
	    ctxt->ops->check_pmc(ctxt, rcx))
4224 4225 4226 4227 4228
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

4229 4230
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
4231 4232
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4233 4234 4235 4236 4237 4238 4239
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
4240 4241
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4242 4243 4244 4245 4246
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

4247
#define D(_y) { .flags = (_y) }
4248 4249 4250
#define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
#define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
		      .intercept = x86_intercept_##_i, .check_perm = (_p) }
4251
#define N    D(NotImpl)
4252
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4253 4254
#define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
#define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4255
#define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4256
#define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4257
#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4258
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4259
#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4260
#define II(_f, _e, _i) \
4261
	{ .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4262
#define IIP(_f, _e, _i, _p) \
4263 4264
	{ .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
	  .intercept = x86_intercept_##_i, .check_perm = (_p) }
4265
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4266

4267
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
4268
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4269
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
4270
#define F2bv(_f, _e)  F((_f) | ByteOp, _e), F(_f, _e)
4271 4272
#define I2bvIP(_f, _e, _i, _p) \
	IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4273

4274 4275 4276
#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4277

4278 4279
static const struct opcode group7_rm0[] = {
	N,
4280
	I(SrcNone | Priv | EmulateOnUD,	em_hypercall),
4281 4282 4283
	N, N, N, N, N, N,
};

4284
static const struct opcode group7_rm1[] = {
4285 4286
	DI(SrcNone | Priv, monitor),
	DI(SrcNone | Priv, mwait),
4287 4288 4289
	N, N, N, N, N, N,
};

4290
static const struct opcode group7_rm3[] = {
4291
	DIP(SrcNone | Prot | Priv,		vmrun,		check_svme_pa),
4292
	II(SrcNone  | Prot | EmulateOnUD,	em_hypercall,	vmmcall),
4293 4294 4295 4296 4297 4298
	DIP(SrcNone | Prot | Priv,		vmload,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		vmsave,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		stgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		clgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		skinit,		check_svme),
	DIP(SrcNone | Prot | Priv,		invlpga,	check_svme),
4299
};
4300

4301
static const struct opcode group7_rm7[] = {
4302
	N,
4303
	DIP(SrcNone, rdtscp, check_rdtsc),
4304 4305
	N, N, N, N, N, N,
};
4306

4307
static const struct opcode group1[] = {
4308 4309 4310 4311 4312 4313 4314 4315
	F(Lock, em_add),
	F(Lock | PageTable, em_or),
	F(Lock, em_adc),
	F(Lock, em_sbb),
	F(Lock | PageTable, em_and),
	F(Lock, em_sub),
	F(Lock, em_xor),
	F(NoWrite, em_cmp),
4316 4317
};

4318
static const struct opcode group1A[] = {
4319
	I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4320 4321
};

4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332
static const struct opcode group2[] = {
	F(DstMem | ModRM, em_rol),
	F(DstMem | ModRM, em_ror),
	F(DstMem | ModRM, em_rcl),
	F(DstMem | ModRM, em_rcr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_shr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_sar),
};

4333
static const struct opcode group3[] = {
4334 4335
	F(DstMem | SrcImm | NoWrite, em_test),
	F(DstMem | SrcImm | NoWrite, em_test),
4336 4337
	F(DstMem | SrcNone | Lock, em_not),
	F(DstMem | SrcNone | Lock, em_neg),
4338 4339
	F(DstXacc | Src2Mem, em_mul_ex),
	F(DstXacc | Src2Mem, em_imul_ex),
4340 4341
	F(DstXacc | Src2Mem, em_div_ex),
	F(DstXacc | Src2Mem, em_idiv_ex),
4342 4343
};

4344
static const struct opcode group4[] = {
4345 4346
	F(ByteOp | DstMem | SrcNone | Lock, em_inc),
	F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4347 4348 4349
	N, N, N, N, N, N,
};

4350
static const struct opcode group5[] = {
4351 4352
	F(DstMem | SrcNone | Lock,		em_inc),
	F(DstMem | SrcNone | Lock,		em_dec),
4353
	I(SrcMem | NearBranch,			em_call_near_abs),
4354
	I(SrcMemFAddr | ImplicitOps,		em_call_far),
4355
	I(SrcMem | NearBranch,			em_jmp_abs),
4356
	I(SrcMemFAddr | ImplicitOps,		em_jmp_far),
4357
	I(SrcMem | Stack | TwoMemOp,		em_push), D(Undefined),
4358 4359
};

4360
static const struct opcode group6[] = {
4361 4362
	DI(Prot | DstMem,	sldt),
	DI(Prot | DstMem,	str),
A
Avi Kivity 已提交
4363
	II(Prot | Priv | SrcMem16, em_lldt, lldt),
A
Avi Kivity 已提交
4364
	II(Prot | Priv | SrcMem16, em_ltr, ltr),
4365 4366 4367
	N, N, N, N,
};

4368
static const struct group_dual group7 = { {
4369 4370
	II(Mov | DstMem,			em_sgdt, sgdt),
	II(Mov | DstMem,			em_sidt, sidt),
4371 4372 4373 4374 4375
	II(SrcMem | Priv,			em_lgdt, lgdt),
	II(SrcMem | Priv,			em_lidt, lidt),
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	II(SrcMem | ByteOp | Priv | NoAccess,	em_invlpg, invlpg),
4376
}, {
4377
	EXT(0, group7_rm0),
4378
	EXT(0, group7_rm1),
4379
	N, EXT(0, group7_rm3),
4380 4381 4382
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	EXT(0, group7_rm7),
4383 4384
} };

4385
static const struct opcode group8[] = {
4386
	N, N, N, N,
4387 4388 4389 4390
	F(DstMem | SrcImmByte | NoWrite,		em_bt),
	F(DstMem | SrcImmByte | Lock | PageTable,	em_bts),
	F(DstMem | SrcImmByte | Lock,			em_btr),
	F(DstMem | SrcImmByte | Lock | PageTable,	em_btc),
4391 4392
};

4393
static const struct group_dual group9 = { {
4394
	N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4395 4396 4397 4398
}, {
	N, N, N, N, N, N, N, N,
} };

4399
static const struct opcode group11[] = {
4400
	I(DstMem | SrcImm | Mov | PageTable, em_mov),
4401
	X7(D(Undefined)),
4402 4403
};

4404
static const struct gprefix pfx_0f_ae_7 = {
4405
	I(SrcMem | ByteOp, em_clflush), N, N, N,
4406 4407 4408
};

static const struct group_dual group15 = { {
4409 4410 4411
	I(ModRM | Aligned16, em_fxsave),
	I(ModRM | Aligned16, em_fxrstor),
	N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4412 4413 4414 4415
}, {
	N, N, N, N, N, N, N, N,
} };

4416
static const struct gprefix pfx_0f_6f_0f_7f = {
4417
	I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4418 4419
};

4420 4421 4422 4423
static const struct instr_dual instr_dual_0f_2b = {
	I(0, em_mov), N
};

4424
static const struct gprefix pfx_0f_2b = {
4425
	ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4426 4427
};

4428
static const struct gprefix pfx_0f_28_0f_29 = {
4429
	I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4430 4431
};

4432 4433 4434 4435
static const struct gprefix pfx_0f_e7 = {
	N, I(Sse, em_mov), N, N,
};

4436
static const struct escape escape_d9 = { {
4437
	N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_db = { {
	N, N, N, N, N, N, N, N,
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_dd = { {
4479
	N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

4499 4500 4501 4502
static const struct instr_dual instr_dual_0f_c3 = {
	I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
};

4503 4504 4505 4506
static const struct mode_dual mode_dual_63 = {
	N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
};

4507
static const struct opcode opcode_table[256] = {
4508
	/* 0x00 - 0x07 */
4509
	F6ALU(Lock, em_add),
4510 4511
	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4512
	/* 0x08 - 0x0F */
4513
	F6ALU(Lock | PageTable, em_or),
4514 4515
	I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
	N,
4516
	/* 0x10 - 0x17 */
4517
	F6ALU(Lock, em_adc),
4518 4519
	I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4520
	/* 0x18 - 0x1F */
4521
	F6ALU(Lock, em_sbb),
4522 4523
	I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4524
	/* 0x20 - 0x27 */
4525
	F6ALU(Lock | PageTable, em_and), N, N,
4526
	/* 0x28 - 0x2F */
4527
	F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4528
	/* 0x30 - 0x37 */
4529
	F6ALU(Lock, em_xor), N, N,
4530
	/* 0x38 - 0x3F */
4531
	F6ALU(NoWrite, em_cmp), N, N,
4532
	/* 0x40 - 0x4F */
4533
	X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4534
	/* 0x50 - 0x57 */
4535
	X8(I(SrcReg | Stack, em_push)),
4536
	/* 0x58 - 0x5F */
4537
	X8(I(DstReg | Stack, em_pop)),
4538
	/* 0x60 - 0x67 */
4539 4540
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
4541
	N, MD(ModRM, &mode_dual_63),
4542 4543
	N, N, N, N,
	/* 0x68 - 0x6F */
4544 4545
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4546 4547
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4548
	I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4549
	I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4550
	/* 0x70 - 0x7F */
4551
	X16(D(SrcImmByte | NearBranch)),
4552
	/* 0x80 - 0x87 */
4553 4554 4555 4556
	G(ByteOp | DstMem | SrcImm, group1),
	G(DstMem | SrcImm, group1),
	G(ByteOp | DstMem | SrcImm | No64, group1),
	G(DstMem | SrcImmByte, group1),
4557
	F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4558
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4559
	/* 0x88 - 0x8F */
4560
	I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4561
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4562
	I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4563 4564 4565
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
4566
	/* 0x90 - 0x97 */
4567
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4568
	/* 0x98 - 0x9F */
4569
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4570
	I(SrcImmFAddr | No64, em_call_far), N,
4571
	II(ImplicitOps | Stack, em_pushf, pushf),
P
Paolo Bonzini 已提交
4572 4573
	II(ImplicitOps | Stack, em_popf, popf),
	I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4574
	/* 0xA0 - 0xA7 */
4575
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4576
	I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4577 4578
	I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
	F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4579
	/* 0xA8 - 0xAF */
4580
	F2bv(DstAcc | SrcImm | NoWrite, em_test),
4581 4582
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4583
	F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4584
	/* 0xB0 - 0xB7 */
4585
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4586
	/* 0xB8 - 0xBF */
4587
	X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4588
	/* 0xC0 - 0xC7 */
4589
	G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4590 4591
	I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
	I(ImplicitOps | NearBranch, em_ret),
4592 4593
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4594
	G(ByteOp, group11), G(0, group11),
4595
	/* 0xC8 - 0xCF */
A
Avi Kivity 已提交
4596
	I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4597 4598
	I(ImplicitOps | SrcImmU16, em_ret_far_imm),
	I(ImplicitOps, em_ret_far),
4599
	D(ImplicitOps), DI(SrcImmByte, intn),
4600
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4601
	/* 0xD0 - 0xD7 */
4602 4603
	G(Src2One | ByteOp, group2), G(Src2One, group2),
	G(Src2CL | ByteOp, group2), G(Src2CL, group2),
P
Paolo Bonzini 已提交
4604
	I(DstAcc | SrcImmUByte | No64, em_aam),
P
Paolo Bonzini 已提交
4605 4606
	I(DstAcc | SrcImmUByte | No64, em_aad),
	F(DstAcc | ByteOp | No64, em_salc),
P
Paolo Bonzini 已提交
4607
	I(DstAcc | SrcXLat | ByteOp, em_mov),
4608
	/* 0xD8 - 0xDF */
4609
	N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4610
	/* 0xE0 - 0xE7 */
4611 4612
	X3(I(SrcImmByte | NearBranch, em_loop)),
	I(SrcImmByte | NearBranch, em_jcxz),
4613 4614
	I2bvIP(SrcImmUByte | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4615
	/* 0xE8 - 0xEF */
4616 4617 4618
	I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
	I(SrcImmFAddr | No64, em_jmp_far),
	D(SrcImmByte | ImplicitOps | NearBranch),
4619 4620
	I2bvIP(SrcDX | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4621
	/* 0xF0 - 0xF7 */
4622
	N, DI(ImplicitOps, icebp), N, N,
4623 4624
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
4625
	/* 0xF8 - 0xFF */
4626 4627
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4628 4629 4630
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

4631
static const struct opcode twobyte_table[256] = {
4632
	/* 0x00 - 0x0F */
4633
	G(0, group6), GD(0, &group7), N, N,
4634
	N, I(ImplicitOps | EmulateOnUD, em_syscall),
4635
	II(ImplicitOps | Priv, em_clts, clts), N,
4636
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4637
	N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4638
	/* 0x10 - 0x1F */
P
Paolo Bonzini 已提交
4639
	N, N, N, N, N, N, N, N,
4640 4641
	D(ImplicitOps | ModRM | SrcMem | NoAccess),
	N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4642
	/* 0x20 - 0x2F */
4643 4644 4645 4646 4647 4648
	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
						check_cr_write),
	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
						check_dr_write),
4649
	N, N, N, N,
4650 4651
	GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
	GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4652
	N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4653
	N, N, N, N,
4654
	/* 0x30 - 0x3F */
4655
	II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4656
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4657
	II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4658
	IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4659 4660
	I(ImplicitOps | EmulateOnUD, em_sysenter),
	I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4661
	N, N,
4662 4663
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
4664
	X16(D(DstReg | SrcMem | ModRM)),
4665 4666 4667
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
4668 4669 4670 4671
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4672
	/* 0x70 - 0x7F */
4673 4674 4675 4676
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4677
	/* 0x80 - 0x8F */
4678
	X16(D(SrcImm | NearBranch)),
4679
	/* 0x90 - 0x9F */
4680
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4681
	/* 0xA0 - 0xA7 */
4682
	I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4683 4684
	II(ImplicitOps, em_cpuid, cpuid),
	F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4685 4686
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4687
	/* 0xA8 - 0xAF */
4688
	I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4689
	II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4690
	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4691 4692
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4693
	GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4694
	/* 0xB0 - 0xB7 */
4695
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4696
	I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4697
	F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4698 4699
	I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4700
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4701 4702
	/* 0xB8 - 0xBF */
	N, N,
4703
	G(BitOp, group8),
4704
	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4705 4706
	I(DstReg | SrcMem | ModRM, em_bsf_c),
	I(DstReg | SrcMem | ModRM, em_bsr_c),
4707
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
A
Avi Kivity 已提交
4708
	/* 0xC0 - 0xC7 */
4709
	F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4710
	N, ID(0, &instr_dual_0f_c3),
4711
	N, N, N, GD(0, &group9),
A
Avi Kivity 已提交
4712 4713
	/* 0xC8 - 0xCF */
	X8(I(DstReg, em_bswap)),
4714 4715 4716
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
4717 4718
	N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
	N, N, N, N, N, N, N, N,
4719 4720 4721 4722
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

4723 4724 4725 4726 4727 4728 4729 4730
static const struct instr_dual instr_dual_0f_38_f0 = {
	I(DstReg | SrcMem | Mov, em_movbe), N
};

static const struct instr_dual instr_dual_0f_38_f1 = {
	I(DstMem | SrcReg | Mov, em_movbe), N
};

4731
static const struct gprefix three_byte_0f_38_f0 = {
4732
	ID(0, &instr_dual_0f_38_f0), N, N, N
4733 4734 4735
};

static const struct gprefix three_byte_0f_38_f1 = {
4736
	ID(0, &instr_dual_0f_38_f1), N, N, N
4737 4738 4739 4740 4741 4742 4743 4744 4745
};

/*
 * Insns below are selected by the prefix which indexed by the third opcode
 * byte.
 */
static const struct opcode opcode_map_0f_38[256] = {
	/* 0x00 - 0x7f */
	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
B
Borislav Petkov 已提交
4746 4747 4748
	/* 0x80 - 0xef */
	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
	/* 0xf0 - 0xf1 */
4749 4750
	GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
	GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
B
Borislav Petkov 已提交
4751 4752
	/* 0xf2 - 0xff */
	N, N, X4(N), X8(N)
4753 4754
};

4755 4756 4757 4758 4759
#undef D
#undef N
#undef G
#undef GD
#undef I
4760
#undef GP
4761
#undef EXT
4762
#undef MD
N
Nadav Amit 已提交
4763
#undef ID
4764

4765
#undef D2bv
4766
#undef D2bvIP
4767
#undef I2bv
4768
#undef I2bvIP
4769
#undef I6ALU
4770

4771
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4772 4773 4774
{
	unsigned size;

4775
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
4788
	op->addr.mem.ea = ctxt->_eip;
4789 4790 4791
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
4792
		op->val = insn_fetch(s8, ctxt);
4793 4794
		break;
	case 2:
4795
		op->val = insn_fetch(s16, ctxt);
4796 4797
		break;
	case 4:
4798
		op->val = insn_fetch(s32, ctxt);
4799
		break;
4800 4801 4802
	case 8:
		op->val = insn_fetch(s64, ctxt);
		break;
4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

4821 4822 4823 4824 4825 4826 4827
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
			  unsigned d)
{
	int rc = X86EMUL_CONTINUE;

	switch (d) {
	case OpReg:
4828
		decode_register_operand(ctxt, op);
4829 4830
		break;
	case OpImmUByte:
4831
		rc = decode_imm(ctxt, op, 1, false);
4832 4833
		break;
	case OpMem:
4834
		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4835 4836 4837
	mem_common:
		*op = ctxt->memop;
		ctxt->memopp = op;
4838
		if (ctxt->d & BitOp)
4839 4840 4841
			fetch_bit_operand(ctxt);
		op->orig_val = op->val;
		break;
4842
	case OpMem64:
4843
		ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4844
		goto mem_common;
4845 4846 4847
	case OpAcc:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4848
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4849 4850 4851
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869
	case OpAccLo:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
	case OpAccHi:
		if (ctxt->d & ByteOp) {
			op->type = OP_NONE;
			break;
		}
		op->type = OP_REG;
		op->bytes = ctxt->op_bytes;
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
4870 4871 4872 4873
	case OpDI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4874
			register_address(ctxt, VCPU_REGS_RDI);
4875 4876
		op->addr.mem.seg = VCPU_SREG_ES;
		op->val = 0;
4877
		op->count = 1;
4878 4879 4880 4881
		break;
	case OpDX:
		op->type = OP_REG;
		op->bytes = 2;
4882
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4883 4884
		fetch_register_operand(op);
		break;
4885
	case OpCL:
4886
		op->type = OP_IMM;
4887
		op->bytes = 1;
4888
		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4889 4890 4891 4892 4893
		break;
	case OpImmByte:
		rc = decode_imm(ctxt, op, 1, true);
		break;
	case OpOne:
4894
		op->type = OP_IMM;
4895 4896 4897 4898 4899 4900
		op->bytes = 1;
		op->val = 1;
		break;
	case OpImm:
		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
		break;
4901 4902 4903
	case OpImm64:
		rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
		break;
4904 4905
	case OpMem8:
		ctxt->memop.bytes = 1;
4906
		if (ctxt->memop.type == OP_REG) {
4907 4908
			ctxt->memop.addr.reg = decode_register(ctxt,
					ctxt->modrm_rm, true);
4909 4910
			fetch_register_operand(&ctxt->memop);
		}
4911
		goto mem_common;
4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927
	case OpMem16:
		ctxt->memop.bytes = 2;
		goto mem_common;
	case OpMem32:
		ctxt->memop.bytes = 4;
		goto mem_common;
	case OpImmU16:
		rc = decode_imm(ctxt, op, 2, false);
		break;
	case OpImmU:
		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
		break;
	case OpSI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4928
			register_address(ctxt, VCPU_REGS_RSI);
B
Bandan Das 已提交
4929
		op->addr.mem.seg = ctxt->seg_override;
4930
		op->val = 0;
4931
		op->count = 1;
4932
		break;
P
Paolo Bonzini 已提交
4933 4934 4935 4936
	case OpXLat:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4937
			address_mask(ctxt,
P
Paolo Bonzini 已提交
4938 4939
				reg_read(ctxt, VCPU_REGS_RBX) +
				(reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
B
Bandan Das 已提交
4940
		op->addr.mem.seg = ctxt->seg_override;
P
Paolo Bonzini 已提交
4941 4942
		op->val = 0;
		break;
4943 4944 4945 4946 4947 4948 4949 4950 4951
	case OpImmFAddr:
		op->type = OP_IMM;
		op->addr.mem.ea = ctxt->_eip;
		op->bytes = ctxt->op_bytes + 2;
		insn_fetch_arr(op->valptr, op->bytes, ctxt);
		break;
	case OpMemFAddr:
		ctxt->memop.bytes = ctxt->op_bytes + 2;
		goto mem_common;
4952
	case OpES:
4953
		op->type = OP_IMM;
4954 4955 4956
		op->val = VCPU_SREG_ES;
		break;
	case OpCS:
4957
		op->type = OP_IMM;
4958 4959 4960
		op->val = VCPU_SREG_CS;
		break;
	case OpSS:
4961
		op->type = OP_IMM;
4962 4963 4964
		op->val = VCPU_SREG_SS;
		break;
	case OpDS:
4965
		op->type = OP_IMM;
4966 4967 4968
		op->val = VCPU_SREG_DS;
		break;
	case OpFS:
4969
		op->type = OP_IMM;
4970 4971 4972
		op->val = VCPU_SREG_FS;
		break;
	case OpGS:
4973
		op->type = OP_IMM;
4974 4975
		op->val = VCPU_SREG_GS;
		break;
4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986
	case OpImplicit:
		/* Special instructions do their own operand decoding. */
	default:
		op->type = OP_NONE; /* Disable writeback. */
		break;
	}

done:
	return rc;
}

4987
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4988 4989 4990
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
4991
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4992
	bool op_prefix = false;
B
Bandan Das 已提交
4993
	bool has_seg_override = false;
4994
	struct opcode opcode;
4995

4996 4997
	ctxt->memop.type = OP_NONE;
	ctxt->memopp = NULL;
4998
	ctxt->_eip = ctxt->eip;
4999 5000
	ctxt->fetch.ptr = ctxt->fetch.data;
	ctxt->fetch.end = ctxt->fetch.data + insn_len;
B
Borislav Petkov 已提交
5001
	ctxt->opcode_len = 1;
5002
	if (insn_len > 0)
5003
		memcpy(ctxt->fetch.data, insn, insn_len);
5004
	else {
5005
		rc = __do_insn_fetch_bytes(ctxt, 1);
5006 5007 5008
		if (rc != X86EMUL_CONTINUE)
			return rc;
	}
5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
5026
		return EMULATION_FAILED;
5027 5028
	}

5029 5030
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
5031 5032 5033

	/* Legacy prefixes. */
	for (;;) {
5034
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
5035
		case 0x66:	/* operand-size override */
5036
			op_prefix = true;
5037
			/* switch between 2/4 bytes */
5038
			ctxt->op_bytes = def_op_bytes ^ 6;
5039 5040 5041 5042
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
5043
				ctxt->ad_bytes = def_ad_bytes ^ 12;
5044 5045
			else
				/* switch between 2/4 bytes */
5046
				ctxt->ad_bytes = def_ad_bytes ^ 6;
5047 5048 5049 5050 5051
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
B
Bandan Das 已提交
5052 5053
			has_seg_override = true;
			ctxt->seg_override = (ctxt->b >> 3) & 3;
5054 5055 5056
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
B
Bandan Das 已提交
5057 5058
			has_seg_override = true;
			ctxt->seg_override = ctxt->b & 7;
5059 5060 5061 5062
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
5063
			ctxt->rex_prefix = ctxt->b;
5064 5065
			continue;
		case 0xf0:	/* LOCK */
5066
			ctxt->lock_prefix = 1;
5067 5068 5069
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
5070
			ctxt->rep_prefix = ctxt->b;
5071 5072 5073 5074 5075 5076 5077
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

5078
		ctxt->rex_prefix = 0;
5079 5080 5081 5082 5083
	}

done_prefixes:

	/* REX prefix. */
5084 5085
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
5086 5087

	/* Opcode byte(s). */
5088
	opcode = opcode_table[ctxt->b];
5089
	/* Two-byte opcode? */
5090
	if (ctxt->b == 0x0f) {
B
Borislav Petkov 已提交
5091
		ctxt->opcode_len = 2;
5092
		ctxt->b = insn_fetch(u8, ctxt);
5093
		opcode = twobyte_table[ctxt->b];
5094 5095 5096 5097 5098 5099 5100

		/* 0F_38 opcode map */
		if (ctxt->b == 0x38) {
			ctxt->opcode_len = 3;
			ctxt->b = insn_fetch(u8, ctxt);
			opcode = opcode_map_0f_38[ctxt->b];
		}
5101
	}
5102
	ctxt->d = opcode.flags;
5103

5104 5105 5106
	if (ctxt->d & ModRM)
		ctxt->modrm = insn_fetch(u8, ctxt);

5107 5108
	/* vex-prefix instructions are not implemented */
	if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5109
	    (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5110 5111 5112
		ctxt->d = NotImpl;
	}

5113 5114
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
5115
		case Group:
5116
			goffset = (ctxt->modrm >> 3) & 7;
5117 5118 5119
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
5120 5121
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
5122 5123 5124 5125 5126
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
5127
			goffset = ctxt->modrm & 7;
5128
			opcode = opcode.u.group[goffset];
5129 5130
			break;
		case Prefix:
5131
			if (ctxt->rep_prefix && op_prefix)
5132
				return EMULATION_FAILED;
5133
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5134 5135 5136 5137 5138 5139 5140
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
5141 5142 5143 5144 5145 5146
		case Escape:
			if (ctxt->modrm > 0xbf)
				opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
			else
				opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
			break;
5147 5148 5149 5150 5151 5152
		case InstrDual:
			if ((ctxt->modrm >> 6) == 3)
				opcode = opcode.u.idual->mod3;
			else
				opcode = opcode.u.idual->mod012;
			break;
5153 5154 5155 5156 5157 5158
		case ModeDual:
			if (ctxt->mode == X86EMUL_MODE_PROT64)
				opcode = opcode.u.mdual->mode64;
			else
				opcode = opcode.u.mdual->mode32;
			break;
5159
		default:
5160
			return EMULATION_FAILED;
5161
		}
5162

5163
		ctxt->d &= ~(u64)GroupMask;
5164
		ctxt->d |= opcode.flags;
5165 5166
	}

5167 5168 5169 5170
	/* Unrecognised? */
	if (ctxt->d == 0)
		return EMULATION_FAILED;

5171
	ctxt->execute = opcode.u.execute;
5172

5173 5174 5175
	if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
		return EMULATION_FAILED;

5176
	if (unlikely(ctxt->d &
5177 5178
	    (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
	     No16))) {
5179 5180 5181 5182 5183 5184
		/*
		 * These are copied unconditionally here, and checked unconditionally
		 * in x86_emulate_insn.
		 */
		ctxt->check_perm = opcode.check_perm;
		ctxt->intercept = opcode.intercept;
5185

5186 5187
		if (ctxt->d & NotImpl)
			return EMULATION_FAILED;
5188

5189 5190 5191 5192 5193 5194
		if (mode == X86EMUL_MODE_PROT64) {
			if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
				ctxt->op_bytes = 8;
			else if (ctxt->d & NearBranch)
				ctxt->op_bytes = 8;
		}
5195

5196 5197 5198 5199 5200 5201 5202
		if (ctxt->d & Op3264) {
			if (mode == X86EMUL_MODE_PROT64)
				ctxt->op_bytes = 8;
			else
				ctxt->op_bytes = 4;
		}

5203 5204 5205
		if ((ctxt->d & No16) && ctxt->op_bytes == 2)
			ctxt->op_bytes = 4;

5206 5207 5208 5209 5210
		if (ctxt->d & Sse)
			ctxt->op_bytes = 16;
		else if (ctxt->d & Mmx)
			ctxt->op_bytes = 8;
	}
A
Avi Kivity 已提交
5211

5212
	/* ModRM and SIB bytes. */
5213
	if (ctxt->d & ModRM) {
5214
		rc = decode_modrm(ctxt, &ctxt->memop);
B
Bandan Das 已提交
5215 5216 5217 5218
		if (!has_seg_override) {
			has_seg_override = true;
			ctxt->seg_override = ctxt->modrm_seg;
		}
5219
	} else if (ctxt->d & MemAbs)
5220
		rc = decode_abs(ctxt, &ctxt->memop);
5221 5222 5223
	if (rc != X86EMUL_CONTINUE)
		goto done;

B
Bandan Das 已提交
5224 5225
	if (!has_seg_override)
		ctxt->seg_override = VCPU_SREG_DS;
5226

B
Bandan Das 已提交
5227
	ctxt->memop.addr.mem.seg = ctxt->seg_override;
5228 5229 5230 5231 5232

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
5233
	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5234 5235 5236
	if (rc != X86EMUL_CONTINUE)
		goto done;

5237 5238 5239 5240
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
5241
	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5242 5243 5244
	if (rc != X86EMUL_CONTINUE)
		goto done;

5245
	/* Decode and fetch the destination operand: register or memory. */
5246
	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5247

5248
	if (ctxt->rip_relative && likely(ctxt->memopp))
5249 5250
		ctxt->memopp->addr.mem.ea = address_mask(ctxt,
					ctxt->memopp->addr.mem.ea + ctxt->_eip);
5251

5252
done:
5253
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5254 5255
}

5256 5257 5258 5259 5260
bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
{
	return ctxt->d & PageTable;
}

5261 5262 5263 5264 5265 5266 5267 5268 5269
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
5270 5271 5272
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
5273
		 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5274
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
5275
		    ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5276 5277 5278 5279 5280
		return true;

	return false;
}

A
Avi Kivity 已提交
5281 5282
static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
{
R
Radim Krčmář 已提交
5283
	int rc;
A
Avi Kivity 已提交
5284 5285

	ctxt->ops->get_fpu(ctxt);
R
Radim Krčmář 已提交
5286
	rc = asm_safe("fwait");
A
Avi Kivity 已提交
5287 5288
	ctxt->ops->put_fpu(ctxt);

R
Radim Krčmář 已提交
5289
	if (unlikely(rc != X86EMUL_CONTINUE))
A
Avi Kivity 已提交
5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301
		return emulate_exception(ctxt, MF_VECTOR, 0, false);

	return X86EMUL_CONTINUE;
}

static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
				       struct operand *op)
{
	if (op->type == OP_MM)
		read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
}

5302 5303 5304
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
{
	ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5305

5306 5307
	if (!(ctxt->d & ByteOp))
		fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5308

5309
	asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5310
	    : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5311
	      [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5312
	    : "c"(ctxt->src2.val));
5313

5314
	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5315 5316
	if (!fop) /* exception is returned in fop variable */
		return emulate_de(ctxt);
5317 5318
	return X86EMUL_CONTINUE;
}
5319

5320 5321
void init_decode_cache(struct x86_emulate_ctxt *ctxt)
{
B
Bandan Das 已提交
5322 5323
	memset(&ctxt->rip_relative, 0,
	       (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5324 5325 5326 5327 5328 5329

	ctxt->io_read.pos = 0;
	ctxt->io_read.end = 0;
	ctxt->mem_read.end = 0;
}

5330
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5331
{
5332
	const struct x86_emulate_ops *ops = ctxt->ops;
5333
	int rc = X86EMUL_CONTINUE;
5334
	int saved_dst_type = ctxt->dst.type;
5335
	unsigned emul_flags;
5336

5337
	ctxt->mem_read.pos = 0;
5338

5339 5340
	/* LOCK prefix is allowed only with some instructions */
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5341
		rc = emulate_ud(ctxt);
5342 5343 5344
		goto done;
	}

5345
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5346
		rc = emulate_ud(ctxt);
5347 5348 5349
		goto done;
	}

5350
	emul_flags = ctxt->ops->get_hflags(ctxt);
5351 5352 5353 5354 5355 5356 5357
	if (unlikely(ctxt->d &
		     (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
		if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
				(ctxt->d & Undefined)) {
			rc = emulate_ud(ctxt);
			goto done;
		}
A
Avi Kivity 已提交
5358

5359 5360 5361
		if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
		    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
			rc = emulate_ud(ctxt);
A
Avi Kivity 已提交
5362
			goto done;
5363
		}
A
Avi Kivity 已提交
5364

5365 5366
		if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
			rc = emulate_nm(ctxt);
5367
			goto done;
5368
		}
5369

5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382
		if (ctxt->d & Mmx) {
			rc = flush_pending_x87_faults(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			/*
			 * Now that we know the fpu is exception safe, we can fetch
			 * operands from it.
			 */
			fetch_possible_mmx_operand(ctxt, &ctxt->src);
			fetch_possible_mmx_operand(ctxt, &ctxt->src2);
			if (!(ctxt->d & Mov))
				fetch_possible_mmx_operand(ctxt, &ctxt->dst);
		}
5383

5384
		if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5385 5386 5387 5388 5389
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_PRE_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}
5390

5391 5392 5393 5394 5395 5396
		/* Instruction can only be executed in protected mode */
		if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
			rc = emulate_ud(ctxt);
			goto done;
		}

5397 5398
		/* Privileged instruction can be executed only in CPL=0 */
		if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5399 5400 5401 5402
			if (ctxt->d & PrivUD)
				rc = emulate_ud(ctxt);
			else
				rc = emulate_gp(ctxt, 0);
5403
			goto done;
5404
		}
5405

5406
		/* Do instruction specific permission checks */
5407
		if (ctxt->d & CheckPerm) {
5408 5409 5410 5411 5412
			rc = ctxt->check_perm(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

5413
		if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5414 5415 5416 5417 5418 5419 5420 5421 5422
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_POST_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

		if (ctxt->rep_prefix && (ctxt->d & String)) {
			/* All REP prefixes have the same first termination condition */
			if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5423
				string_registers_quirk(ctxt);
5424
				ctxt->eip = ctxt->_eip;
5425
				ctxt->eflags &= ~X86_EFLAGS_RF;
5426 5427
				goto done;
			}
5428 5429 5430
		}
	}

5431 5432 5433
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
5434
		if (rc != X86EMUL_CONTINUE)
5435
			goto done;
5436
		ctxt->src.orig_val64 = ctxt->src.val64;
5437 5438
	}

5439 5440 5441
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
5442 5443 5444 5445
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

5446
	if ((ctxt->d & DstMask) == ImplicitOps)
5447 5448 5449
		goto special_insn;


5450
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5451
		/* optimisation - avoid slow emulated read if Mov */
5452 5453
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
5454
		if (rc != X86EMUL_CONTINUE) {
5455 5456
			if (!(ctxt->d & NoWrite) &&
			    rc == X86EMUL_PROPAGATE_FAULT &&
5457 5458
			    ctxt->exception.vector == PF_VECTOR)
				ctxt->exception.error_code |= PFERR_WRITE_MASK;
5459
			goto done;
5460
		}
5461
	}
5462 5463
	/* Copy full 64-bit value for CMPXCHG8B.  */
	ctxt->dst.orig_val64 = ctxt->dst.val64;
5464

5465 5466
special_insn:

5467
	if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5468
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
5469
					      X86_ICPT_POST_MEMACCESS);
5470 5471 5472 5473
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

5474
	if (ctxt->rep_prefix && (ctxt->d & String))
5475
		ctxt->eflags |= X86_EFLAGS_RF;
5476
	else
5477
		ctxt->eflags &= ~X86_EFLAGS_RF;
5478

5479
	if (ctxt->execute) {
5480 5481 5482 5483 5484 5485 5486
		if (ctxt->d & Fastop) {
			void (*fop)(struct fastop *) = (void *)ctxt->execute;
			rc = fastop(ctxt, fop);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			goto writeback;
		}
5487
		rc = ctxt->execute(ctxt);
5488 5489 5490 5491 5492
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

B
Borislav Petkov 已提交
5493
	if (ctxt->opcode_len == 2)
A
Avi Kivity 已提交
5494
		goto twobyte_insn;
5495 5496
	else if (ctxt->opcode_len == 3)
		goto threebyte_insn;
A
Avi Kivity 已提交
5497

5498
	switch (ctxt->b) {
5499
	case 0x70 ... 0x7f: /* jcc (short) */
5500
		if (test_cc(ctxt->b, ctxt->eflags))
5501
			rc = jmp_rel(ctxt, ctxt->src.val);
5502
		break;
N
Nitin A Kamble 已提交
5503
	case 0x8d: /* lea r16/r32, m */
5504
		ctxt->dst.val = ctxt->src.addr.mem.ea;
N
Nitin A Kamble 已提交
5505
		break;
5506
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
5507
		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5508 5509 5510
			ctxt->dst.type = OP_NONE;
		else
			rc = em_xchg(ctxt);
5511
		break;
5512
	case 0x98: /* cbw/cwde/cdqe */
5513 5514 5515 5516
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5517 5518
		}
		break;
5519
	case 0xcc:		/* int3 */
5520 5521
		rc = emulate_int(ctxt, 3);
		break;
5522
	case 0xcd:		/* int n */
5523
		rc = emulate_int(ctxt, ctxt->src.val);
5524 5525
		break;
	case 0xce:		/* into */
5526
		if (ctxt->eflags & X86_EFLAGS_OF)
5527
			rc = emulate_int(ctxt, 4);
5528
		break;
5529
	case 0xe9: /* jmp rel */
5530
	case 0xeb: /* jmp rel short */
5531
		rc = jmp_rel(ctxt, ctxt->src.val);
5532
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
5533
		break;
5534
	case 0xf4:              /* hlt */
5535
		ctxt->ops->halt(ctxt);
5536
		break;
5537 5538
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
5539
		ctxt->eflags ^= X86_EFLAGS_CF;
5540 5541
		break;
	case 0xf8: /* clc */
5542
		ctxt->eflags &= ~X86_EFLAGS_CF;
5543
		break;
5544
	case 0xf9: /* stc */
5545
		ctxt->eflags |= X86_EFLAGS_CF;
5546
		break;
5547
	case 0xfc: /* cld */
5548
		ctxt->eflags &= ~X86_EFLAGS_DF;
5549 5550
		break;
	case 0xfd: /* std */
5551
		ctxt->eflags |= X86_EFLAGS_DF;
5552
		break;
5553 5554
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
5555
	}
5556

5557 5558 5559
	if (rc != X86EMUL_CONTINUE)
		goto done;

5560
writeback:
5561 5562 5563 5564 5565 5566
	if (ctxt->d & SrcWrite) {
		BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
		rc = writeback(ctxt, &ctxt->src);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
5567 5568 5569 5570 5571
	if (!(ctxt->d & NoWrite)) {
		rc = writeback(ctxt, &ctxt->dst);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
5572

5573 5574 5575 5576
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
5577
	ctxt->dst.type = saved_dst_type;
5578

5579
	if ((ctxt->d & SrcMask) == SrcSI)
5580
		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5581

5582
	if ((ctxt->d & DstMask) == DstDI)
5583
		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5584

5585
	if (ctxt->rep_prefix && (ctxt->d & String)) {
5586
		unsigned int count;
5587
		struct read_cache *r = &ctxt->io_read;
5588 5589 5590 5591
		if ((ctxt->d & SrcMask) == SrcSI)
			count = ctxt->src.count;
		else
			count = ctxt->dst.count;
5592
		register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5593

5594 5595 5596 5597 5598
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
5599
			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5600 5601 5602 5603 5604 5605
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
5606
				ctxt->mem_read.end = 0;
5607
				writeback_registers(ctxt);
5608 5609 5610
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
5611
		}
5612
		ctxt->eflags &= ~X86_EFLAGS_RF;
5613
	}
5614

5615
	ctxt->eip = ctxt->_eip;
5616 5617

done:
5618 5619
	if (rc == X86EMUL_PROPAGATE_FAULT) {
		WARN_ON(ctxt->exception.vector > 0x1f);
5620
		ctxt->have_exception = true;
5621
	}
5622 5623 5624
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

5625 5626 5627
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);

5628
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
5629 5630

twobyte_insn:
5631
	switch (ctxt->b) {
5632
	case 0x09:		/* wbinvd */
5633
		(ctxt->ops->wbinvd)(ctxt);
5634 5635
		break;
	case 0x08:		/* invd */
5636 5637
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
P
Paolo Bonzini 已提交
5638
	case 0x1f:		/* nop */
5639 5640
		break;
	case 0x20: /* mov cr, reg */
5641
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5642
		break;
A
Avi Kivity 已提交
5643
	case 0x21: /* mov from dr to reg */
5644
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
A
Avi Kivity 已提交
5645 5646
		break;
	case 0x40 ... 0x4f:	/* cmov */
5647 5648
		if (test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.val = ctxt->src.val;
5649
		else if (ctxt->op_bytes != 4)
5650
			ctxt->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
5651
		break;
5652
	case 0x80 ... 0x8f: /* jnz rel, etc*/
5653
		if (test_cc(ctxt->b, ctxt->eflags))
5654
			rc = jmp_rel(ctxt, ctxt->src.val);
5655
		break;
5656
	case 0x90 ... 0x9f:     /* setcc r/m8 */
5657
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5658
		break;
A
Avi Kivity 已提交
5659
	case 0xb6 ... 0xb7:	/* movzx */
5660
		ctxt->dst.bytes = ctxt->op_bytes;
5661
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5662
						       : (u16) ctxt->src.val;
A
Avi Kivity 已提交
5663 5664
		break;
	case 0xbe ... 0xbf:	/* movsx */
5665
		ctxt->dst.bytes = ctxt->op_bytes;
5666
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5667
							(s16) ctxt->src.val;
A
Avi Kivity 已提交
5668
		break;
5669 5670
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
5671
	}
5672

5673 5674
threebyte_insn:

5675 5676 5677
	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
5678 5679 5680
	goto writeback;

cannot_emulate:
5681
	return EMULATION_FAILED;
A
Avi Kivity 已提交
5682
}
5683 5684 5685 5686 5687 5688 5689 5690 5691 5692

void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
{
	invalidate_registers(ctxt);
}

void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
{
	writeback_registers(ctxt);
}
5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703

bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->rep_prefix && (ctxt->d & String))
		return false;

	if (ctxt->d & TwoMemOp)
		return false;

	return true;
}