emulate.c 150.9 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
A
Avi Kivity 已提交
2
/******************************************************************************
3
 * emulate.c
A
Avi Kivity 已提交
4 5 6 7 8 9
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10
 * privileged instructions:
A
Avi Kivity 已提交
11 12
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
13
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
14 15 16 17 18 19 20
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

21
#include <linux/kvm_host.h>
22
#include "kvm_cache_regs.h"
23
#include "kvm_emulate.h"
24
#include <linux/stringify.h>
25
#include <asm/fpu/api.h>
26
#include <asm/debugreg.h>
27
#include <asm/nospec-branch.h>
A
Avi Kivity 已提交
28

29
#include "x86.h"
30
#include "tss.h"
31
#include "mmu.h"
32
#include "pmu.h"
33

34 35 36
/*
 * Operand types
 */
37 38 39 40 41 42 43 44 45
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
46 47 48
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
49
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
50 51 52 53 54 55 56
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
57 58 59 60 61 62
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
63
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
64
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
P
Paolo Bonzini 已提交
65
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
66 67
#define OpAccLo           29ull  /* Low part of extended acc (AX/AX/EAX/RAX) */
#define OpAccHi           30ull  /* High part of extended acc (-/DX/EDX/RDX) */
68 69

#define OpBits             5  /* Width of operand field */
70
#define OpMask             ((1ull << OpBits) - 1)
71

A
Avi Kivity 已提交
72 73 74 75 76 77 78 79 80 81
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
82
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
83
/* Destination operand type. */
84 85 86 87 88 89 90
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
91
#define DstMem16    (OpMem16 << DstShift)
92 93
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
94
#define DstAccLo    (OpAccLo << DstShift)
95
#define DstMask     (OpMask << DstShift)
A
Avi Kivity 已提交
96
/* Source operand type. */
97 98 99 100 101 102 103 104 105 106 107 108
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
P
Paolo Bonzini 已提交
109
#define SrcXLat     (OpXLat << SrcShift)
110 111 112 113
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
114
#define SrcImm64    (OpImm64 << SrcShift)
115
#define SrcDX       (OpDX << SrcShift)
116
#define SrcMem8     (OpMem8 << SrcShift)
117
#define SrcAccHi    (OpAccHi << SrcShift)
118
#define SrcMask     (OpMask << SrcShift)
119 120 121 122 123 124 125 126 127
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
128
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
129
#define InstrDual   (6<<15)     /* Alternate instruction decoding of mod == 3 */
130
#define ModeDual    (7<<15)     /* Different instruction for 32/64 bit */
131
#define Sse         (1<<18)     /* SSE Vector instruction */
132 133 134 135
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
136
/* Misc flags */
137
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
138
#define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141
#define Undefined   (1<<25) /* No Such Instruction */
142
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
143
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
144
#define No64	    (1<<28)
145
#define PageTable   (1 << 29)   /* instruction used to write page table */
146
#define NotImpl     (1 << 30)   /* instruction is not implemented */
147
/* Source 2 operand type */
148
#define Src2Shift   (31)
149
#define Src2None    (OpNone << Src2Shift)
150
#define Src2Mem     (OpMem << Src2Shift)
151 152 153 154
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
155 156 157 158 159 160
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
161
#define Src2Mask    (OpMask << Src2Shift)
A
Avi Kivity 已提交
162
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
163
#define AlignMask   ((u64)7 << 41)
164
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
165 166 167
#define Unaligned   ((u64)2 << 41)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)3 << 41)  /* Advanced Vector Extensions */
#define Aligned16   ((u64)4 << 41)  /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
169
#define NoWrite     ((u64)1 << 45)  /* No writeback */
170
#define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
171
#define NoMod	    ((u64)1 << 47)  /* Mod field is ignored */
172 173
#define Intercept   ((u64)1 << 48)  /* Has valid intercept field */
#define CheckPerm   ((u64)1 << 49)  /* Has valid check_perm field */
174
#define PrivUD      ((u64)1 << 51)  /* #UD instead of #GP on CPL > 0 */
175
#define NearBranch  ((u64)1 << 52)  /* Near branches */
176
#define No16	    ((u64)1 << 53)  /* No 16 bit operand */
177
#define IncSP       ((u64)1 << 54)  /* SP is incremented before ModRM calc */
178
#define TwoMemOp    ((u64)1 << 55)  /* Instruction has two memory operand */
A
Avi Kivity 已提交
179

180
#define DstXacc     (DstAccLo | SrcAccHi | SrcWrite)
A
Avi Kivity 已提交
181

182 183 184 185 186 187 188 189
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
190

191 192 193
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

194
struct opcode {
195 196
	u64 flags : 56;
	u64 intercept : 8;
197
	union {
198
		int (*execute)(struct x86_emulate_ctxt *ctxt);
199 200 201
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
202
		const struct escape *esc;
203
		const struct instr_dual *idual;
204
		const struct mode_dual *mdual;
205
		void (*fastop)(struct fastop *fake);
206
	} u;
207
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
208 209 210 211 212
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
213 214
};

215 216 217 218 219 220 221
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

222 223 224 225 226
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

227 228 229 230 231
struct instr_dual {
	struct opcode mod012;
	struct opcode mod3;
};

232 233 234 235 236
struct mode_dual {
	struct opcode mode32;
	struct opcode mode64;
};

237 238
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a

239 240 241 242 243 244 245
enum x86_transfer_type {
	X86_TRANSFER_NONE,
	X86_TRANSFER_CALL_JMP,
	X86_TRANSFER_RET,
	X86_TRANSFER_TASK_SWITCH,
};

246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

A
Avi Kivity 已提交
282 283 284 285
/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
286 287
#define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
		     X86_EFLAGS_PF|X86_EFLAGS_CF)
A
Avi Kivity 已提交
288

289 290 291 292 293 294
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

295 296 297 298 299 300 301 302 303 304 305 306 307
/*
 * fastop functions have a special calling convention:
 *
 * dst:    rax        (in/out)
 * src:    rdx        (in/out)
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
 * ex:     rsi        (in:fastop pointer, out:zero if exception)
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 */
308
static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
309

310
#define __FOP_FUNC(name) \
311 312 313 314
	".align " __stringify(FASTOP_SIZE) " \n\t" \
	".type " name ", @function \n\t" \
	name ":\n\t"

315 316 317 318 319 320 321 322 323
#define FOP_FUNC(name) \
	__FOP_FUNC(#name)

#define __FOP_RET(name) \
	"ret \n\t" \
	".size " name ", .-" name "\n\t"

#define FOP_RET(name) \
	__FOP_RET(#name)
324 325 326 327 328

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
329 330
	    ".align " __stringify(FASTOP_SIZE) " \n\t" \
	    "em_" #op ":\n\t"
331 332 333 334

#define FOP_END \
	    ".popsection")

335 336 337 338
#define __FOPNOP(name) \
	__FOP_FUNC(name) \
	__FOP_RET(name)

339
#define FOPNOP() \
340
	__FOPNOP(__stringify(__UNIQUE_ID(nop)))
341

342
#define FOP1E(op,  dst) \
343 344 345
	__FOP_FUNC(#op "_" #dst) \
	"10: " #op " %" #dst " \n\t" \
	__FOP_RET(#op "_" #dst)
346 347 348

#define FOP1EEX(op,  dst) \
	FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
349 350 351 352 353 354 355 356 357

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

358 359 360 361 362 363 364 365 366
/* 1-operand, using src2 (for MUL/DIV r/m) */
#define FASTOP1SRC2(op, name) \
	FOP_START(name) \
	FOP1E(op, cl) \
	FOP1E(op, cx) \
	FOP1E(op, ecx) \
	ON64(FOP1E(op, rcx)) \
	FOP_END

367 368 369 370 371 372 373 374 375
/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
#define FASTOP1SRC2EX(op, name) \
	FOP_START(name) \
	FOP1EEX(op, cl) \
	FOP1EEX(op, cx) \
	FOP1EEX(op, ecx) \
	ON64(FOP1EEX(op, rcx)) \
	FOP_END

376
#define FOP2E(op,  dst, src)	   \
377 378 379
	__FOP_FUNC(#op "_" #dst "_" #src) \
	#op " %" #src ", %" #dst " \n\t" \
	__FOP_RET(#op "_" #dst "_" #src)
380 381 382

#define FASTOP2(op) \
	FOP_START(op) \
383 384 385 386
	FOP2E(op##b, al, dl) \
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
387 388
	FOP_END

389 390 391 392
/* 2 operand, word only */
#define FASTOP2W(op) \
	FOP_START(op) \
	FOPNOP() \
393 394 395
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
396 397
	FOP_END

398 399 400 401 402 403 404 405 406
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

407 408 409 410 411 412 413 414 415
/* 2 operand, src and dest are reversed */
#define FASTOP2R(op, name) \
	FOP_START(name) \
	FOP2E(op##b, dl, al) \
	FOP2E(op##w, dx, ax) \
	FOP2E(op##l, edx, eax) \
	ON64(FOP2E(op##q, rdx, rax)) \
	FOP_END

416
#define FOP3E(op,  dst, src, src2) \
417 418 419
	__FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
	#op " %" #src2 ", %" #src ", %" #dst " \n\t"\
	__FOP_RET(#op "_" #dst "_" #src "_" #src2)
420 421 422 423 424

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
425 426 427
	FOP3E(op##w, ax, dx, cl) \
	FOP3E(op##l, eax, edx, cl) \
	ON64(FOP3E(op##q, rax, rdx, cl)) \
428 429
	FOP_END

430
/* Special case for SETcc - 1 instruction per cc */
431 432 433 434 435
#define FOP_SETCC(op) \
	".align 4 \n\t" \
	".type " #op ", @function \n\t" \
	#op ": \n\t" \
	#op " %al \n\t" \
436
	__FOP_RET(#op)
437

438 439 440 441
asm(".pushsection .fixup, \"ax\"\n"
    ".global kvm_fastop_exception \n"
    "kvm_fastop_exception: xor %esi, %esi; ret\n"
    ".popsection");
442

443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461
FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

462 463 464 465
FOP_START(salc)
FOP_FUNC(salc)
"pushf; sbb %al, %al; popf \n\t"
FOP_RET(salc)
P
Paolo Bonzini 已提交
466 467
FOP_END;

R
Radim Krčmář 已提交
468 469
/*
 * XXX: inoutclob user must know where the argument is being expanded.
470
 *      Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
R
Radim Krčmář 已提交
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487
 */
#define asm_safe(insn, inoutclob...) \
({ \
	int _fault = 0; \
 \
	asm volatile("1:" insn "\n" \
	             "2:\n" \
	             ".pushsection .fixup, \"ax\"\n" \
	             "3: movl $1, %[_fault]\n" \
	             "   jmp  2b\n" \
	             ".popsection\n" \
	             _ASM_EXTABLE(1b, 3b) \
	             : [_fault] "+qm"(_fault) inoutclob ); \
 \
	_fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
})

488 489 490 491 492 493
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
494 495 496 497 498
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
499
		.dst_val    = ctxt->dst.val64,
500 501 502
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
503 504 505
		.next_rip   = ctxt->eip,
	};

506
	return ctxt->ops->intercept(ctxt, &info, stage);
507 508
}

A
Avi Kivity 已提交
509 510 511 512 513
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
static void assign_register(unsigned long *reg, u64 val, int bytes)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (bytes) {
	case 1:
		*(u8 *)reg = (u8)val;
		break;
	case 2:
		*(u16 *)reg = (u16)val;
		break;
	case 4:
		*reg = (u32)val;
		break;	/* 64b: zero-extend */
	case 8:
		*reg = val;
		break;
	}
}

533
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
534
{
535
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
536 537
}

A
Avi Kivity 已提交
538 539 540 541 542 543 544 545 546 547 548
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

A
Avi Kivity 已提交
549 550 551 552 553
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

A
Avi Kivity 已提交
554
/* Access/update address held in a register, based on addressing mode. */
555
static inline unsigned long
556
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
557
{
558
	if (ctxt->ad_bytes == sizeof(unsigned long))
559 560
		return reg;
	else
561
		return reg & ad_mask(ctxt);
562 563 564
}

static inline unsigned long
565
register_address(struct x86_emulate_ctxt *ctxt, int reg)
566
{
567
	return address_mask(ctxt, reg_read(ctxt, reg));
568 569
}

570 571 572 573 574
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

575
static inline void
576
register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
577
{
578
	ulong *preg = reg_rmw(ctxt, reg);
579

580
	assign_register(preg, *preg + inc, ctxt->ad_bytes);
581 582 583 584
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
585
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
586
}
A
Avi Kivity 已提交
587

588 589 590 591 592 593 594
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

595
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
596 597 598 599
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

600
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
601 602
}

603 604
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
605
{
606
	WARN_ON(vec > 0x1f);
607 608 609
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
610
	return X86EMUL_PROPAGATE_FAULT;
611 612
}

613 614 615 616 617
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

618
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
619
{
620
	return emulate_exception(ctxt, GP_VECTOR, err, true);
621 622
}

623 624 625 626 627
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

628
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
629
{
630
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
631 632
}

633
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
634
{
635
	return emulate_exception(ctxt, TS_VECTOR, err, true);
636 637
}

638 639
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
640
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
641 642
}

A
Avi Kivity 已提交
643 644 645 646 647
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

668 669 670 671 672 673 674 675 676 677 678
static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
{
	return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
}

static inline bool emul_is_noncanonical_address(u64 la,
						struct x86_emulate_ctxt *ctxt)
{
	return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la;
}

679 680 681 682 683 684
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
685 686
 * subject to the same check.  FXSAVE and FXRSTOR are checked here too as their
 * 512 bytes of data must be aligned to a 16 byte boundary.
687
 */
688
static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
689
{
690
	u64 alignment = ctxt->d & AlignMask;
691 692

	if (likely(size < 16))
693
		return 1;
694

695 696 697
	switch (alignment) {
	case Unaligned:
	case Avx:
698
		return 1;
699
	case Aligned16:
700
		return 16;
701 702
	case Aligned:
	default:
703
		return size;
704
	}
705 706
}

707 708 709 710
static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
				       struct segmented_address addr,
				       unsigned *max_size, unsigned size,
				       bool write, bool fetch,
711
				       enum x86emul_mode mode, ulong *linear)
712
{
713 714
	struct desc_struct desc;
	bool usable;
715
	ulong la;
716
	u32 lim;
717
	u16 sel;
718
	u8  va_bits;
719

720
	la = seg_base(ctxt, addr.seg) + addr.ea;
721
	*max_size = 0;
722
	switch (mode) {
723
	case X86EMUL_MODE_PROT64:
724
		*linear = la;
725 726
		va_bits = ctxt_virt_addr_bits(ctxt);
		if (get_canonical(la, va_bits) != la)
727
			goto bad;
728

729
		*max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
730 731
		if (size > *max_size)
			goto bad;
732 733
		break;
	default:
734
		*linear = la = (u32)la;
735 736
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
737 738
		if (!usable)
			goto bad;
739 740 741
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
742 743
			goto bad;
		/* unreadable code segment */
744
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
745 746
			goto bad;
		lim = desc_limit_scaled(&desc);
747
		if (!(desc.type & 8) && (desc.type & 4)) {
G
Guo Chao 已提交
748
			/* expand-down segment */
749
			if (addr.ea <= lim)
750 751 752
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
		}
753 754
		if (addr.ea > lim)
			goto bad;
755 756 757 758 759 760 761
		if (lim == 0xffffffff)
			*max_size = ~0u;
		else {
			*max_size = (u64)lim + 1 - addr.ea;
			if (size > *max_size)
				goto bad;
		}
762 763
		break;
	}
764
	if (la & (insn_alignment(ctxt, size) - 1))
765
		return emulate_gp(ctxt, 0);
766
	return X86EMUL_CONTINUE;
767 768
bad:
	if (addr.seg == VCPU_SREG_SS)
769
		return emulate_ss(ctxt, 0);
770
	else
771
		return emulate_gp(ctxt, 0);
772 773
}

774 775 776 777 778
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
779
	unsigned max_size;
780 781
	return __linearize(ctxt, addr, &max_size, size, write, false,
			   ctxt->mode, linear);
782 783
}

784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
			     enum x86emul_mode mode)
{
	ulong linear;
	int rc;
	unsigned max_size;
	struct segmented_address addr = { .seg = VCPU_SREG_CS,
					   .ea = dst };

	if (ctxt->op_bytes != sizeof(unsigned long))
		addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
	rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
	if (rc == X86EMUL_CONTINUE)
		ctxt->_eip = addr.ea;
	return rc;
}

static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
{
	return assign_eip(ctxt, dst, ctxt->mode);
804 805
}

806 807 808 809
static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
			  const struct desc_struct *cs_desc)
{
	enum x86emul_mode mode = ctxt->mode;
810
	int rc;
811 812

#ifdef CONFIG_X86_64
813 814 815
	if (ctxt->mode >= X86EMUL_MODE_PROT16) {
		if (cs_desc->l) {
			u64 efer = 0;
816

817 818 819 820 821
			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
			if (efer & EFER_LMA)
				mode = X86EMUL_MODE_PROT64;
		} else
			mode = X86EMUL_MODE_PROT32; /* temporary value */
822 823 824 825
	}
#endif
	if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
		mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
826 827 828 829
	rc = assign_eip(ctxt, dst, mode);
	if (rc == X86EMUL_CONTINUE)
		ctxt->mode = mode;
	return rc;
830 831 832 833 834 835
}

static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
{
	return assign_eip_near(ctxt, ctxt->_eip + rel);
}
836

837 838 839
static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
			      void *data, unsigned size)
{
840
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
841 842 843 844 845 846
}

static int linear_write_system(struct x86_emulate_ctxt *ctxt,
			       ulong linear, void *data,
			       unsigned int size)
{
847
	return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
848 849
}

850 851 852 853 854
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
855 856 857
	int rc;
	ulong linear;

858
	rc = linearize(ctxt, addr, size, false, &linear);
859 860
	if (rc != X86EMUL_CONTINUE)
		return rc;
861
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
862 863
}

864 865 866 867 868 869 870 871 872 873 874
static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
			       struct segmented_address addr,
			       void *data,
			       unsigned int size)
{
	int rc;
	ulong linear;

	rc = linearize(ctxt, addr, size, true, &linear);
	if (rc != X86EMUL_CONTINUE)
		return rc;
875
	return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
876 877
}

878
/*
879
 * Prefetch the remaining bytes of the instruction without crossing page
880 881
 * boundary if they are not in fetch_cache yet.
 */
882
static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
883 884
{
	int rc;
885
	unsigned size, max_size;
886
	unsigned long linear;
887
	int cur_size = ctxt->fetch.end - ctxt->fetch.data;
888
	struct segmented_address addr = { .seg = VCPU_SREG_CS,
889 890
					   .ea = ctxt->eip + cur_size };

891 892 893 894 895 896 897 898 899 900
	/*
	 * We do not know exactly how many bytes will be needed, and
	 * __linearize is expensive, so fetch as much as possible.  We
	 * just have to avoid going beyond the 15 byte limit, the end
	 * of the segment, or the end of the page.
	 *
	 * __linearize is called with size 0 so that it does not do any
	 * boundary check itself.  Instead, we use max_size to check
	 * against op_size.
	 */
901 902
	rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
			 &linear);
903 904 905
	if (unlikely(rc != X86EMUL_CONTINUE))
		return rc;

906
	size = min_t(unsigned, 15UL ^ cur_size, max_size);
907
	size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
908 909 910 911 912 913 914 915

	/*
	 * One instruction can only straddle two pages,
	 * and one has been loaded at the beginning of
	 * x86_decode_insn.  So, if not enough bytes
	 * still, we must have hit the 15-byte boundary.
	 */
	if (unlikely(size < op_size))
916 917
		return emulate_gp(ctxt, 0);

918
	rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
919 920 921
			      size, &ctxt->exception);
	if (unlikely(rc != X86EMUL_CONTINUE))
		return rc;
922
	ctxt->fetch.end += size;
923
	return X86EMUL_CONTINUE;
924 925
}

926 927
static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
					       unsigned size)
928
{
929 930 931 932
	unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;

	if (unlikely(done_size < size))
		return __do_insn_fetch_bytes(ctxt, size - done_size);
933 934
	else
		return X86EMUL_CONTINUE;
935 936
}

937
/* Fetch next part of the instruction being emulated. */
938
#define insn_fetch(_type, _ctxt)					\
939 940 941
({	_type _x;							\
									\
	rc = do_insn_fetch_bytes(_ctxt, sizeof(_type));			\
942 943
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
944
	ctxt->_eip += sizeof(_type);					\
945
	memcpy(&_x, ctxt->fetch.ptr, sizeof(_type));			\
946
	ctxt->fetch.ptr += sizeof(_type);				\
947
	_x;								\
948 949
})

950
#define insn_fetch_arr(_arr, _size, _ctxt)				\
951 952
({									\
	rc = do_insn_fetch_bytes(_ctxt, _size);				\
953 954
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
955
	ctxt->_eip += (_size);						\
956 957
	memcpy(_arr, ctxt->fetch.ptr, _size);				\
	ctxt->fetch.ptr += (_size);					\
958 959
})

960 961 962 963 964
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
965
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
966
			     int byteop)
A
Avi Kivity 已提交
967 968
{
	void *p;
969
	int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
A
Avi Kivity 已提交
970 971

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
972 973 974
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
A
Avi Kivity 已提交
975 976 977 978
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
979
			   struct segmented_address addr,
A
Avi Kivity 已提交
980 981 982 983 984 985 986
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
987
	rc = segmented_read_std(ctxt, addr, size, 2);
988
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
989
		return rc;
990
	addr.ea += 2;
991
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
992 993 994
	return rc;
}

995 996 997 998 999 1000 1001 1002 1003 1004
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);

1005 1006
FASTOP1SRC2(mul, mul_ex);
FASTOP1SRC2(imul, imul_ex);
1007 1008
FASTOP1SRC2EX(div, div_ex);
FASTOP1SRC2EX(idiv, idiv_ex);
1009

1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
FASTOP3WCL(shld);
FASTOP3WCL(shrd);

FASTOP2W(imul);

FASTOP1(not);
FASTOP1(neg);
FASTOP1(inc);
FASTOP1(dec);

FASTOP2CL(rol);
FASTOP2CL(ror);
FASTOP2CL(rcl);
FASTOP2CL(rcr);
FASTOP2CL(shl);
FASTOP2CL(shr);
FASTOP2CL(sar);

FASTOP2W(bsf);
FASTOP2W(bsr);
FASTOP2W(bt);
FASTOP2W(bts);
FASTOP2W(btr);
FASTOP2W(btc);

1035 1036
FASTOP2(xadd);

1037 1038
FASTOP2R(cmp, cmp_r);

1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
{
	/* If src is zero, do not writeback, but update flags */
	if (ctxt->src.val == 0)
		ctxt->dst.type = OP_NONE;
	return fastop(ctxt, em_bsf);
}

static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
{
	/* If src is zero, do not writeback, but update flags */
	if (ctxt->src.val == 0)
		ctxt->dst.type = OP_NONE;
	return fastop(ctxt, em_bsr);
}

1055
static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1056
{
1057 1058
	u8 rc;
	void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1059

1060
	flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1061 1062
	asm("push %[flags]; popf; " CALL_NOSPEC
	    : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1063
	return rc;
1064 1065
}

1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
static void emulator_get_fpu(void)
{
	fpregs_lock();

	fpregs_assert_state_consistent();
	if (test_thread_flag(TIF_NEED_FPU_LOAD))
		switch_fpu_return();
}

static void emulator_put_fpu(void)
{
	fpregs_unlock();
}

1098
static void read_sse_reg(sse128_t *data, int reg)
A
Avi Kivity 已提交
1099
{
1100
	emulator_get_fpu();
A
Avi Kivity 已提交
1101
	switch (reg) {
1102 1103 1104 1105 1106 1107 1108 1109
	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
1110
#ifdef CONFIG_X86_64
1111 1112 1113 1114 1115 1116 1117 1118
	case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
1119 1120 1121
#endif
	default: BUG();
	}
1122
	emulator_put_fpu();
A
Avi Kivity 已提交
1123 1124
}

1125
static void write_sse_reg(sse128_t *data, int reg)
A
Avi Kivity 已提交
1126
{
1127
	emulator_get_fpu();
A
Avi Kivity 已提交
1128
	switch (reg) {
1129 1130 1131 1132 1133 1134 1135 1136
	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
A
Avi Kivity 已提交
1137
#ifdef CONFIG_X86_64
1138 1139 1140 1141 1142 1143 1144 1145
	case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
A
Avi Kivity 已提交
1146 1147 1148
#endif
	default: BUG();
	}
1149
	emulator_put_fpu();
A
Avi Kivity 已提交
1150 1151
}

1152
static void read_mmx_reg(u64 *data, int reg)
A
Avi Kivity 已提交
1153
{
1154
	emulator_get_fpu();
A
Avi Kivity 已提交
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
	switch (reg) {
	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
	case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
	case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
	case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
	case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
	case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
	default: BUG();
	}
1166
	emulator_put_fpu();
A
Avi Kivity 已提交
1167 1168
}

1169
static void write_mmx_reg(u64 *data, int reg)
A
Avi Kivity 已提交
1170
{
1171
	emulator_get_fpu();
A
Avi Kivity 已提交
1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
	switch (reg) {
	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
	case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
	case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
	case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
	case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
	case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
	default: BUG();
	}
1183
	emulator_put_fpu();
A
Avi Kivity 已提交
1184 1185
}

1186 1187 1188 1189 1190
static int em_fninit(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

1191
	emulator_get_fpu();
1192
	asm volatile("fninit");
1193
	emulator_put_fpu();
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203
	return X86EMUL_CONTINUE;
}

static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
{
	u16 fcw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

1204
	emulator_get_fpu();
1205
	asm volatile("fnstcw %0": "+m"(fcw));
1206
	emulator_put_fpu();
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219

	ctxt->dst.val = fcw;

	return X86EMUL_CONTINUE;
}

static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
{
	u16 fsw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

1220
	emulator_get_fpu();
1221
	asm volatile("fnstsw %0": "+m"(fsw));
1222
	emulator_put_fpu();
1223 1224 1225 1226 1227 1228

	ctxt->dst.val = fsw;

	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1229
static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1230
				    struct operand *op)
1231
{
1232
	unsigned reg = ctxt->modrm_reg;
1233

1234 1235
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
1236

1237
	if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1238 1239 1240
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
1241
		read_sse_reg(&op->vec_val, reg);
A
Avi Kivity 已提交
1242 1243
		return;
	}
A
Avi Kivity 已提交
1244 1245 1246 1247 1248 1249 1250
	if (ctxt->d & Mmx) {
		reg &= 7;
		op->type = OP_MM;
		op->bytes = 8;
		op->addr.mm = reg;
		return;
	}
A
Avi Kivity 已提交
1251

1252
	op->type = OP_REG;
1253 1254 1255
	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);

1256
	fetch_register_operand(op);
1257 1258 1259
	op->orig_val = op->val;
}

1260 1261 1262 1263 1264 1265
static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
{
	if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
		ctxt->modrm_seg = VCPU_SREG_SS;
}

1266
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1267
			struct operand *op)
1268 1269
{
	u8 sib;
B
Bandan Das 已提交
1270
	int index_reg, base_reg, scale;
1271
	int rc = X86EMUL_CONTINUE;
1272
	ulong modrm_ea = 0;
1273

B
Bandan Das 已提交
1274 1275 1276
	ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
	index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
	base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1277

B
Bandan Das 已提交
1278
	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1279
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
B
Bandan Das 已提交
1280
	ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1281
	ctxt->modrm_seg = VCPU_SREG_DS;
1282

1283
	if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1284
		op->type = OP_REG;
1285
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1286
		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1287
				ctxt->d & ByteOp);
1288
		if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1289 1290
			op->type = OP_XMM;
			op->bytes = 16;
1291
			op->addr.xmm = ctxt->modrm_rm;
1292
			read_sse_reg(&op->vec_val, ctxt->modrm_rm);
A
Avi Kivity 已提交
1293 1294
			return rc;
		}
A
Avi Kivity 已提交
1295 1296 1297
		if (ctxt->d & Mmx) {
			op->type = OP_MM;
			op->bytes = 8;
1298
			op->addr.mm = ctxt->modrm_rm & 7;
A
Avi Kivity 已提交
1299 1300
			return rc;
		}
1301
		fetch_register_operand(op);
1302 1303 1304
		return rc;
	}

1305 1306
	op->type = OP_MEM;

1307
	if (ctxt->ad_bytes == 2) {
1308 1309 1310 1311
		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1312 1313

		/* 16-bit ModR/M decode. */
1314
		switch (ctxt->modrm_mod) {
1315
		case 0:
1316
			if (ctxt->modrm_rm == 6)
1317
				modrm_ea += insn_fetch(u16, ctxt);
1318 1319
			break;
		case 1:
1320
			modrm_ea += insn_fetch(s8, ctxt);
1321 1322
			break;
		case 2:
1323
			modrm_ea += insn_fetch(u16, ctxt);
1324 1325
			break;
		}
1326
		switch (ctxt->modrm_rm) {
1327
		case 0:
1328
			modrm_ea += bx + si;
1329 1330
			break;
		case 1:
1331
			modrm_ea += bx + di;
1332 1333
			break;
		case 2:
1334
			modrm_ea += bp + si;
1335 1336
			break;
		case 3:
1337
			modrm_ea += bp + di;
1338 1339
			break;
		case 4:
1340
			modrm_ea += si;
1341 1342
			break;
		case 5:
1343
			modrm_ea += di;
1344 1345
			break;
		case 6:
1346
			if (ctxt->modrm_mod != 0)
1347
				modrm_ea += bp;
1348 1349
			break;
		case 7:
1350
			modrm_ea += bx;
1351 1352
			break;
		}
1353 1354 1355
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
1356
		modrm_ea = (u16)modrm_ea;
1357 1358
	} else {
		/* 32/64-bit ModR/M decode. */
1359
		if ((ctxt->modrm_rm & 7) == 4) {
1360
			sib = insn_fetch(u8, ctxt);
1361 1362 1363 1364
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

1365
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1366
				modrm_ea += insn_fetch(s32, ctxt);
1367
			else {
1368
				modrm_ea += reg_read(ctxt, base_reg);
1369
				adjust_modrm_seg(ctxt, base_reg);
1370 1371 1372 1373
				/* Increment ESP on POP [ESP] */
				if ((ctxt->d & IncSP) &&
				    base_reg == VCPU_REGS_RSP)
					modrm_ea += ctxt->op_bytes;
1374
			}
1375
			if (index_reg != 4)
1376
				modrm_ea += reg_read(ctxt, index_reg) << scale;
1377
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1378
			modrm_ea += insn_fetch(s32, ctxt);
1379
			if (ctxt->mode == X86EMUL_MODE_PROT64)
1380
				ctxt->rip_relative = 1;
1381 1382
		} else {
			base_reg = ctxt->modrm_rm;
1383
			modrm_ea += reg_read(ctxt, base_reg);
1384 1385
			adjust_modrm_seg(ctxt, base_reg);
		}
1386
		switch (ctxt->modrm_mod) {
1387
		case 1:
1388
			modrm_ea += insn_fetch(s8, ctxt);
1389 1390
			break;
		case 2:
1391
			modrm_ea += insn_fetch(s32, ctxt);
1392 1393 1394
			break;
		}
	}
1395
	op->addr.mem.ea = modrm_ea;
1396 1397 1398
	if (ctxt->ad_bytes != 8)
		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;

1399 1400 1401 1402 1403
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
1404
		      struct operand *op)
1405
{
1406
	int rc = X86EMUL_CONTINUE;
1407

1408
	op->type = OP_MEM;
1409
	switch (ctxt->ad_bytes) {
1410
	case 2:
1411
		op->addr.mem.ea = insn_fetch(u16, ctxt);
1412 1413
		break;
	case 4:
1414
		op->addr.mem.ea = insn_fetch(u32, ctxt);
1415 1416
		break;
	case 8:
1417
		op->addr.mem.ea = insn_fetch(u64, ctxt);
1418 1419 1420 1421 1422 1423
		break;
	}
done:
	return rc;
}

1424
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1425
{
1426
	long sv = 0, mask;
1427

1428
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1429
		mask = ~((long)ctxt->dst.bytes * 8 - 1);
1430

1431 1432 1433 1434
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1435 1436
		else
			sv = (s64)ctxt->src.val & (s64)mask;
1437

1438 1439
		ctxt->dst.addr.mem.ea = address_mask(ctxt,
					   ctxt->dst.addr.mem.ea + (sv >> 3));
1440
	}
1441 1442

	/* only subword offset */
1443
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1444 1445
}

1446 1447
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1448
{
1449
	int rc;
1450
	struct read_cache *mc = &ctxt->mem_read;
A
Avi Kivity 已提交
1451

1452 1453
	if (mc->pos < mc->end)
		goto read_cached;
A
Avi Kivity 已提交
1454

1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
	WARN_ON((mc->end + size) >= sizeof(mc->data));

	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
				      &ctxt->exception);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	mc->end += size;

read_cached:
	memcpy(dest, mc->data + mc->pos, size);
	mc->pos += size;
1467 1468
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1469

1470 1471 1472 1473 1474
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1475 1476 1477
	int rc;
	ulong linear;

1478
	rc = linearize(ctxt, addr, size, false, &linear);
1479 1480
	if (rc != X86EMUL_CONTINUE)
		return rc;
1481
	return read_emulated(ctxt, linear, data, size);
1482 1483 1484 1485 1486 1487 1488
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1489 1490 1491
	int rc;
	ulong linear;

1492
	rc = linearize(ctxt, addr, size, true, &linear);
1493 1494
	if (rc != X86EMUL_CONTINUE)
		return rc;
1495 1496
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1497 1498 1499 1500 1501 1502 1503
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1504 1505 1506
	int rc;
	ulong linear;

1507
	rc = linearize(ctxt, addr, size, true, &linear);
1508 1509
	if (rc != X86EMUL_CONTINUE)
		return rc;
1510 1511
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1512 1513
}

1514 1515 1516 1517
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1518
	struct read_cache *rc = &ctxt->io_read;
1519

1520 1521
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1522
		unsigned int count = ctxt->rep_prefix ?
1523
			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1524
		in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1525 1526
			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1527
		n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1528 1529 1530
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1531
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1532 1533
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1534 1535
	}

1536
	if (ctxt->rep_prefix && (ctxt->d & String) &&
1537
	    !(ctxt->eflags & X86_EFLAGS_DF)) {
1538 1539 1540 1541 1542 1543 1544 1545
		ctxt->dst.data = rc->data + rc->pos;
		ctxt->dst.type = OP_MEM_STR;
		ctxt->dst.count = (rc->end - rc->pos) / size;
		rc->pos = rc->end;
	} else {
		memcpy(dest, rc->data + rc->pos, size);
		rc->pos += size;
	}
1546 1547
	return 1;
}
A
Avi Kivity 已提交
1548

1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
				     u16 index, struct desc_struct *desc)
{
	struct desc_ptr dt;
	ulong addr;

	ctxt->ops->get_idt(ctxt, &dt);

	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, index << 3 | 0x2);

	addr = dt.address + index * 8;
1561
	return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1562 1563
}

1564 1565 1566
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1567
	const struct x86_emulate_ops *ops = ctxt->ops;
1568
	u32 base3 = 0;
1569

1570 1571
	if (selector & 1 << 2) {
		struct desc_struct desc;
1572 1573
		u16 sel;

1574
		memset(dt, 0, sizeof(*dt));
1575 1576
		if (!ops->get_segment(ctxt, &sel, &desc, &base3,
				      VCPU_SREG_LDTR))
1577
			return;
1578

1579
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1580
		dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1581
	} else
1582
		ops->get_gdt(ctxt, dt);
1583
}
1584

1585 1586
static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
			      u16 selector, ulong *desc_addr_p)
1587 1588 1589 1590
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1591

1592
	get_descriptor_table_ptr(ctxt, selector, &dt);
1593

1594 1595
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1596

1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
	addr = dt.address + index * 8;

#ifdef CONFIG_X86_64
	if (addr >> 32 != 0) {
		u64 efer = 0;

		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (!(efer & EFER_LMA))
			addr &= (u32)-1;
	}
#endif

	*desc_addr_p = addr;
	return X86EMUL_CONTINUE;
}

/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, struct desc_struct *desc,
				   ulong *desc_addr_p)
{
	int rc;

	rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
	if (rc != X86EMUL_CONTINUE)
		return rc;

1624
	return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1625
}
1626

1627 1628 1629 1630
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
1631
	int rc;
1632
	ulong addr;
A
Avi Kivity 已提交
1633

1634 1635 1636
	rc = get_descriptor_ptr(ctxt, selector, &addr);
	if (rc != X86EMUL_CONTINUE)
		return rc;
A
Avi Kivity 已提交
1637

1638
	return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1639
}
1640

1641
static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1642
				     u16 selector, int seg, u8 cpl,
1643
				     enum x86_transfer_type transfer,
1644
				     struct desc_struct *desc)
1645
{
1646
	struct desc_struct seg_desc, old_desc;
1647
	u8 dpl, rpl;
1648 1649 1650
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1651
	ulong desc_addr;
1652
	int ret;
1653
	u16 dummy;
1654
	u32 base3 = 0;
1655

1656
	memset(&seg_desc, 0, sizeof(seg_desc));
1657

1658 1659 1660
	if (ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor (keep limit etc. for
		 * unreal mode) */
1661
		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1662 1663
		set_desc_base(&seg_desc, selector << 4);
		goto load;
1664 1665 1666 1667 1668 1669 1670 1671 1672
	} else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
		/* VM86 needs a clean new segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
		seg_desc.dpl = 3;
		goto load;
1673 1674
	}

1675 1676
	rpl = selector & 3;

1677 1678 1679 1680
	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702
	/* NULL selector is not valid for TR, CS and (except for long mode) SS */
	if (null_selector) {
		if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
			goto exception;

		if (seg == VCPU_SREG_SS) {
			if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
				goto exception;

			/*
			 * ctxt->ops->set_segment expects the CPL to be in
			 * SS.DPL, so fake an expand-up 32-bit data segment.
			 */
			seg_desc.type = 3;
			seg_desc.p = 1;
			seg_desc.s = 1;
			seg_desc.dpl = cpl;
			seg_desc.d = 1;
			seg_desc.g = 1;
		}

		/* Skip all following checks */
1703
		goto load;
1704
	}
1705

1706
	ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1707 1708 1709 1710
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
1711 1712
	err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
							   GP_VECTOR;
1713

G
Guo Chao 已提交
1714
	/* can't load system descriptor into segment selector */
1715 1716 1717
	if (seg <= VCPU_SREG_GS && !seg_desc.s) {
		if (transfer == X86_TRANSFER_CALL_JMP)
			return X86EMUL_UNHANDLEABLE;
1718
		goto exception;
1719
	}
1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	dpl = seg_desc.dpl;

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1736
		break;
1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
1750 1751 1752 1753 1754 1755 1756 1757 1758
		/* in long-mode d/b must be clear if l is set */
		if (seg_desc.d && seg_desc.l) {
			u64 efer = 0;

			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
			if (efer & EFER_LMA)
				goto exception;
		}

1759 1760
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1761
		break;
1762 1763 1764
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
1765 1766 1767 1768 1769 1770
		old_desc = seg_desc;
		seg_desc.type |= 2; /* busy */
		ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
						  sizeof(seg_desc), &ctxt->exception);
		if (ret != X86EMUL_CONTINUE)
			return ret;
1771 1772 1773 1774 1775 1776
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1777
		/*
1778 1779 1780
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1781
		 */
1782 1783 1784 1785
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1786
		break;
1787 1788 1789 1790
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
1791 1792 1793 1794 1795 1796 1797
		if (!(seg_desc.type & 1)) {
			seg_desc.type |= 1;
			ret = write_segment_descriptor(ctxt, selector,
						       &seg_desc);
			if (ret != X86EMUL_CONTINUE)
				return ret;
		}
1798
	} else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1799
		ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1800 1801
		if (ret != X86EMUL_CONTINUE)
			return ret;
1802 1803
		if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
				((u64)base3 << 32), ctxt))
1804
			return emulate_gp(ctxt, 0);
1805 1806
	}
load:
1807
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1808 1809
	if (desc)
		*desc = seg_desc;
1810 1811
	return X86EMUL_CONTINUE;
exception:
1812
	return emulate_exception(ctxt, err_vec, err_code, true);
1813 1814
}

1815 1816 1817 1818
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
	u8 cpl = ctxt->ops->cpl(ctxt);
1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833

	/*
	 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
	 * they can load it at CPL<3 (Intel's manual says only LSS can,
	 * but it's wrong).
	 *
	 * However, the Intel manual says that putting IST=1/DPL=3 in
	 * an interrupt gate will result in SS=3 (the AMD manual instead
	 * says it doesn't), so allow SS=3 in __load_segment_descriptor
	 * and only forbid it here.
	 */
	if (seg == VCPU_SREG_SS && selector == 3 &&
	    ctxt->mode == X86EMUL_MODE_PROT64)
		return emulate_exception(ctxt, GP_VECTOR, 0, true);

1834 1835
	return __load_segment_descriptor(ctxt, selector, seg, cpl,
					 X86_TRANSFER_NONE, NULL);
1836 1837
}

1838 1839
static void write_register_operand(struct operand *op)
{
1840
	return assign_register(op->addr.reg, op->val, op->bytes);
1841 1842
}

1843
static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1844
{
1845
	switch (op->type) {
1846
	case OP_REG:
1847
		write_register_operand(op);
A
Avi Kivity 已提交
1848
		break;
1849
	case OP_MEM:
1850
		if (ctxt->lock_prefix)
P
Paolo Bonzini 已提交
1851 1852 1853 1854 1855 1856 1857
			return segmented_cmpxchg(ctxt,
						 op->addr.mem,
						 &op->orig_val,
						 &op->val,
						 op->bytes);
		else
			return segmented_write(ctxt,
1858 1859 1860
					       op->addr.mem,
					       &op->val,
					       op->bytes);
1861
		break;
1862
	case OP_MEM_STR:
P
Paolo Bonzini 已提交
1863 1864 1865 1866
		return segmented_write(ctxt,
				       op->addr.mem,
				       op->data,
				       op->bytes * op->count);
1867
		break;
A
Avi Kivity 已提交
1868
	case OP_XMM:
1869
		write_sse_reg(&op->vec_val, op->addr.xmm);
A
Avi Kivity 已提交
1870
		break;
A
Avi Kivity 已提交
1871
	case OP_MM:
1872
		write_mmx_reg(&op->mm_val, op->addr.mm);
A
Avi Kivity 已提交
1873
		break;
1874 1875
	case OP_NONE:
		/* no writeback */
1876
		break;
1877
	default:
1878
		break;
A
Avi Kivity 已提交
1879
	}
1880 1881
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1882

1883
static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1884
{
1885
	struct segmented_address addr;
1886

1887
	rsp_increment(ctxt, -bytes);
1888
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1889 1890
	addr.seg = VCPU_SREG_SS;

1891 1892 1893 1894 1895
	return segmented_write(ctxt, addr, data, bytes);
}

static int em_push(struct x86_emulate_ctxt *ctxt)
{
1896
	/* Disable writeback. */
1897
	ctxt->dst.type = OP_NONE;
1898
	return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1899
}
1900

1901 1902 1903 1904
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1905
	struct segmented_address addr;
1906

1907
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1908
	addr.seg = VCPU_SREG_SS;
1909
	rc = segmented_read(ctxt, addr, dest, len);
1910 1911 1912
	if (rc != X86EMUL_CONTINUE)
		return rc;

1913
	rsp_increment(ctxt, len);
1914
	return rc;
1915 1916
}

1917 1918
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1919
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1920 1921
}

1922
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1923
			void *dest, int len)
1924 1925
{
	int rc;
1926
	unsigned long val, change_mask;
1927
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1928
	int cpl = ctxt->ops->cpl(ctxt);
1929

1930
	rc = emulate_pop(ctxt, &val, len);
1931 1932
	if (rc != X86EMUL_CONTINUE)
		return rc;
1933

1934 1935 1936 1937
	change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
		      X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
		      X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
		      X86_EFLAGS_AC | X86_EFLAGS_ID;
1938

1939 1940 1941 1942 1943
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
1944
			change_mask |= X86_EFLAGS_IOPL;
1945
		if (cpl <= iopl)
1946
			change_mask |= X86_EFLAGS_IF;
1947 1948
		break;
	case X86EMUL_MODE_VM86:
1949 1950
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1951
		change_mask |= X86_EFLAGS_IF;
1952 1953
		break;
	default: /* real mode */
1954
		change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1955
		break;
1956
	}
1957 1958 1959 1960 1961

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1962 1963
}

1964 1965
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1966 1967 1968 1969
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1970 1971
}

A
Avi Kivity 已提交
1972 1973 1974 1975 1976
static int em_enter(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned frame_size = ctxt->src.val;
	unsigned nesting_level = ctxt->src2.val & 31;
1977
	ulong rbp;
A
Avi Kivity 已提交
1978 1979 1980 1981

	if (nesting_level)
		return X86EMUL_UNHANDLEABLE;

1982 1983
	rbp = reg_read(ctxt, VCPU_REGS_RBP);
	rc = push(ctxt, &rbp, stack_size(ctxt));
A
Avi Kivity 已提交
1984 1985
	if (rc != X86EMUL_CONTINUE)
		return rc;
1986
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
A
Avi Kivity 已提交
1987
		      stack_mask(ctxt));
1988 1989
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
A
Avi Kivity 已提交
1990 1991 1992 1993
		      stack_mask(ctxt));
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1994 1995
static int em_leave(struct x86_emulate_ctxt *ctxt)
{
1996
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
A
Avi Kivity 已提交
1997
		      stack_mask(ctxt));
1998
	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
A
Avi Kivity 已提交
1999 2000
}

2001
static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
2002
{
2003 2004
	int seg = ctxt->src2.val;

2005
	ctxt->src.val = get_segment_selector(ctxt, seg);
2006 2007 2008 2009
	if (ctxt->op_bytes == 4) {
		rsp_increment(ctxt, -2);
		ctxt->op_bytes = 2;
	}
2010

2011
	return em_push(ctxt);
2012 2013
}

2014
static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
2015
{
2016
	int seg = ctxt->src2.val;
2017 2018
	unsigned long selector;
	int rc;
2019

2020
	rc = emulate_pop(ctxt, &selector, 2);
2021 2022 2023
	if (rc != X86EMUL_CONTINUE)
		return rc;

2024 2025
	if (ctxt->modrm_reg == VCPU_SREG_SS)
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2026 2027
	if (ctxt->op_bytes > 2)
		rsp_increment(ctxt, ctxt->op_bytes - 2);
2028

2029
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
2030
	return rc;
2031 2032
}

2033
static int em_pusha(struct x86_emulate_ctxt *ctxt)
2034
{
2035
	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2036 2037
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
2038

2039 2040
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
2041
		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2042

2043
		rc = em_push(ctxt);
2044 2045
		if (rc != X86EMUL_CONTINUE)
			return rc;
2046

2047
		++reg;
2048 2049
	}

2050
	return rc;
2051 2052
}

2053 2054
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
2055
	ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2056 2057 2058
	return em_push(ctxt);
}

2059
static int em_popa(struct x86_emulate_ctxt *ctxt)
2060
{
2061 2062
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
2063
	u32 val;
2064

2065 2066
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
2067
			rsp_increment(ctxt, ctxt->op_bytes);
2068 2069
			--reg;
		}
2070

2071
		rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2072 2073
		if (rc != X86EMUL_CONTINUE)
			break;
2074
		assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2075
		--reg;
2076
	}
2077
	return rc;
2078 2079
}

2080
static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2081
{
2082
	const struct x86_emulate_ops *ops = ctxt->ops;
2083
	int rc;
2084 2085 2086 2087 2088 2089
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
2090
	ctxt->src.val = ctxt->eflags;
2091
	rc = em_push(ctxt);
2092 2093
	if (rc != X86EMUL_CONTINUE)
		return rc;
2094

2095
	ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2096

2097
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2098
	rc = em_push(ctxt);
2099 2100
	if (rc != X86EMUL_CONTINUE)
		return rc;
2101

2102
	ctxt->src.val = ctxt->_eip;
2103
	rc = em_push(ctxt);
2104 2105 2106
	if (rc != X86EMUL_CONTINUE)
		return rc;

2107
	ops->get_idt(ctxt, &dt);
2108 2109 2110 2111

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

2112
	rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2113 2114 2115
	if (rc != X86EMUL_CONTINUE)
		return rc;

2116
	rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2117 2118 2119
	if (rc != X86EMUL_CONTINUE)
		return rc;

2120
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2121 2122 2123
	if (rc != X86EMUL_CONTINUE)
		return rc;

2124
	ctxt->_eip = eip;
2125 2126 2127 2128

	return rc;
}

2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
{
	int rc;

	invalidate_registers(ctxt);
	rc = __emulate_int_real(ctxt, irq);
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);
	return rc;
}

2140
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2141 2142 2143
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
2144
		return __emulate_int_real(ctxt, irq);
2145 2146 2147 2148 2149 2150 2151 2152 2153 2154
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

2155
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2156
{
2157 2158 2159 2160
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
2161 2162 2163 2164 2165
	unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
			     X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
			     X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
			     X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
			     X86_EFLAGS_AC | X86_EFLAGS_ID |
W
Wanpeng Li 已提交
2166
			     X86_EFLAGS_FIXED;
2167 2168
	unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
				  X86_EFLAGS_VIP;
2169

2170
	/* TODO: Add stack limit check */
2171

2172
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2173

2174 2175
	if (rc != X86EMUL_CONTINUE)
		return rc;
2176

2177 2178
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
2179

2180
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2181

2182 2183
	if (rc != X86EMUL_CONTINUE)
		return rc;
2184

2185
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2186

2187 2188
	if (rc != X86EMUL_CONTINUE)
		return rc;
2189

2190
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2191

2192 2193
	if (rc != X86EMUL_CONTINUE)
		return rc;
2194

2195
	ctxt->_eip = temp_eip;
2196

2197
	if (ctxt->op_bytes == 4)
2198
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2199
	else if (ctxt->op_bytes == 2) {
2200 2201
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
2202
	}
2203 2204

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
W
Wanpeng Li 已提交
2205
	ctxt->eflags |= X86_EFLAGS_FIXED;
2206
	ctxt->ops->set_nmi_mask(ctxt, false);
2207 2208

	return rc;
2209 2210
}

2211
static int em_iret(struct x86_emulate_ctxt *ctxt)
2212
{
2213 2214
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
2215
		return emulate_iret_real(ctxt);
2216 2217 2218 2219
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
2220
	default:
2221 2222
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
2223 2224 2225
	}
}

2226 2227 2228
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
2229 2230
	unsigned short sel;
	struct desc_struct new_desc;
2231 2232
	u8 cpl = ctxt->ops->cpl(ctxt);

2233
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2234

2235 2236
	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
				       X86_TRANSFER_CALL_JMP,
2237
				       &new_desc);
2238 2239 2240
	if (rc != X86EMUL_CONTINUE)
		return rc;

2241
	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2242 2243 2244 2245
	/* Error handling is not implemented. */
	if (rc != X86EMUL_CONTINUE)
		return X86EMUL_UNHANDLEABLE;

2246
	return rc;
2247 2248
}

2249
static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2250
{
2251 2252
	return assign_eip_near(ctxt, ctxt->src.val);
}
2253

2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264
static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	long int old_eip;

	old_eip = ctxt->_eip;
	rc = assign_eip_near(ctxt, ctxt->src.val);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->src.val = old_eip;
	rc = em_push(ctxt);
2265
	return rc;
2266 2267
}

2268
static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2269
{
2270
	u64 old = ctxt->dst.orig_val64;
2271

2272 2273 2274
	if (ctxt->dst.bytes == 16)
		return X86EMUL_UNHANDLEABLE;

2275 2276 2277 2278
	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2279
		ctxt->eflags &= ~X86_EFLAGS_ZF;
2280
	} else {
2281 2282
		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
			(u32) reg_read(ctxt, VCPU_REGS_RBX);
2283

2284
		ctxt->eflags |= X86_EFLAGS_ZF;
2285
	}
2286
	return X86EMUL_CONTINUE;
2287 2288
}

2289 2290
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
2291 2292 2293 2294 2295 2296 2297 2298
	int rc;
	unsigned long eip;

	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	return assign_eip_near(ctxt, eip);
2299 2300
}

2301
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2302 2303
{
	int rc;
2304
	unsigned long eip, cs;
2305
	int cpl = ctxt->ops->cpl(ctxt);
2306
	struct desc_struct new_desc;
2307

2308
	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2309
	if (rc != X86EMUL_CONTINUE)
2310
		return rc;
2311
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2312
	if (rc != X86EMUL_CONTINUE)
2313
		return rc;
2314 2315 2316
	/* Outer-privilege level return is not implemented */
	if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
		return X86EMUL_UNHANDLEABLE;
2317 2318
	rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
				       X86_TRANSFER_RET,
2319 2320 2321
				       &new_desc);
	if (rc != X86EMUL_CONTINUE)
		return rc;
2322
	rc = assign_eip_far(ctxt, eip, &new_desc);
2323 2324 2325 2326
	/* Error handling is not implemented. */
	if (rc != X86EMUL_CONTINUE)
		return X86EMUL_UNHANDLEABLE;

2327 2328 2329
	return rc;
}

2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
{
        int rc;

        rc = em_ret_far(ctxt);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        rsp_increment(ctxt, ctxt->src.val);
        return X86EMUL_CONTINUE;
}

2341 2342 2343
static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
{
	/* Save real source value, then compare EAX against destination. */
2344 2345
	ctxt->dst.orig_val = ctxt->dst.val;
	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2346
	ctxt->src.orig_val = ctxt->src.val;
2347
	ctxt->src.val = ctxt->dst.orig_val;
2348
	fastop(ctxt, em_cmp);
2349

2350
	if (ctxt->eflags & X86_EFLAGS_ZF) {
2351 2352
		/* Success: write back to memory; no update of EAX */
		ctxt->src.type = OP_NONE;
2353 2354 2355
		ctxt->dst.val = ctxt->src.orig_val;
	} else {
		/* Failure: write the value we saw to EAX. */
2356 2357 2358 2359
		ctxt->src.type = OP_REG;
		ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
		ctxt->src.val = ctxt->dst.orig_val;
		/* Create write-cycle to dest by writing the same value */
2360
		ctxt->dst.val = ctxt->dst.orig_val;
2361 2362 2363 2364
	}
	return X86EMUL_CONTINUE;
}

2365
static int em_lseg(struct x86_emulate_ctxt *ctxt)
2366
{
2367
	int seg = ctxt->src2.val;
2368 2369 2370
	unsigned short sel;
	int rc;

2371
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2372

2373
	rc = load_segment_descriptor(ctxt, sel, seg);
2374 2375 2376
	if (rc != X86EMUL_CONTINUE)
		return rc;

2377
	ctxt->dst.val = ctxt->src.val;
2378 2379 2380
	return rc;
}

2381 2382
static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
{
2383
#ifdef CONFIG_X86_64
2384
	return ctxt->ops->guest_has_long_mode(ctxt);
2385 2386 2387
#else
	return false;
#endif
2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
}

static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
{
	desc->g    = (flags >> 23) & 1;
	desc->d    = (flags >> 22) & 1;
	desc->l    = (flags >> 21) & 1;
	desc->avl  = (flags >> 20) & 1;
	desc->p    = (flags >> 15) & 1;
	desc->dpl  = (flags >> 13) & 3;
	desc->s    = (flags >> 12) & 1;
	desc->type = (flags >>  8) & 15;
}

2402 2403
static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
			   int n)
2404 2405 2406 2407 2408
{
	struct desc_struct desc;
	int offset;
	u16 selector;

2409
	selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2410 2411 2412 2413 2414 2415

	if (n < 3)
		offset = 0x7f84 + n * 12;
	else
		offset = 0x7f2c + (n - 3) * 12;

2416 2417 2418
	set_desc_base(&desc,      GET_SMSTATE(u32, smstate, offset + 8));
	set_desc_limit(&desc,     GET_SMSTATE(u32, smstate, offset + 4));
	rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2419 2420 2421 2422
	ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
	return X86EMUL_CONTINUE;
}

2423
#ifdef CONFIG_X86_64
2424 2425
static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
			   int n)
2426 2427 2428 2429 2430 2431 2432 2433
{
	struct desc_struct desc;
	int offset;
	u16 selector;
	u32 base3;

	offset = 0x7e00 + n * 16;

2434 2435 2436 2437 2438
	selector =                GET_SMSTATE(u16, smstate, offset);
	rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
	set_desc_limit(&desc,     GET_SMSTATE(u32, smstate, offset + 4));
	set_desc_base(&desc,      GET_SMSTATE(u32, smstate, offset + 8));
	base3 =                   GET_SMSTATE(u32, smstate, offset + 12);
2439 2440 2441 2442

	ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
	return X86EMUL_CONTINUE;
}
2443
#endif
2444 2445

static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2446
				    u64 cr0, u64 cr3, u64 cr4)
2447 2448
{
	int bad;
2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460
	u64 pcid;

	/* In order to later set CR4.PCIDE, CR3[11:0] must be zero.  */
	pcid = 0;
	if (cr4 & X86_CR4_PCIDE) {
		pcid = cr3 & 0xfff;
		cr3 &= ~0xfff;
	}

	bad = ctxt->ops->set_cr(ctxt, 3, cr3);
	if (bad)
		return X86EMUL_UNHANDLEABLE;
2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478

	/*
	 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
	 * Then enable protected mode.	However, PCID cannot be enabled
	 * if EFER.LMA=0, so set it separately.
	 */
	bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
	if (bad)
		return X86EMUL_UNHANDLEABLE;

	bad = ctxt->ops->set_cr(ctxt, 0, cr0);
	if (bad)
		return X86EMUL_UNHANDLEABLE;

	if (cr4 & X86_CR4_PCIDE) {
		bad = ctxt->ops->set_cr(ctxt, 4, cr4);
		if (bad)
			return X86EMUL_UNHANDLEABLE;
2479 2480 2481 2482 2483 2484
		if (pcid) {
			bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
			if (bad)
				return X86EMUL_UNHANDLEABLE;
		}

2485 2486 2487 2488 2489
	}

	return X86EMUL_CONTINUE;
}

2490 2491
static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
			     const char *smstate)
2492 2493 2494 2495
{
	struct desc_struct desc;
	struct desc_ptr dt;
	u16 selector;
2496
	u32 val, cr0, cr3, cr4;
2497 2498
	int i;

2499 2500 2501 2502
	cr0 =                      GET_SMSTATE(u32, smstate, 0x7ffc);
	cr3 =                      GET_SMSTATE(u32, smstate, 0x7ff8);
	ctxt->eflags =             GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
	ctxt->_eip =               GET_SMSTATE(u32, smstate, 0x7ff0);
2503 2504

	for (i = 0; i < 8; i++)
2505
		*reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2506

2507
	val = GET_SMSTATE(u32, smstate, 0x7fcc);
2508
	ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2509
	val = GET_SMSTATE(u32, smstate, 0x7fc8);
2510 2511
	ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);

2512 2513 2514 2515
	selector =                 GET_SMSTATE(u32, smstate, 0x7fc4);
	set_desc_base(&desc,       GET_SMSTATE(u32, smstate, 0x7f64));
	set_desc_limit(&desc,      GET_SMSTATE(u32, smstate, 0x7f60));
	rsm_set_desc_flags(&desc,  GET_SMSTATE(u32, smstate, 0x7f5c));
2516 2517
	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);

2518 2519 2520 2521
	selector =                 GET_SMSTATE(u32, smstate, 0x7fc0);
	set_desc_base(&desc,       GET_SMSTATE(u32, smstate, 0x7f80));
	set_desc_limit(&desc,      GET_SMSTATE(u32, smstate, 0x7f7c));
	rsm_set_desc_flags(&desc,  GET_SMSTATE(u32, smstate, 0x7f78));
2522 2523
	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);

2524 2525
	dt.address =               GET_SMSTATE(u32, smstate, 0x7f74);
	dt.size =                  GET_SMSTATE(u32, smstate, 0x7f70);
2526 2527
	ctxt->ops->set_gdt(ctxt, &dt);

2528 2529
	dt.address =               GET_SMSTATE(u32, smstate, 0x7f58);
	dt.size =                  GET_SMSTATE(u32, smstate, 0x7f54);
2530 2531 2532
	ctxt->ops->set_idt(ctxt, &dt);

	for (i = 0; i < 6; i++) {
2533
		int r = rsm_load_seg_32(ctxt, smstate, i);
2534 2535 2536 2537
		if (r != X86EMUL_CONTINUE)
			return r;
	}

2538
	cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2539

2540
	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2541

2542
	return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2543 2544
}

2545
#ifdef CONFIG_X86_64
2546 2547
static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
			     const char *smstate)
2548 2549 2550
{
	struct desc_struct desc;
	struct desc_ptr dt;
2551
	u64 val, cr0, cr3, cr4;
2552 2553
	u32 base3;
	u16 selector;
2554
	int i, r;
2555 2556

	for (i = 0; i < 16; i++)
2557
		*reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2558

2559 2560
	ctxt->_eip   = GET_SMSTATE(u64, smstate, 0x7f78);
	ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2561

2562
	val = GET_SMSTATE(u32, smstate, 0x7f68);
2563
	ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2564
	val = GET_SMSTATE(u32, smstate, 0x7f60);
2565 2566
	ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);

2567 2568 2569 2570 2571
	cr0 =                       GET_SMSTATE(u64, smstate, 0x7f58);
	cr3 =                       GET_SMSTATE(u64, smstate, 0x7f50);
	cr4 =                       GET_SMSTATE(u64, smstate, 0x7f48);
	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
	val =                       GET_SMSTATE(u64, smstate, 0x7ed0);
2572 2573
	ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);

2574 2575 2576 2577 2578
	selector =                  GET_SMSTATE(u32, smstate, 0x7e90);
	rsm_set_desc_flags(&desc,   GET_SMSTATE(u32, smstate, 0x7e92) << 8);
	set_desc_limit(&desc,       GET_SMSTATE(u32, smstate, 0x7e94));
	set_desc_base(&desc,        GET_SMSTATE(u32, smstate, 0x7e98));
	base3 =                     GET_SMSTATE(u32, smstate, 0x7e9c);
2579 2580
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);

2581 2582
	dt.size =                   GET_SMSTATE(u32, smstate, 0x7e84);
	dt.address =                GET_SMSTATE(u64, smstate, 0x7e88);
2583 2584
	ctxt->ops->set_idt(ctxt, &dt);

2585 2586 2587 2588 2589
	selector =                  GET_SMSTATE(u32, smstate, 0x7e70);
	rsm_set_desc_flags(&desc,   GET_SMSTATE(u32, smstate, 0x7e72) << 8);
	set_desc_limit(&desc,       GET_SMSTATE(u32, smstate, 0x7e74));
	set_desc_base(&desc,        GET_SMSTATE(u32, smstate, 0x7e78));
	base3 =                     GET_SMSTATE(u32, smstate, 0x7e7c);
2590 2591
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);

2592 2593
	dt.size =                   GET_SMSTATE(u32, smstate, 0x7e64);
	dt.address =                GET_SMSTATE(u64, smstate, 0x7e68);
2594 2595
	ctxt->ops->set_gdt(ctxt, &dt);

2596
	r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2597 2598 2599
	if (r != X86EMUL_CONTINUE)
		return r;

2600
	for (i = 0; i < 6; i++) {
2601
		r = rsm_load_seg_64(ctxt, smstate, i);
2602 2603 2604 2605
		if (r != X86EMUL_CONTINUE)
			return r;
	}

2606
	return X86EMUL_CONTINUE;
2607
}
2608
#endif
2609

P
Paolo Bonzini 已提交
2610 2611
static int em_rsm(struct x86_emulate_ctxt *ctxt)
{
2612
	unsigned long cr0, cr4, efer;
2613
	char buf[512];
2614 2615 2616
	u64 smbase;
	int ret;

2617
	if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
P
Paolo Bonzini 已提交
2618 2619
		return emulate_ud(ctxt);

2620 2621 2622 2623 2624 2625
	smbase = ctxt->ops->get_smbase(ctxt);

	ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
	if (ret != X86EMUL_CONTINUE)
		return X86EMUL_UNHANDLEABLE;

2626 2627 2628 2629 2630 2631
	if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
		ctxt->ops->set_nmi_mask(ctxt, false);

	ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
		~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));

2632 2633
	/*
	 * Get back to real mode, to prepare a safe state in which to load
2634 2635
	 * CR0/CR3/CR4/EFER.  It's all a bit more complicated if the vCPU
	 * supports long mode.
2636
	 */
2637 2638 2639 2640
	if (emulator_has_longmode(ctxt)) {
		struct desc_struct cs_desc;

		/* Zero CR4.PCIDE before CR0.PG.  */
2641 2642
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		if (cr4 & X86_CR4_PCIDE)
2643 2644 2645 2646 2647 2648 2649 2650 2651 2652
			ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);

		/* A 32-bit code segment is required to clear EFER.LMA.  */
		memset(&cs_desc, 0, sizeof(cs_desc));
		cs_desc.type = 0xb;
		cs_desc.s = cs_desc.g = cs_desc.p = 1;
		ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
	}

	/* For the 64-bit case, this will clear EFER.LMA.  */
2653 2654 2655
	cr0 = ctxt->ops->get_cr(ctxt, 0);
	if (cr0 & X86_CR0_PE)
		ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2656

2657 2658 2659 2660 2661 2662 2663 2664 2665 2666
	if (emulator_has_longmode(ctxt)) {
		/* Clear CR4.PAE before clearing EFER.LME. */
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		if (cr4 & X86_CR4_PAE)
			ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);

		/* And finally go back to 32-bit mode.  */
		efer = 0;
		ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
	}
2667

2668 2669 2670 2671 2672
	/*
	 * Give pre_leave_smm() a chance to make ISA-specific changes to the
	 * vCPU state (e.g. enter guest mode) before loading state from the SMM
	 * state-save area.
	 */
2673
	if (ctxt->ops->pre_leave_smm(ctxt, buf))
2674 2675
		return X86EMUL_UNHANDLEABLE;

2676
#ifdef CONFIG_X86_64
2677
	if (emulator_has_longmode(ctxt))
2678
		ret = rsm_load_state_64(ctxt, buf);
2679
	else
2680
#endif
2681
		ret = rsm_load_state_32(ctxt, buf);
2682 2683 2684 2685 2686 2687

	if (ret != X86EMUL_CONTINUE) {
		/* FIXME: should triple fault */
		return X86EMUL_UNHANDLEABLE;
	}

2688 2689
	ctxt->ops->post_leave_smm(ctxt);

2690
	return X86EMUL_CONTINUE;
P
Paolo Bonzini 已提交
2691 2692
}

2693
static void
2694
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2695
			struct desc_struct *cs, struct desc_struct *ss)
2696 2697
{
	cs->l = 0;		/* will be adjusted later */
2698
	set_desc_base(cs, 0);	/* flat segment */
2699
	cs->g = 1;		/* 4kb granularity */
2700
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
2701 2702 2703
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
2704 2705
	cs->p = 1;
	cs->d = 1;
2706
	cs->avl = 0;
2707

2708 2709
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
2710 2711 2712
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
2713
	ss->d = 1;		/* 32bit stack segment */
2714
	ss->dpl = 0;
2715
	ss->p = 1;
2716 2717
	ss->l = 0;
	ss->avl = 0;
2718 2719
}

2720 2721 2722 2723 2724
static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

	eax = ecx = 0;
2725
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2726
	return is_guest_vendor_intel(ebx, ecx, edx);
2727 2728
}

2729 2730
static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
{
2731
	const struct x86_emulate_ops *ops = ctxt->ops;
2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742
	u32 eax, ebx, ecx, edx;

	/*
	 * syscall should always be enabled in longmode - so only become
	 * vendor specific (cpuid) if other modes are active...
	 */
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return true;

	eax = 0x00000000;
	ecx = 0x00000000;
2743
	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2744
	/*
2745 2746 2747 2748
	 * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a
	 * 64bit guest with a 32bit compat-app running will #UD !! While this
	 * behaviour can be fixed (by emulating) into AMD response - CPUs of
	 * AMD can't behave like Intel.
2749
	 */
2750
	if (is_guest_vendor_intel(ebx, ecx, edx))
2751 2752
		return false;

2753 2754
	if (is_guest_vendor_amd(ebx, ecx, edx) ||
	    is_guest_vendor_hygon(ebx, ecx, edx))
2755 2756 2757 2758 2759 2760
		return true;

	/*
	 * default: (not Intel, not AMD, not Hygon), apply Intel's
	 * stricter rules...
	 */
2761 2762 2763
	return false;
}

2764
static int em_syscall(struct x86_emulate_ctxt *ctxt)
2765
{
2766
	const struct x86_emulate_ops *ops = ctxt->ops;
2767
	struct desc_struct cs, ss;
2768
	u64 msr_data;
2769
	u16 cs_sel, ss_sel;
2770
	u64 efer = 0;
2771 2772

	/* syscall is not available in real mode */
2773
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2774 2775
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
2776

2777 2778 2779
	if (!(em_syscall_is_enabled(ctxt)))
		return emulate_ud(ctxt);

2780
	ops->get_msr(ctxt, MSR_EFER, &efer);
2781 2782 2783
	if (!(efer & EFER_SCE))
		return emulate_ud(ctxt);

2784
	setup_syscalls_segments(ctxt, &cs, &ss);
2785
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
2786
	msr_data >>= 32;
2787 2788
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
2789

2790
	if (efer & EFER_LMA) {
2791
		cs.d = 0;
2792 2793
		cs.l = 1;
	}
2794 2795
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2796

2797
	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2798
	if (efer & EFER_LMA) {
2799
#ifdef CONFIG_X86_64
2800
		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2801

2802
		ops->get_msr(ctxt,
2803 2804
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
2805
		ctxt->_eip = msr_data;
2806

2807
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2808
		ctxt->eflags &= ~msr_data;
W
Wanpeng Li 已提交
2809
		ctxt->eflags |= X86_EFLAGS_FIXED;
2810 2811 2812
#endif
	} else {
		/* legacy mode */
2813
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
2814
		ctxt->_eip = (u32)msr_data;
2815

2816
		ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2817 2818
	}

2819
	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2820
	return X86EMUL_CONTINUE;
2821 2822
}

2823
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2824
{
2825
	const struct x86_emulate_ops *ops = ctxt->ops;
2826
	struct desc_struct cs, ss;
2827
	u64 msr_data;
2828
	u16 cs_sel, ss_sel;
2829
	u64 efer = 0;
2830

2831
	ops->get_msr(ctxt, MSR_EFER, &efer);
2832
	/* inject #GP if in real mode */
2833 2834
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
2835

2836 2837 2838 2839
	/*
	 * Not recognized on AMD in compat mode (but is recognized in legacy
	 * mode).
	 */
2840
	if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2841 2842 2843
	    && !vendor_intel(ctxt))
		return emulate_ud(ctxt);

2844
	/* sysenter/sysexit have not been tested in 64bit mode. */
2845
	if (ctxt->mode == X86EMUL_MODE_PROT64)
2846
		return X86EMUL_UNHANDLEABLE;
2847

2848
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2849 2850
	if ((msr_data & 0xfffc) == 0x0)
		return emulate_gp(ctxt, 0);
2851

2852
	setup_syscalls_segments(ctxt, &cs, &ss);
2853
	ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2854
	cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2855
	ss_sel = cs_sel + 8;
2856
	if (efer & EFER_LMA) {
2857
		cs.d = 0;
2858 2859 2860
		cs.l = 1;
	}

2861 2862
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2863

2864
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2865
	ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2866

2867
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2868 2869
	*reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
							      (u32)msr_data;
2870

2871
	return X86EMUL_CONTINUE;
2872 2873
}

2874
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2875
{
2876
	const struct x86_emulate_ops *ops = ctxt->ops;
2877
	struct desc_struct cs, ss;
2878
	u64 msr_data, rcx, rdx;
2879
	int usermode;
X
Xiao Guangrong 已提交
2880
	u16 cs_sel = 0, ss_sel = 0;
2881

2882 2883
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2884 2885
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
2886

2887
	setup_syscalls_segments(ctxt, &cs, &ss);
2888

2889
	if ((ctxt->rex_prefix & 0x8) != 0x0)
2890 2891 2892 2893
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

2894 2895 2896
	rcx = reg_read(ctxt, VCPU_REGS_RCX);
	rdx = reg_read(ctxt, VCPU_REGS_RDX);

2897 2898
	cs.dpl = 3;
	ss.dpl = 3;
2899
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2900 2901
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
2902
		cs_sel = (u16)(msr_data + 16);
2903 2904
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2905
		ss_sel = (u16)(msr_data + 24);
2906 2907
		rcx = (u32)rcx;
		rdx = (u32)rdx;
2908 2909
		break;
	case X86EMUL_MODE_PROT64:
2910
		cs_sel = (u16)(msr_data + 32);
2911 2912
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2913 2914
		ss_sel = cs_sel + 8;
		cs.d = 0;
2915
		cs.l = 1;
2916 2917
		if (emul_is_noncanonical_address(rcx, ctxt) ||
		    emul_is_noncanonical_address(rdx, ctxt))
2918
			return emulate_gp(ctxt, 0);
2919 2920
		break;
	}
2921 2922
	cs_sel |= SEGMENT_RPL_MASK;
	ss_sel |= SEGMENT_RPL_MASK;
2923

2924 2925
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2926

2927 2928
	ctxt->_eip = rdx;
	*reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2929

2930
	return X86EMUL_CONTINUE;
2931 2932
}

2933
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2934 2935 2936 2937 2938 2939
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
2940
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2941
	return ctxt->ops->cpl(ctxt) > iopl;
2942 2943
}

2944 2945 2946
#define VMWARE_PORT_VMPORT	(0x5658)
#define VMWARE_PORT_VMRPC	(0x5659)

2947 2948 2949
static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2950
	const struct x86_emulate_ops *ops = ctxt->ops;
2951
	struct desc_struct tr_seg;
2952
	u32 base3;
2953
	int r;
2954
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2955
	unsigned mask = (1 << len) - 1;
2956
	unsigned long base;
2957

2958 2959 2960 2961 2962 2963 2964 2965
	/*
	 * VMware allows access to these ports even if denied
	 * by TSS I/O permission bitmap. Mimic behavior.
	 */
	if (enable_vmware_backdoor &&
	    ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
		return true;

2966
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2967
	if (!tr_seg.p)
2968
		return false;
2969
	if (desc_limit_scaled(&tr_seg) < 103)
2970
		return false;
2971 2972 2973 2974
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2975
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2976 2977
	if (r != X86EMUL_CONTINUE)
		return false;
2978
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2979
		return false;
2980
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2981 2982 2983 2984 2985 2986 2987 2988 2989 2990
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2991 2992 2993
	if (ctxt->perm_ok)
		return true;

2994 2995
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2996
			return false;
2997 2998 2999

	ctxt->perm_ok = true;

3000 3001 3002
	return true;
}

3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026
static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
{
	/*
	 * Intel CPUs mask the counter and pointers in quite strange
	 * manner when ECX is zero due to REP-string optimizations.
	 */
#ifdef CONFIG_X86_64
	if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
		return;

	*reg_write(ctxt, VCPU_REGS_RCX) = 0;

	switch (ctxt->b) {
	case 0xa4:	/* movsb */
	case 0xa5:	/* movsd/w */
		*reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
		/* fall through */
	case 0xaa:	/* stosb */
	case 0xab:	/* stosd/w */
		*reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
	}
#endif
}

3027 3028 3029
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
3030
	tss->ip = ctxt->_eip;
3031
	tss->flag = ctxt->eflags;
3032 3033 3034 3035 3036 3037 3038 3039
	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->si = reg_read(ctxt, VCPU_REGS_RSI);
	tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3040

3041 3042 3043 3044 3045
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3046 3047 3048 3049 3050 3051
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;
3052
	u8 cpl;
3053

3054
	ctxt->_eip = tss->ip;
3055
	ctxt->eflags = tss->flag | 2;
3056 3057 3058 3059 3060 3061 3062 3063
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3064 3065 3066 3067 3068

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
3069 3070 3071 3072 3073
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3074

3075 3076
	cpl = tss->cs & 3;

3077
	/*
G
Guo Chao 已提交
3078
	 * Now load segment descriptors. If fault happens at this stage
3079 3080
	 * it is handled in a context of new task
	 */
3081
	ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3082
					X86_TRANSFER_TASK_SWITCH, NULL);
3083 3084
	if (ret != X86EMUL_CONTINUE)
		return ret;
3085
	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3086
					X86_TRANSFER_TASK_SWITCH, NULL);
3087 3088
	if (ret != X86EMUL_CONTINUE)
		return ret;
3089
	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3090
					X86_TRANSFER_TASK_SWITCH, NULL);
3091 3092
	if (ret != X86EMUL_CONTINUE)
		return ret;
3093
	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3094
					X86_TRANSFER_TASK_SWITCH, NULL);
3095 3096
	if (ret != X86EMUL_CONTINUE)
		return ret;
3097
	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3098
					X86_TRANSFER_TASK_SWITCH, NULL);
3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
	struct tss_segment_16 tss_seg;
	int ret;
3111
	u32 new_tss_base = get_desc_base(new_desc);
3112

3113
	ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3114
	if (ret != X86EMUL_CONTINUE)
3115 3116
		return ret;

3117
	save_state_to_tss16(ctxt, &tss_seg);
3118

3119
	ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3120
	if (ret != X86EMUL_CONTINUE)
3121 3122
		return ret;

3123
	ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3124
	if (ret != X86EMUL_CONTINUE)
3125 3126 3127 3128 3129
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

3130 3131
		ret = linear_write_system(ctxt, new_tss_base,
					  &tss_seg.prev_task_link,
3132
					  sizeof(tss_seg.prev_task_link));
3133
		if (ret != X86EMUL_CONTINUE)
3134 3135 3136
			return ret;
	}

3137
	return load_state_from_tss16(ctxt, &tss_seg);
3138 3139 3140 3141 3142
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
3143
	/* CR3 and ldt selector are not saved intentionally */
3144
	tss->eip = ctxt->_eip;
3145
	tss->eflags = ctxt->eflags;
3146 3147 3148 3149 3150 3151 3152 3153
	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3154

3155 3156 3157 3158 3159 3160
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3161 3162 3163 3164 3165 3166
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;
3167
	u8 cpl;
3168

3169
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3170
		return emulate_gp(ctxt, 0);
3171
	ctxt->_eip = tss->eip;
3172
	ctxt->eflags = tss->eflags | 2;
3173 3174

	/* General purpose registers */
3175 3176 3177 3178 3179 3180 3181 3182
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3183 3184 3185

	/*
	 * SDM says that segment selectors are loaded before segment
3186 3187
	 * descriptors.  This is important because CPL checks will
	 * use CS.RPL.
3188
	 */
3189 3190 3191 3192 3193 3194 3195
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3196

3197 3198 3199 3200 3201
	/*
	 * If we're switching between Protected Mode and VM86, we need to make
	 * sure to update the mode before loading the segment descriptors so
	 * that the selectors are interpreted correctly.
	 */
3202
	if (ctxt->eflags & X86_EFLAGS_VM) {
3203
		ctxt->mode = X86EMUL_MODE_VM86;
3204 3205
		cpl = 3;
	} else {
3206
		ctxt->mode = X86EMUL_MODE_PROT32;
3207 3208
		cpl = tss->cs & 3;
	}
3209

3210 3211 3212 3213
	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
3214
	ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3215
					cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3216 3217
	if (ret != X86EMUL_CONTINUE)
		return ret;
3218
	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3219
					X86_TRANSFER_TASK_SWITCH, NULL);
3220 3221
	if (ret != X86EMUL_CONTINUE)
		return ret;
3222
	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3223
					X86_TRANSFER_TASK_SWITCH, NULL);
3224 3225
	if (ret != X86EMUL_CONTINUE)
		return ret;
3226
	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3227
					X86_TRANSFER_TASK_SWITCH, NULL);
3228 3229
	if (ret != X86EMUL_CONTINUE)
		return ret;
3230
	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3231
					X86_TRANSFER_TASK_SWITCH, NULL);
3232 3233
	if (ret != X86EMUL_CONTINUE)
		return ret;
3234
	ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3235
					X86_TRANSFER_TASK_SWITCH, NULL);
3236 3237
	if (ret != X86EMUL_CONTINUE)
		return ret;
3238
	ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3239
					X86_TRANSFER_TASK_SWITCH, NULL);
3240

3241
	return ret;
3242 3243 3244 3245 3246 3247 3248 3249
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
	struct tss_segment_32 tss_seg;
	int ret;
3250
	u32 new_tss_base = get_desc_base(new_desc);
3251 3252
	u32 eip_offset = offsetof(struct tss_segment_32, eip);
	u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3253

3254
	ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3255
	if (ret != X86EMUL_CONTINUE)
3256 3257
		return ret;

3258
	save_state_to_tss32(ctxt, &tss_seg);
3259

3260
	/* Only GP registers and segment selectors are saved */
3261 3262
	ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
				  ldt_sel_offset - eip_offset);
3263
	if (ret != X86EMUL_CONTINUE)
3264 3265
		return ret;

3266
	ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3267
	if (ret != X86EMUL_CONTINUE)
3268 3269 3270 3271 3272
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

3273 3274
		ret = linear_write_system(ctxt, new_tss_base,
					  &tss_seg.prev_task_link,
3275
					  sizeof(tss_seg.prev_task_link));
3276
		if (ret != X86EMUL_CONTINUE)
3277 3278 3279
			return ret;
	}

3280
	return load_state_from_tss32(ctxt, &tss_seg);
3281 3282 3283
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3284
				   u16 tss_selector, int idt_index, int reason,
3285
				   bool has_error_code, u32 error_code)
3286
{
3287
	const struct x86_emulate_ops *ops = ctxt->ops;
3288 3289
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
3290
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3291
	ulong old_tss_base =
3292
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3293
	u32 desc_limit;
3294
	ulong desc_addr, dr7;
3295 3296 3297

	/* FIXME: old_tss_base == ~0 ? */

3298
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3299 3300
	if (ret != X86EMUL_CONTINUE)
		return ret;
3301
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3302 3303 3304 3305 3306
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

3307 3308 3309 3310 3311
	/*
	 * Check privileges. The three cases are task switch caused by...
	 *
	 * 1. jmp/call/int to task gate: Check against DPL of the task gate
	 * 2. Exception/IRQ/iret: No check is performed
3312 3313
	 * 3. jmp/call to TSS/task-gate: No check is performed since the
	 *    hardware checks it before exiting.
3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329
	 */
	if (reason == TASK_SWITCH_GATE) {
		if (idt_index != -1) {
			/* Software interrupts */
			struct desc_struct task_gate_desc;
			int dpl;

			ret = read_interrupt_descriptor(ctxt, idt_index,
							&task_gate_desc);
			if (ret != X86EMUL_CONTINUE)
				return ret;

			dpl = task_gate_desc.dpl;
			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
				return emulate_gp(ctxt, (idt_index << 3) | 0x2);
		}
3330 3331
	}

3332 3333 3334 3335
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
3336
		return emulate_ts(ctxt, tss_selector & 0xfffc);
3337 3338 3339 3340
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3341
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3342 3343 3344 3345 3346 3347
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
G
Guo Chao 已提交
3348
	   note that old_tss_sel is not used after this point */
3349 3350 3351 3352
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
3353
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3354 3355
				     old_tss_base, &next_tss_desc);
	else
3356
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3357
				     old_tss_base, &next_tss_desc);
3358 3359
	if (ret != X86EMUL_CONTINUE)
		return ret;
3360 3361 3362 3363 3364 3365

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
3366
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3367 3368
	}

3369
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
3370
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3371

3372
	if (has_error_code) {
3373 3374 3375
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
3376
		ret = em_push(ctxt);
3377 3378
	}

3379 3380 3381
	ops->get_dr(ctxt, 7, &dr7);
	ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));

3382 3383 3384 3385
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3386
			 u16 tss_selector, int idt_index, int reason,
3387
			 bool has_error_code, u32 error_code)
3388 3389 3390
{
	int rc;

3391
	invalidate_registers(ctxt);
3392 3393
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
3394

3395
	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3396
				     has_error_code, error_code);
3397

3398
	if (rc == X86EMUL_CONTINUE) {
3399
		ctxt->eip = ctxt->_eip;
3400 3401
		writeback_registers(ctxt);
	}
3402

3403
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3404 3405
}

3406 3407
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
		struct operand *op)
3408
{
3409
	int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3410

3411 3412
	register_address_increment(ctxt, reg, df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, reg);
3413 3414
}

3415 3416 3417 3418 3419 3420
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
3421
	al = ctxt->dst.val;
3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

3439
	ctxt->dst.val = al;
3440
	/* Set PF, ZF, SF */
3441 3442 3443
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
3444
	fastop(ctxt, em_or);
3445 3446 3447 3448 3449 3450 3451 3452
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474
static int em_aam(struct x86_emulate_ctxt *ctxt)
{
	u8 al, ah;

	if (ctxt->src.val == 0)
		return emulate_de(ctxt);

	al = ctxt->dst.val & 0xff;
	ah = al / ctxt->src.val;
	al %= ctxt->src.val;

	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);

	/* Set PF, ZF, SF */
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
	fastop(ctxt, em_or);

	return X86EMUL_CONTINUE;
}

3475 3476 3477 3478 3479 3480 3481 3482 3483
static int em_aad(struct x86_emulate_ctxt *ctxt)
{
	u8 al = ctxt->dst.val & 0xff;
	u8 ah = (ctxt->dst.val >> 8) & 0xff;

	al = (al + (ah * ctxt->src.val)) & 0xff;

	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;

3484 3485 3486 3487 3488
	/* Set PF, ZF, SF */
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
	fastop(ctxt, em_or);
3489 3490 3491 3492

	return X86EMUL_CONTINUE;
}

3493 3494
static int em_call(struct x86_emulate_ctxt *ctxt)
{
3495
	int rc;
3496 3497 3498
	long rel = ctxt->src.val;

	ctxt->src.val = (unsigned long)ctxt->_eip;
3499 3500 3501
	rc = jmp_rel(ctxt, rel);
	if (rc != X86EMUL_CONTINUE)
		return rc;
3502 3503 3504
	return em_push(ctxt);
}

3505 3506 3507 3508 3509
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;
3510 3511 3512
	struct desc_struct old_desc, new_desc;
	const struct x86_emulate_ops *ops = ctxt->ops;
	int cpl = ctxt->ops->cpl(ctxt);
3513
	enum x86emul_mode prev_mode = ctxt->mode;
3514

3515
	old_eip = ctxt->_eip;
3516
	ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3517

3518
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3519 3520
	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
				       X86_TRANSFER_CALL_JMP, &new_desc);
3521
	if (rc != X86EMUL_CONTINUE)
3522
		return rc;
3523

3524
	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3525 3526
	if (rc != X86EMUL_CONTINUE)
		goto fail;
3527

3528
	ctxt->src.val = old_cs;
3529
	rc = em_push(ctxt);
3530
	if (rc != X86EMUL_CONTINUE)
3531
		goto fail;
3532

3533
	ctxt->src.val = old_eip;
3534 3535 3536
	rc = em_push(ctxt);
	/* If we failed, we tainted the memory, but the very least we should
	   restore cs */
3537 3538
	if (rc != X86EMUL_CONTINUE) {
		pr_warn_once("faulting far call emulation tainted memory\n");
3539
		goto fail;
3540
	}
3541 3542 3543
	return rc;
fail:
	ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3544
	ctxt->mode = prev_mode;
3545 3546
	return rc;

3547 3548
}

3549 3550 3551
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;
3552
	unsigned long eip;
3553

3554 3555 3556 3557
	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	rc = assign_eip_near(ctxt, eip);
3558 3559
	if (rc != X86EMUL_CONTINUE)
		return rc;
3560
	rsp_increment(ctxt, ctxt->src.val);
3561 3562 3563
	return X86EMUL_CONTINUE;
}

3564 3565 3566
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
3567 3568
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
3569 3570

	/* Write back the memory destination with implicit LOCK prefix. */
3571 3572
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
3573 3574 3575
	return X86EMUL_CONTINUE;
}

3576 3577
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
3578
	ctxt->dst.val = ctxt->src2.val;
3579
	return fastop(ctxt, em_imul);
3580 3581
}

3582 3583
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
3584 3585
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
3586
	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3587
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3588 3589 3590 3591

	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3592 3593 3594 3595 3596 3597 3598 3599 3600 3601
static int em_rdpid(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc_aux = 0;

	if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
		return emulate_gp(ctxt, 0);
	ctxt->dst.val = tsc_aux;
	return X86EMUL_CONTINUE;
}

3602 3603 3604 3605
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

3606
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3607 3608
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3609 3610 3611
	return X86EMUL_CONTINUE;
}

3612 3613 3614 3615
static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
{
	u64 pmc;

3616
	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3617
		return emulate_gp(ctxt, 0);
3618 3619
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3620 3621 3622
	return X86EMUL_CONTINUE;
}

3623 3624
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
3625
	memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3626 3627 3628
	return X86EMUL_CONTINUE;
}

B
Borislav Petkov 已提交
3629 3630 3631 3632
static int em_movbe(struct x86_emulate_ctxt *ctxt)
{
	u16 tmp;

3633
	if (!ctxt->ops->guest_has_movbe(ctxt))
B
Borislav Petkov 已提交
3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656
		return emulate_ud(ctxt);

	switch (ctxt->op_bytes) {
	case 2:
		/*
		 * From MOVBE definition: "...When the operand size is 16 bits,
		 * the upper word of the destination register remains unchanged
		 * ..."
		 *
		 * Both casting ->valptr and ->val to u16 breaks strict aliasing
		 * rules so we have to do the operation almost per hand.
		 */
		tmp = (u16)ctxt->src.val;
		ctxt->dst.val &= ~0xffffUL;
		ctxt->dst.val |= (unsigned long)swab16(tmp);
		break;
	case 4:
		ctxt->dst.val = swab32((u32)ctxt->src.val);
		break;
	case 8:
		ctxt->dst.val = swab64(ctxt->src.val);
		break;
	default:
3657
		BUG();
B
Borislav Petkov 已提交
3658 3659 3660 3661
	}
	return X86EMUL_CONTINUE;
}

3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689
static int em_cr_write(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_dr_write(struct x86_emulate_ctxt *ctxt)
{
	unsigned long val;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		val = ctxt->src.val & ~0ULL;
	else
		val = ctxt->src.val & ~0U;

	/* #UD condition is already handled. */
	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3690 3691 3692 3693
static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3694 3695 3696
	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
	if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3697 3698 3699 3700 3701 3702 3703 3704 3705
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3706
	if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3707 3708
		return emulate_gp(ctxt, 0);

3709 3710
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
	*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3711 3712 3713
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3714
static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3715
{
P
Paolo Bonzini 已提交
3716 3717 3718 3719
	if (segment > VCPU_SREG_GS &&
	    (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
	    ctxt->ops->cpl(ctxt) > 0)
		return emulate_gp(ctxt, 0);
3720

P
Paolo Bonzini 已提交
3721
	ctxt->dst.val = get_segment_selector(ctxt, segment);
3722 3723
	if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
		ctxt->dst.bytes = 2;
3724 3725 3726
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3727 3728 3729 3730 3731 3732 3733 3734
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->modrm_reg > VCPU_SREG_GS)
		return emulate_ud(ctxt);

	return em_store_sreg(ctxt, ctxt->modrm_reg);
}

3735 3736
static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
3737
	u16 sel = ctxt->src.val;
3738

3739
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3740 3741
		return emulate_ud(ctxt);

3742
	if (ctxt->modrm_reg == VCPU_SREG_SS)
3743 3744 3745
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
3746 3747
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3748 3749
}

P
Paolo Bonzini 已提交
3750 3751 3752 3753 3754
static int em_sldt(struct x86_emulate_ctxt *ctxt)
{
	return em_store_sreg(ctxt, VCPU_SREG_LDTR);
}

A
Avi Kivity 已提交
3755 3756 3757 3758 3759 3760 3761 3762 3763
static int em_lldt(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
}

P
Paolo Bonzini 已提交
3764 3765 3766 3767 3768
static int em_str(struct x86_emulate_ctxt *ctxt)
{
	return em_store_sreg(ctxt, VCPU_SREG_TR);
}

A
Avi Kivity 已提交
3769 3770 3771 3772 3773 3774 3775 3776 3777
static int em_ltr(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
}

3778 3779
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
3780 3781 3782
	int rc;
	ulong linear;

3783
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3784
	if (rc == X86EMUL_CONTINUE)
3785
		ctxt->ops->invlpg(ctxt, linear);
3786
	/* Disable writeback. */
3787
	ctxt->dst.type = OP_NONE;
3788 3789 3790
	return X86EMUL_CONTINUE;
}

3791 3792 3793 3794 3795 3796 3797 3798 3799 3800
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

3801
static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3802
{
3803
	int rc = ctxt->ops->fix_hypercall(ctxt);
3804 3805 3806 3807 3808

	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
3809
	ctxt->_eip = ctxt->eip;
3810
	/* Disable writeback. */
3811
	ctxt->dst.type = OP_NONE;
3812 3813 3814
	return X86EMUL_CONTINUE;
}

3815 3816 3817 3818 3819 3820
static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
				  void (*get)(struct x86_emulate_ctxt *ctxt,
					      struct desc_ptr *ptr))
{
	struct desc_ptr desc_ptr;

P
Paolo Bonzini 已提交
3821 3822 3823 3824
	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
	    ctxt->ops->cpl(ctxt) > 0)
		return emulate_gp(ctxt, 0);

3825 3826 3827 3828 3829 3830 3831 3832 3833
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
	get(ctxt, &desc_ptr);
	if (ctxt->op_bytes == 2) {
		ctxt->op_bytes = 4;
		desc_ptr.address &= 0x00ffffff;
	}
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
3834 3835
	return segmented_write_std(ctxt, ctxt->dst.addr.mem,
				   &desc_ptr, 2 + ctxt->op_bytes);
3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847
}

static int em_sgdt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
}

static int em_sidt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
}

3848
static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3849 3850 3851 3852
{
	struct desc_ptr desc_ptr;
	int rc;

3853 3854
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
3855
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3856
			     &desc_ptr.size, &desc_ptr.address,
3857
			     ctxt->op_bytes);
3858 3859
	if (rc != X86EMUL_CONTINUE)
		return rc;
3860
	if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3861
	    emul_is_noncanonical_address(desc_ptr.address, ctxt))
3862
		return emulate_gp(ctxt, 0);
3863 3864 3865 3866
	if (lgdt)
		ctxt->ops->set_gdt(ctxt, &desc_ptr);
	else
		ctxt->ops->set_idt(ctxt, &desc_ptr);
3867
	/* Disable writeback. */
3868
	ctxt->dst.type = OP_NONE;
3869 3870 3871
	return X86EMUL_CONTINUE;
}

3872 3873 3874 3875 3876
static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	return em_lgdt_lidt(ctxt, true);
}

3877 3878
static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
3879
	return em_lgdt_lidt(ctxt, false);
3880 3881 3882 3883
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
P
Paolo Bonzini 已提交
3884 3885 3886 3887
	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
	    ctxt->ops->cpl(ctxt) > 0)
		return emulate_gp(ctxt, 0);

3888 3889
	if (ctxt->dst.type == OP_MEM)
		ctxt->dst.bytes = 2;
3890
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3891 3892 3893 3894 3895 3896
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3897 3898
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
3899 3900 3901
	return X86EMUL_CONTINUE;
}

3902 3903
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
3904 3905
	int rc = X86EMUL_CONTINUE;

3906
	register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3907
	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3908
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3909
		rc = jmp_rel(ctxt, ctxt->src.val);
3910

3911
	return rc;
3912 3913 3914 3915
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
3916 3917
	int rc = X86EMUL_CONTINUE;

3918
	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3919
		rc = jmp_rel(ctxt, ctxt->src.val);
3920

3921
	return rc;
3922 3923
}

3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941
static int em_in(struct x86_emulate_ctxt *ctxt)
{
	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
			     &ctxt->dst.val))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
}

static int em_out(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				    &ctxt->src.val, 1);
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3961 3962 3963
static int em_cpuid(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;
K
Kyle Huey 已提交
3964 3965 3966 3967 3968 3969 3970
	u64 msr = 0;

	ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
	if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
	    ctxt->ops->cpl(ctxt)) {
		return emulate_gp(ctxt, 0);
	}
A
Avi Kivity 已提交
3971

3972 3973
	eax = reg_read(ctxt, VCPU_REGS_RAX);
	ecx = reg_read(ctxt, VCPU_REGS_RCX);
3974
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3975 3976 3977 3978
	*reg_write(ctxt, VCPU_REGS_RAX) = eax;
	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;
	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = edx;
A
Avi Kivity 已提交
3979 3980 3981
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3982 3983 3984 3985
static int em_sahf(struct x86_emulate_ctxt *ctxt)
{
	u32 flags;

3986 3987
	flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
		X86_EFLAGS_SF;
P
Paolo Bonzini 已提交
3988 3989 3990 3991 3992 3993 3994
	flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;

	ctxt->eflags &= ~0xffUL;
	ctxt->eflags |= flags | X86_EFLAGS_FIXED;
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3995 3996
static int em_lahf(struct x86_emulate_ctxt *ctxt)
{
3997 3998
	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
A
Avi Kivity 已提交
3999 4000 4001
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016
static int em_bswap(struct x86_emulate_ctxt *ctxt)
{
	switch (ctxt->op_bytes) {
#ifdef CONFIG_X86_64
	case 8:
		asm("bswap %0" : "+r"(ctxt->dst.val));
		break;
#endif
	default:
		asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
		break;
	}
	return X86EMUL_CONTINUE;
}

4017 4018 4019 4020 4021 4022
static int em_clflush(struct x86_emulate_ctxt *ctxt)
{
	/* emulating clflush regardless of cpuid */
	return X86EMUL_CONTINUE;
}

4023 4024 4025 4026 4027 4028
static int em_movsxd(struct x86_emulate_ctxt *ctxt)
{
	ctxt->dst.val = (s32) ctxt->src.val;
	return X86EMUL_CONTINUE;
}

4029 4030
static int check_fxsr(struct x86_emulate_ctxt *ctxt)
{
4031
	if (!ctxt->ops->guest_has_fxsr(ctxt))
4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046
		return emulate_ud(ctxt);

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	/*
	 * Don't emulate a case that should never be hit, instead of working
	 * around a lack of fxsave64/fxrstor64 on old compilers.
	 */
	if (ctxt->mode >= X86EMUL_MODE_PROT64)
		return X86EMUL_UNHANDLEABLE;

	return X86EMUL_CONTINUE;
}

4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065
/*
 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
 * and restore MXCSR.
 */
static size_t __fxstate_size(int nregs)
{
	return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
}

static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
{
	bool cr4_osfxsr;
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return __fxstate_size(16);

	cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
	return __fxstate_size(cr4_osfxsr ? 8 : 0);
}

4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092
/*
 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
 *  1) 16 bit mode
 *  2) 32 bit mode
 *     - like (1), but FIP and FDP (foo) are only 16 bit.  At least Intel CPUs
 *       preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
 *       save and restore
 *  3) 64-bit mode with REX.W prefix
 *     - like (2), but XMM 8-15 are being saved and restored
 *  4) 64-bit mode without REX.W prefix
 *     - like (3), but FIP and FDP are 64 bit
 *
 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
 * desired result.  (4) is not emulated.
 *
 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
 * and FPU DS) should match.
 */
static int em_fxsave(struct x86_emulate_ctxt *ctxt)
{
	struct fxregs_state fx_state;
	int rc;

	rc = check_fxsr(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

4093 4094
	emulator_get_fpu();

4095 4096
	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));

4097 4098
	emulator_put_fpu();

4099 4100 4101
	if (rc != X86EMUL_CONTINUE)
		return rc;

4102 4103
	return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
		                   fxstate_size(ctxt));
4104 4105
}

4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125
/*
 * FXRSTOR might restore XMM registers not provided by the guest. Fill
 * in the host registers (via FXSAVE) instead, so they won't be modified.
 * (preemption has to stay disabled until FXRSTOR).
 *
 * Use noinline to keep the stack for other functions called by callers small.
 */
static noinline int fxregs_fixup(struct fxregs_state *fx_state,
				 const size_t used_size)
{
	struct fxregs_state fx_tmp;
	int rc;

	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
	memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
	       __fxstate_size(16) - used_size);

	return rc;
}

4126 4127 4128 4129
static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
{
	struct fxregs_state fx_state;
	int rc;
4130
	size_t size;
4131 4132 4133 4134 4135

	rc = check_fxsr(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

4136 4137 4138 4139 4140
	size = fxstate_size(ctxt);
	rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
	if (rc != X86EMUL_CONTINUE)
		return rc;

4141 4142
	emulator_get_fpu();

4143
	if (size < __fxstate_size(16)) {
4144
		rc = fxregs_fixup(&fx_state, size);
4145 4146 4147
		if (rc != X86EMUL_CONTINUE)
			goto out;
	}
4148

4149 4150 4151 4152
	if (fx_state.mxcsr >> 16) {
		rc = emulate_gp(ctxt, 0);
		goto out;
	}
4153 4154 4155 4156

	if (rc == X86EMUL_CONTINUE)
		rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));

4157
out:
4158 4159
	emulator_put_fpu();

4160 4161 4162
	return rc;
}

4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176
static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ecx, edx;

	eax = reg_read(ctxt, VCPU_REGS_RAX);
	edx = reg_read(ctxt, VCPU_REGS_RDX);
	ecx = reg_read(ctxt, VCPU_REGS_RCX);

	if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
4191
	if (!valid_cr(ctxt->modrm_reg))
4192 4193 4194 4195 4196 4197 4198
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
4199 4200
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
4201
	u64 efer = 0;
4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
4219
		u64 cr4;
4220 4221 4222 4223
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

4224 4225
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4226 4227 4228 4229 4230 4231 4232 4233 4234 4235

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

4236
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4237 4238
		if (efer & EFER_LMA) {
			u64 maxphyaddr;
4239
			u32 eax, ebx, ecx, edx;
4240

4241 4242 4243
			eax = 0x80000008;
			ecx = 0;
			if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4244
						 &edx, true))
4245 4246 4247
				maxphyaddr = eax & 0xff;
			else
				maxphyaddr = 36;
4248 4249
			rsvd = rsvd_bits(maxphyaddr, 63);
			if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4250
				rsvd &= ~X86_CR3_PCID_NOFLUSH;
4251
		}
4252 4253 4254 4255 4256 4257 4258

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
4259
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

4271 4272 4273 4274
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

4275
	ctxt->ops->get_dr(ctxt, 7, &dr7);
4276 4277 4278 4279 4280 4281 4282

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
4283
	int dr = ctxt->modrm_reg;
4284 4285 4286 4287 4288
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

4289
	cr4 = ctxt->ops->get_cr(ctxt, 4);
4290 4291 4292
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

4293 4294 4295 4296
	if (check_dr7_gd(ctxt)) {
		ulong dr6;

		ctxt->ops->get_dr(ctxt, 6, &dr6);
4297
		dr6 &= ~DR_TRAP_BITS;
4298 4299
		dr6 |= DR6_BD | DR6_RTM;
		ctxt->ops->set_dr(ctxt, 6, dr6);
4300
		return emulate_db(ctxt);
4301
	}
4302 4303 4304 4305 4306 4307

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
4308 4309
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
4310 4311 4312 4313 4314 4315 4316

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

4317 4318
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
4319
	u64 efer = 0;
4320

4321
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4322 4323 4324 4325 4326 4327 4328 4329 4330

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
4331
	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4332 4333

	/* Valid physical address? */
4334
	if (rax & 0xffff000000000000ULL)
4335 4336 4337 4338 4339
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

4340 4341
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
4342
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4343

4344
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4345 4346 4347 4348 4349
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

4350 4351
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
4352
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4353
	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4354

4355 4356 4357 4358 4359 4360 4361
	/*
	 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
	 * in Ring3 when CR4.PCE=0.
	 */
	if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
		return X86EMUL_CONTINUE;

4362
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4363
	    ctxt->ops->check_pmc(ctxt, rcx))
4364 4365 4366 4367 4368
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

4369 4370
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
4371 4372
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4373 4374 4375 4376 4377 4378 4379
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
4380 4381
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4382 4383 4384 4385 4386
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

4387
#define D(_y) { .flags = (_y) }
4388 4389 4390
#define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
#define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
		      .intercept = x86_intercept_##_i, .check_perm = (_p) }
4391
#define N    D(NotImpl)
4392
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4393 4394
#define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
#define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4395
#define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4396
#define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4397
#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4398
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4399
#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4400
#define II(_f, _e, _i) \
4401
	{ .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4402
#define IIP(_f, _e, _i, _p) \
4403 4404
	{ .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
	  .intercept = x86_intercept_##_i, .check_perm = (_p) }
4405
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4406

4407
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
4408
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4409
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
4410
#define F2bv(_f, _e)  F((_f) | ByteOp, _e), F(_f, _e)
4411 4412
#define I2bvIP(_f, _e, _i, _p) \
	IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4413

4414 4415 4416
#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4417

4418 4419
static const struct opcode group7_rm0[] = {
	N,
4420
	I(SrcNone | Priv | EmulateOnUD,	em_hypercall),
4421 4422 4423
	N, N, N, N, N, N,
};

4424
static const struct opcode group7_rm1[] = {
4425 4426
	DI(SrcNone | Priv, monitor),
	DI(SrcNone | Priv, mwait),
4427 4428 4429
	N, N, N, N, N, N,
};

4430 4431 4432 4433 4434 4435
static const struct opcode group7_rm2[] = {
	N,
	II(ImplicitOps | Priv,			em_xsetbv,	xsetbv),
	N, N, N, N, N, N,
};

4436
static const struct opcode group7_rm3[] = {
4437
	DIP(SrcNone | Prot | Priv,		vmrun,		check_svme_pa),
4438
	II(SrcNone  | Prot | EmulateOnUD,	em_hypercall,	vmmcall),
4439 4440 4441 4442 4443 4444
	DIP(SrcNone | Prot | Priv,		vmload,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		vmsave,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		stgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		clgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		skinit,		check_svme),
	DIP(SrcNone | Prot | Priv,		invlpga,	check_svme),
4445
};
4446

4447
static const struct opcode group7_rm7[] = {
4448
	N,
4449
	DIP(SrcNone, rdtscp, check_rdtsc),
4450 4451
	N, N, N, N, N, N,
};
4452

4453
static const struct opcode group1[] = {
4454 4455 4456 4457 4458 4459 4460 4461
	F(Lock, em_add),
	F(Lock | PageTable, em_or),
	F(Lock, em_adc),
	F(Lock, em_sbb),
	F(Lock | PageTable, em_and),
	F(Lock, em_sub),
	F(Lock, em_xor),
	F(NoWrite, em_cmp),
4462 4463
};

4464
static const struct opcode group1A[] = {
4465
	I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4466 4467
};

4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478
static const struct opcode group2[] = {
	F(DstMem | ModRM, em_rol),
	F(DstMem | ModRM, em_ror),
	F(DstMem | ModRM, em_rcl),
	F(DstMem | ModRM, em_rcr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_shr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_sar),
};

4479
static const struct opcode group3[] = {
4480 4481
	F(DstMem | SrcImm | NoWrite, em_test),
	F(DstMem | SrcImm | NoWrite, em_test),
4482 4483
	F(DstMem | SrcNone | Lock, em_not),
	F(DstMem | SrcNone | Lock, em_neg),
4484 4485
	F(DstXacc | Src2Mem, em_mul_ex),
	F(DstXacc | Src2Mem, em_imul_ex),
4486 4487
	F(DstXacc | Src2Mem, em_div_ex),
	F(DstXacc | Src2Mem, em_idiv_ex),
4488 4489
};

4490
static const struct opcode group4[] = {
4491 4492
	F(ByteOp | DstMem | SrcNone | Lock, em_inc),
	F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4493 4494 4495
	N, N, N, N, N, N,
};

4496
static const struct opcode group5[] = {
4497 4498
	F(DstMem | SrcNone | Lock,		em_inc),
	F(DstMem | SrcNone | Lock,		em_dec),
4499
	I(SrcMem | NearBranch,			em_call_near_abs),
4500
	I(SrcMemFAddr | ImplicitOps,		em_call_far),
4501
	I(SrcMem | NearBranch,			em_jmp_abs),
4502
	I(SrcMemFAddr | ImplicitOps,		em_jmp_far),
4503
	I(SrcMem | Stack | TwoMemOp,		em_push), D(Undefined),
4504 4505
};

4506
static const struct opcode group6[] = {
P
Paolo Bonzini 已提交
4507 4508
	II(Prot | DstMem,	   em_sldt, sldt),
	II(Prot | DstMem,	   em_str, str),
A
Avi Kivity 已提交
4509
	II(Prot | Priv | SrcMem16, em_lldt, lldt),
A
Avi Kivity 已提交
4510
	II(Prot | Priv | SrcMem16, em_ltr, ltr),
4511 4512 4513
	N, N, N, N,
};

4514
static const struct group_dual group7 = { {
4515 4516
	II(Mov | DstMem,			em_sgdt, sgdt),
	II(Mov | DstMem,			em_sidt, sidt),
4517 4518 4519 4520 4521
	II(SrcMem | Priv,			em_lgdt, lgdt),
	II(SrcMem | Priv,			em_lidt, lidt),
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	II(SrcMem | ByteOp | Priv | NoAccess,	em_invlpg, invlpg),
4522
}, {
4523
	EXT(0, group7_rm0),
4524
	EXT(0, group7_rm1),
4525 4526
	EXT(0, group7_rm2),
	EXT(0, group7_rm3),
4527 4528 4529
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	EXT(0, group7_rm7),
4530 4531
} };

4532
static const struct opcode group8[] = {
4533
	N, N, N, N,
4534 4535 4536 4537
	F(DstMem | SrcImmByte | NoWrite,		em_bt),
	F(DstMem | SrcImmByte | Lock | PageTable,	em_bts),
	F(DstMem | SrcImmByte | Lock,			em_btr),
	F(DstMem | SrcImmByte | Lock | PageTable,	em_btc),
4538 4539
};

P
Paolo Bonzini 已提交
4540 4541 4542 4543 4544 4545 4546 4547 4548
/*
 * The "memory" destination is actually always a register, since we come
 * from the register case of group9.
 */
static const struct gprefix pfx_0f_c7_7 = {
	N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
};


4549
static const struct group_dual group9 = { {
4550
	N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4551
}, {
P
Paolo Bonzini 已提交
4552 4553
	N, N, N, N, N, N, N,
	GP(0, &pfx_0f_c7_7),
4554 4555
} };

4556
static const struct opcode group11[] = {
4557
	I(DstMem | SrcImm | Mov | PageTable, em_mov),
4558
	X7(D(Undefined)),
4559 4560
};

4561
static const struct gprefix pfx_0f_ae_7 = {
4562
	I(SrcMem | ByteOp, em_clflush), N, N, N,
4563 4564 4565
};

static const struct group_dual group15 = { {
4566 4567 4568
	I(ModRM | Aligned16, em_fxsave),
	I(ModRM | Aligned16, em_fxrstor),
	N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4569 4570 4571 4572
}, {
	N, N, N, N, N, N, N, N,
} };

4573
static const struct gprefix pfx_0f_6f_0f_7f = {
4574
	I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4575 4576
};

4577 4578 4579 4580
static const struct instr_dual instr_dual_0f_2b = {
	I(0, em_mov), N
};

4581
static const struct gprefix pfx_0f_2b = {
4582
	ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4583 4584
};

4585 4586 4587 4588
static const struct gprefix pfx_0f_10_0f_11 = {
	I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
};

4589
static const struct gprefix pfx_0f_28_0f_29 = {
4590
	I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4591 4592
};

4593 4594 4595 4596
static const struct gprefix pfx_0f_e7 = {
	N, I(Sse, em_mov), N, N,
};

4597
static const struct escape escape_d9 = { {
4598
	N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_db = { {
	N, N, N, N, N, N, N, N,
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_dd = { {
4640
	N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

4660 4661 4662 4663
static const struct instr_dual instr_dual_0f_c3 = {
	I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
};

4664 4665 4666 4667
static const struct mode_dual mode_dual_63 = {
	N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
};

4668
static const struct opcode opcode_table[256] = {
4669
	/* 0x00 - 0x07 */
4670
	F6ALU(Lock, em_add),
4671 4672
	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4673
	/* 0x08 - 0x0F */
4674
	F6ALU(Lock | PageTable, em_or),
4675 4676
	I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
	N,
4677
	/* 0x10 - 0x17 */
4678
	F6ALU(Lock, em_adc),
4679 4680
	I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4681
	/* 0x18 - 0x1F */
4682
	F6ALU(Lock, em_sbb),
4683 4684
	I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4685
	/* 0x20 - 0x27 */
4686
	F6ALU(Lock | PageTable, em_and), N, N,
4687
	/* 0x28 - 0x2F */
4688
	F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4689
	/* 0x30 - 0x37 */
4690
	F6ALU(Lock, em_xor), N, N,
4691
	/* 0x38 - 0x3F */
4692
	F6ALU(NoWrite, em_cmp), N, N,
4693
	/* 0x40 - 0x4F */
4694
	X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4695
	/* 0x50 - 0x57 */
4696
	X8(I(SrcReg | Stack, em_push)),
4697
	/* 0x58 - 0x5F */
4698
	X8(I(DstReg | Stack, em_pop)),
4699
	/* 0x60 - 0x67 */
4700 4701
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
4702
	N, MD(ModRM, &mode_dual_63),
4703 4704
	N, N, N, N,
	/* 0x68 - 0x6F */
4705 4706
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4707 4708
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4709
	I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4710
	I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4711
	/* 0x70 - 0x7F */
4712
	X16(D(SrcImmByte | NearBranch)),
4713
	/* 0x80 - 0x87 */
4714 4715 4716 4717
	G(ByteOp | DstMem | SrcImm, group1),
	G(DstMem | SrcImm, group1),
	G(ByteOp | DstMem | SrcImm | No64, group1),
	G(DstMem | SrcImmByte, group1),
4718
	F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4719
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4720
	/* 0x88 - 0x8F */
4721
	I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4722
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4723
	I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4724 4725 4726
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
4727
	/* 0x90 - 0x97 */
4728
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4729
	/* 0x98 - 0x9F */
4730
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4731
	I(SrcImmFAddr | No64, em_call_far), N,
4732
	II(ImplicitOps | Stack, em_pushf, pushf),
P
Paolo Bonzini 已提交
4733 4734
	II(ImplicitOps | Stack, em_popf, popf),
	I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4735
	/* 0xA0 - 0xA7 */
4736
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4737
	I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4738 4739
	I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
	F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4740
	/* 0xA8 - 0xAF */
4741
	F2bv(DstAcc | SrcImm | NoWrite, em_test),
4742 4743
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4744
	F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4745
	/* 0xB0 - 0xB7 */
4746
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4747
	/* 0xB8 - 0xBF */
4748
	X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4749
	/* 0xC0 - 0xC7 */
4750
	G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4751 4752
	I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
	I(ImplicitOps | NearBranch, em_ret),
4753 4754
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4755
	G(ByteOp, group11), G(0, group11),
4756
	/* 0xC8 - 0xCF */
A
Avi Kivity 已提交
4757
	I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4758 4759
	I(ImplicitOps | SrcImmU16, em_ret_far_imm),
	I(ImplicitOps, em_ret_far),
4760
	D(ImplicitOps), DI(SrcImmByte, intn),
4761
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4762
	/* 0xD0 - 0xD7 */
4763 4764
	G(Src2One | ByteOp, group2), G(Src2One, group2),
	G(Src2CL | ByteOp, group2), G(Src2CL, group2),
P
Paolo Bonzini 已提交
4765
	I(DstAcc | SrcImmUByte | No64, em_aam),
P
Paolo Bonzini 已提交
4766 4767
	I(DstAcc | SrcImmUByte | No64, em_aad),
	F(DstAcc | ByteOp | No64, em_salc),
P
Paolo Bonzini 已提交
4768
	I(DstAcc | SrcXLat | ByteOp, em_mov),
4769
	/* 0xD8 - 0xDF */
4770
	N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4771
	/* 0xE0 - 0xE7 */
4772 4773
	X3(I(SrcImmByte | NearBranch, em_loop)),
	I(SrcImmByte | NearBranch, em_jcxz),
4774 4775
	I2bvIP(SrcImmUByte | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4776
	/* 0xE8 - 0xEF */
4777 4778 4779
	I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
	I(SrcImmFAddr | No64, em_jmp_far),
	D(SrcImmByte | ImplicitOps | NearBranch),
4780 4781
	I2bvIP(SrcDX | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4782
	/* 0xF0 - 0xF7 */
4783
	N, DI(ImplicitOps, icebp), N, N,
4784 4785
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
4786
	/* 0xF8 - 0xFF */
4787 4788
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4789 4790 4791
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

4792
static const struct opcode twobyte_table[256] = {
4793
	/* 0x00 - 0x0F */
4794
	G(0, group6), GD(0, &group7), N, N,
4795
	N, I(ImplicitOps | EmulateOnUD, em_syscall),
4796
	II(ImplicitOps | Priv, em_clts, clts), N,
4797
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4798
	N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4799
	/* 0x10 - 0x1F */
4800 4801 4802
	GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
	GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
	N, N, N, N, N, N,
4803 4804
	D(ImplicitOps | ModRM | SrcMem | NoAccess),
	N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4805
	/* 0x20 - 0x2F */
4806 4807 4808 4809 4810 4811
	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
						check_cr_write),
	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
						check_dr_write),
4812
	N, N, N, N,
4813 4814
	GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
	GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4815
	N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4816
	N, N, N, N,
4817
	/* 0x30 - 0x3F */
4818
	II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4819
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4820
	II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4821
	IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4822 4823
	I(ImplicitOps | EmulateOnUD, em_sysenter),
	I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4824
	N, N,
4825 4826
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
4827
	X16(D(DstReg | SrcMem | ModRM)),
4828 4829 4830
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
4831 4832 4833 4834
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4835
	/* 0x70 - 0x7F */
4836 4837 4838 4839
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4840
	/* 0x80 - 0x8F */
4841
	X16(D(SrcImm | NearBranch)),
4842
	/* 0x90 - 0x9F */
4843
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4844
	/* 0xA0 - 0xA7 */
4845
	I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4846 4847
	II(ImplicitOps, em_cpuid, cpuid),
	F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4848 4849
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4850
	/* 0xA8 - 0xAF */
4851
	I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4852
	II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4853
	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4854 4855
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4856
	GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4857
	/* 0xB0 - 0xB7 */
4858
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4859
	I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4860
	F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4861 4862
	I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4863
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4864 4865
	/* 0xB8 - 0xBF */
	N, N,
4866
	G(BitOp, group8),
4867
	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4868 4869
	I(DstReg | SrcMem | ModRM, em_bsf_c),
	I(DstReg | SrcMem | ModRM, em_bsr_c),
4870
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
A
Avi Kivity 已提交
4871
	/* 0xC0 - 0xC7 */
4872
	F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4873
	N, ID(0, &instr_dual_0f_c3),
4874
	N, N, N, GD(0, &group9),
A
Avi Kivity 已提交
4875 4876
	/* 0xC8 - 0xCF */
	X8(I(DstReg, em_bswap)),
4877 4878 4879
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
4880 4881
	N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
	N, N, N, N, N, N, N, N,
4882 4883 4884 4885
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

4886 4887 4888 4889 4890 4891 4892 4893
static const struct instr_dual instr_dual_0f_38_f0 = {
	I(DstReg | SrcMem | Mov, em_movbe), N
};

static const struct instr_dual instr_dual_0f_38_f1 = {
	I(DstMem | SrcReg | Mov, em_movbe), N
};

4894
static const struct gprefix three_byte_0f_38_f0 = {
4895
	ID(0, &instr_dual_0f_38_f0), N, N, N
4896 4897 4898
};

static const struct gprefix three_byte_0f_38_f1 = {
4899
	ID(0, &instr_dual_0f_38_f1), N, N, N
4900 4901 4902 4903 4904 4905 4906 4907 4908
};

/*
 * Insns below are selected by the prefix which indexed by the third opcode
 * byte.
 */
static const struct opcode opcode_map_0f_38[256] = {
	/* 0x00 - 0x7f */
	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
B
Borislav Petkov 已提交
4909 4910 4911
	/* 0x80 - 0xef */
	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
	/* 0xf0 - 0xf1 */
4912 4913
	GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
	GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
B
Borislav Petkov 已提交
4914 4915
	/* 0xf2 - 0xff */
	N, N, X4(N), X8(N)
4916 4917
};

4918 4919 4920 4921 4922
#undef D
#undef N
#undef G
#undef GD
#undef I
4923
#undef GP
4924
#undef EXT
4925
#undef MD
N
Nadav Amit 已提交
4926
#undef ID
4927

4928
#undef D2bv
4929
#undef D2bvIP
4930
#undef I2bv
4931
#undef I2bvIP
4932
#undef I6ALU
4933

4934
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4935 4936 4937
{
	unsigned size;

4938
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
4951
	op->addr.mem.ea = ctxt->_eip;
4952 4953 4954
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
4955
		op->val = insn_fetch(s8, ctxt);
4956 4957
		break;
	case 2:
4958
		op->val = insn_fetch(s16, ctxt);
4959 4960
		break;
	case 4:
4961
		op->val = insn_fetch(s32, ctxt);
4962
		break;
4963 4964 4965
	case 8:
		op->val = insn_fetch(s64, ctxt);
		break;
4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

4984 4985 4986 4987 4988 4989 4990
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
			  unsigned d)
{
	int rc = X86EMUL_CONTINUE;

	switch (d) {
	case OpReg:
4991
		decode_register_operand(ctxt, op);
4992 4993
		break;
	case OpImmUByte:
4994
		rc = decode_imm(ctxt, op, 1, false);
4995 4996
		break;
	case OpMem:
4997
		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4998 4999 5000
	mem_common:
		*op = ctxt->memop;
		ctxt->memopp = op;
5001
		if (ctxt->d & BitOp)
5002 5003 5004
			fetch_bit_operand(ctxt);
		op->orig_val = op->val;
		break;
5005
	case OpMem64:
5006
		ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
5007
		goto mem_common;
5008 5009 5010
	case OpAcc:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5011
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5012 5013 5014
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032
	case OpAccLo:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
	case OpAccHi:
		if (ctxt->d & ByteOp) {
			op->type = OP_NONE;
			break;
		}
		op->type = OP_REG;
		op->bytes = ctxt->op_bytes;
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
5033 5034 5035 5036
	case OpDI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
5037
			register_address(ctxt, VCPU_REGS_RDI);
5038 5039
		op->addr.mem.seg = VCPU_SREG_ES;
		op->val = 0;
5040
		op->count = 1;
5041 5042 5043 5044
		break;
	case OpDX:
		op->type = OP_REG;
		op->bytes = 2;
5045
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5046 5047
		fetch_register_operand(op);
		break;
5048
	case OpCL:
5049
		op->type = OP_IMM;
5050
		op->bytes = 1;
5051
		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
5052 5053 5054 5055 5056
		break;
	case OpImmByte:
		rc = decode_imm(ctxt, op, 1, true);
		break;
	case OpOne:
5057
		op->type = OP_IMM;
5058 5059 5060 5061 5062 5063
		op->bytes = 1;
		op->val = 1;
		break;
	case OpImm:
		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
		break;
5064 5065 5066
	case OpImm64:
		rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
		break;
5067 5068
	case OpMem8:
		ctxt->memop.bytes = 1;
5069
		if (ctxt->memop.type == OP_REG) {
5070 5071
			ctxt->memop.addr.reg = decode_register(ctxt,
					ctxt->modrm_rm, true);
5072 5073
			fetch_register_operand(&ctxt->memop);
		}
5074
		goto mem_common;
5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090
	case OpMem16:
		ctxt->memop.bytes = 2;
		goto mem_common;
	case OpMem32:
		ctxt->memop.bytes = 4;
		goto mem_common;
	case OpImmU16:
		rc = decode_imm(ctxt, op, 2, false);
		break;
	case OpImmU:
		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
		break;
	case OpSI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
5091
			register_address(ctxt, VCPU_REGS_RSI);
B
Bandan Das 已提交
5092
		op->addr.mem.seg = ctxt->seg_override;
5093
		op->val = 0;
5094
		op->count = 1;
5095
		break;
P
Paolo Bonzini 已提交
5096 5097 5098 5099
	case OpXLat:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
5100
			address_mask(ctxt,
P
Paolo Bonzini 已提交
5101 5102
				reg_read(ctxt, VCPU_REGS_RBX) +
				(reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
B
Bandan Das 已提交
5103
		op->addr.mem.seg = ctxt->seg_override;
P
Paolo Bonzini 已提交
5104 5105
		op->val = 0;
		break;
5106 5107 5108 5109 5110 5111 5112 5113 5114
	case OpImmFAddr:
		op->type = OP_IMM;
		op->addr.mem.ea = ctxt->_eip;
		op->bytes = ctxt->op_bytes + 2;
		insn_fetch_arr(op->valptr, op->bytes, ctxt);
		break;
	case OpMemFAddr:
		ctxt->memop.bytes = ctxt->op_bytes + 2;
		goto mem_common;
5115
	case OpES:
5116
		op->type = OP_IMM;
5117 5118 5119
		op->val = VCPU_SREG_ES;
		break;
	case OpCS:
5120
		op->type = OP_IMM;
5121 5122 5123
		op->val = VCPU_SREG_CS;
		break;
	case OpSS:
5124
		op->type = OP_IMM;
5125 5126 5127
		op->val = VCPU_SREG_SS;
		break;
	case OpDS:
5128
		op->type = OP_IMM;
5129 5130 5131
		op->val = VCPU_SREG_DS;
		break;
	case OpFS:
5132
		op->type = OP_IMM;
5133 5134 5135
		op->val = VCPU_SREG_FS;
		break;
	case OpGS:
5136
		op->type = OP_IMM;
5137 5138
		op->val = VCPU_SREG_GS;
		break;
5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149
	case OpImplicit:
		/* Special instructions do their own operand decoding. */
	default:
		op->type = OP_NONE; /* Disable writeback. */
		break;
	}

done:
	return rc;
}

5150
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5151 5152 5153
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
5154
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5155
	bool op_prefix = false;
B
Bandan Das 已提交
5156
	bool has_seg_override = false;
5157
	struct opcode opcode;
5158 5159
	u16 dummy;
	struct desc_struct desc;
5160

5161 5162
	ctxt->memop.type = OP_NONE;
	ctxt->memopp = NULL;
5163
	ctxt->_eip = ctxt->eip;
5164 5165
	ctxt->fetch.ptr = ctxt->fetch.data;
	ctxt->fetch.end = ctxt->fetch.data + insn_len;
B
Borislav Petkov 已提交
5166
	ctxt->opcode_len = 1;
5167
	ctxt->intercept = x86_intercept_none;
5168
	if (insn_len > 0)
5169
		memcpy(ctxt->fetch.data, insn, insn_len);
5170
	else {
5171
		rc = __do_insn_fetch_bytes(ctxt, 1);
5172
		if (rc != X86EMUL_CONTINUE)
5173
			goto done;
5174
	}
5175 5176 5177 5178

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
5179 5180 5181 5182 5183
		def_op_bytes = def_ad_bytes = 2;
		ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
		if (desc.d)
			def_op_bytes = def_ad_bytes = 4;
		break;
5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
5197
		return EMULATION_FAILED;
5198 5199
	}

5200 5201
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
5202 5203 5204

	/* Legacy prefixes. */
	for (;;) {
5205
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
5206
		case 0x66:	/* operand-size override */
5207
			op_prefix = true;
5208
			/* switch between 2/4 bytes */
5209
			ctxt->op_bytes = def_op_bytes ^ 6;
5210 5211 5212 5213
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
5214
				ctxt->ad_bytes = def_ad_bytes ^ 12;
5215 5216
			else
				/* switch between 2/4 bytes */
5217
				ctxt->ad_bytes = def_ad_bytes ^ 6;
5218 5219
			break;
		case 0x26:	/* ES override */
5220 5221 5222
			has_seg_override = true;
			ctxt->seg_override = VCPU_SREG_ES;
			break;
5223
		case 0x2e:	/* CS override */
5224 5225 5226
			has_seg_override = true;
			ctxt->seg_override = VCPU_SREG_CS;
			break;
5227
		case 0x36:	/* SS override */
5228 5229 5230
			has_seg_override = true;
			ctxt->seg_override = VCPU_SREG_SS;
			break;
5231
		case 0x3e:	/* DS override */
B
Bandan Das 已提交
5232
			has_seg_override = true;
5233
			ctxt->seg_override = VCPU_SREG_DS;
5234 5235
			break;
		case 0x64:	/* FS override */
5236 5237 5238
			has_seg_override = true;
			ctxt->seg_override = VCPU_SREG_FS;
			break;
5239
		case 0x65:	/* GS override */
B
Bandan Das 已提交
5240
			has_seg_override = true;
5241
			ctxt->seg_override = VCPU_SREG_GS;
5242 5243 5244 5245
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
5246
			ctxt->rex_prefix = ctxt->b;
5247 5248
			continue;
		case 0xf0:	/* LOCK */
5249
			ctxt->lock_prefix = 1;
5250 5251 5252
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
5253
			ctxt->rep_prefix = ctxt->b;
5254 5255 5256 5257 5258 5259 5260
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

5261
		ctxt->rex_prefix = 0;
5262 5263 5264 5265 5266
	}

done_prefixes:

	/* REX prefix. */
5267 5268
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
5269 5270

	/* Opcode byte(s). */
5271
	opcode = opcode_table[ctxt->b];
5272
	/* Two-byte opcode? */
5273
	if (ctxt->b == 0x0f) {
B
Borislav Petkov 已提交
5274
		ctxt->opcode_len = 2;
5275
		ctxt->b = insn_fetch(u8, ctxt);
5276
		opcode = twobyte_table[ctxt->b];
5277 5278 5279 5280 5281 5282 5283

		/* 0F_38 opcode map */
		if (ctxt->b == 0x38) {
			ctxt->opcode_len = 3;
			ctxt->b = insn_fetch(u8, ctxt);
			opcode = opcode_map_0f_38[ctxt->b];
		}
5284
	}
5285
	ctxt->d = opcode.flags;
5286

5287 5288 5289
	if (ctxt->d & ModRM)
		ctxt->modrm = insn_fetch(u8, ctxt);

5290 5291
	/* vex-prefix instructions are not implemented */
	if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5292
	    (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5293 5294 5295
		ctxt->d = NotImpl;
	}

5296 5297
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
5298
		case Group:
5299
			goffset = (ctxt->modrm >> 3) & 7;
5300 5301 5302
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
5303 5304
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
5305 5306 5307 5308 5309
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
5310
			goffset = ctxt->modrm & 7;
5311
			opcode = opcode.u.group[goffset];
5312 5313
			break;
		case Prefix:
5314
			if (ctxt->rep_prefix && op_prefix)
5315
				return EMULATION_FAILED;
5316
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5317 5318 5319 5320 5321 5322 5323
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
5324
		case Escape:
5325 5326 5327 5328 5329 5330 5331
			if (ctxt->modrm > 0xbf) {
				size_t size = ARRAY_SIZE(opcode.u.esc->high);
				u32 index = array_index_nospec(
					ctxt->modrm - 0xc0, size);

				opcode = opcode.u.esc->high[index];
			} else {
5332
				opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5333
			}
5334
			break;
5335 5336 5337 5338 5339 5340
		case InstrDual:
			if ((ctxt->modrm >> 6) == 3)
				opcode = opcode.u.idual->mod3;
			else
				opcode = opcode.u.idual->mod012;
			break;
5341 5342 5343 5344 5345 5346
		case ModeDual:
			if (ctxt->mode == X86EMUL_MODE_PROT64)
				opcode = opcode.u.mdual->mode64;
			else
				opcode = opcode.u.mdual->mode32;
			break;
5347
		default:
5348
			return EMULATION_FAILED;
5349
		}
5350

5351
		ctxt->d &= ~(u64)GroupMask;
5352
		ctxt->d |= opcode.flags;
5353 5354
	}

5355 5356 5357 5358
	/* Unrecognised? */
	if (ctxt->d == 0)
		return EMULATION_FAILED;

5359
	ctxt->execute = opcode.u.execute;
5360

5361 5362 5363
	if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
		return EMULATION_FAILED;

5364
	if (unlikely(ctxt->d &
5365 5366
	    (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
	     No16))) {
5367 5368 5369 5370 5371 5372
		/*
		 * These are copied unconditionally here, and checked unconditionally
		 * in x86_emulate_insn.
		 */
		ctxt->check_perm = opcode.check_perm;
		ctxt->intercept = opcode.intercept;
5373

5374 5375
		if (ctxt->d & NotImpl)
			return EMULATION_FAILED;
5376

5377 5378 5379 5380 5381 5382
		if (mode == X86EMUL_MODE_PROT64) {
			if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
				ctxt->op_bytes = 8;
			else if (ctxt->d & NearBranch)
				ctxt->op_bytes = 8;
		}
5383

5384 5385 5386 5387 5388 5389 5390
		if (ctxt->d & Op3264) {
			if (mode == X86EMUL_MODE_PROT64)
				ctxt->op_bytes = 8;
			else
				ctxt->op_bytes = 4;
		}

5391 5392 5393
		if ((ctxt->d & No16) && ctxt->op_bytes == 2)
			ctxt->op_bytes = 4;

5394 5395 5396 5397 5398
		if (ctxt->d & Sse)
			ctxt->op_bytes = 16;
		else if (ctxt->d & Mmx)
			ctxt->op_bytes = 8;
	}
A
Avi Kivity 已提交
5399

5400
	/* ModRM and SIB bytes. */
5401
	if (ctxt->d & ModRM) {
5402
		rc = decode_modrm(ctxt, &ctxt->memop);
B
Bandan Das 已提交
5403 5404 5405 5406
		if (!has_seg_override) {
			has_seg_override = true;
			ctxt->seg_override = ctxt->modrm_seg;
		}
5407
	} else if (ctxt->d & MemAbs)
5408
		rc = decode_abs(ctxt, &ctxt->memop);
5409 5410 5411
	if (rc != X86EMUL_CONTINUE)
		goto done;

B
Bandan Das 已提交
5412 5413
	if (!has_seg_override)
		ctxt->seg_override = VCPU_SREG_DS;
5414

B
Bandan Das 已提交
5415
	ctxt->memop.addr.mem.seg = ctxt->seg_override;
5416 5417 5418 5419 5420

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
5421
	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5422 5423 5424
	if (rc != X86EMUL_CONTINUE)
		goto done;

5425 5426 5427 5428
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
5429
	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5430 5431 5432
	if (rc != X86EMUL_CONTINUE)
		goto done;

5433
	/* Decode and fetch the destination operand: register or memory. */
5434
	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5435

5436
	if (ctxt->rip_relative && likely(ctxt->memopp))
5437 5438
		ctxt->memopp->addr.mem.ea = address_mask(ctxt,
					ctxt->memopp->addr.mem.ea + ctxt->_eip);
5439

5440
done:
5441 5442
	if (rc == X86EMUL_PROPAGATE_FAULT)
		ctxt->have_exception = true;
5443
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5444 5445
}

5446 5447 5448 5449 5450
bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
{
	return ctxt->d & PageTable;
}

5451 5452 5453 5454 5455 5456 5457 5458 5459
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
5460 5461 5462
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
5463
		 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5464
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
5465
		    ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5466 5467 5468 5469 5470
		return true;

	return false;
}

A
Avi Kivity 已提交
5471 5472
static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
{
R
Radim Krčmář 已提交
5473
	int rc;
A
Avi Kivity 已提交
5474

5475
	emulator_get_fpu();
R
Radim Krčmář 已提交
5476
	rc = asm_safe("fwait");
5477
	emulator_put_fpu();
A
Avi Kivity 已提交
5478

R
Radim Krčmář 已提交
5479
	if (unlikely(rc != X86EMUL_CONTINUE))
A
Avi Kivity 已提交
5480 5481 5482 5483 5484
		return emulate_exception(ctxt, MF_VECTOR, 0, false);

	return X86EMUL_CONTINUE;
}

5485
static void fetch_possible_mmx_operand(struct operand *op)
A
Avi Kivity 已提交
5486 5487
{
	if (op->type == OP_MM)
5488
		read_mmx_reg(&op->mm_val, op->addr.mm);
A
Avi Kivity 已提交
5489 5490
}

5491
static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5492 5493
{
	ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5494

5495 5496
	if (!(ctxt->d & ByteOp))
		fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5497

5498
	asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5499
	    : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5500
	      [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5501
	    : "c"(ctxt->src2.val));
5502

5503
	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5504 5505
	if (!fop) /* exception is returned in fop variable */
		return emulate_de(ctxt);
5506 5507
	return X86EMUL_CONTINUE;
}
5508

5509 5510
void init_decode_cache(struct x86_emulate_ctxt *ctxt)
{
B
Bandan Das 已提交
5511 5512
	memset(&ctxt->rip_relative, 0,
	       (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5513 5514 5515 5516 5517 5518

	ctxt->io_read.pos = 0;
	ctxt->io_read.end = 0;
	ctxt->mem_read.end = 0;
}

5519
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5520
{
5521
	const struct x86_emulate_ops *ops = ctxt->ops;
5522
	int rc = X86EMUL_CONTINUE;
5523
	int saved_dst_type = ctxt->dst.type;
5524
	unsigned emul_flags;
5525

5526
	ctxt->mem_read.pos = 0;
5527

5528 5529
	/* LOCK prefix is allowed only with some instructions */
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5530
		rc = emulate_ud(ctxt);
5531 5532 5533
		goto done;
	}

5534
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5535
		rc = emulate_ud(ctxt);
5536 5537 5538
		goto done;
	}

5539
	emul_flags = ctxt->ops->get_hflags(ctxt);
5540 5541 5542 5543 5544 5545 5546
	if (unlikely(ctxt->d &
		     (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
		if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
				(ctxt->d & Undefined)) {
			rc = emulate_ud(ctxt);
			goto done;
		}
A
Avi Kivity 已提交
5547

5548 5549 5550
		if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
		    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
			rc = emulate_ud(ctxt);
A
Avi Kivity 已提交
5551
			goto done;
5552
		}
A
Avi Kivity 已提交
5553

5554 5555
		if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
			rc = emulate_nm(ctxt);
5556
			goto done;
5557
		}
5558

5559 5560 5561 5562 5563 5564 5565 5566
		if (ctxt->d & Mmx) {
			rc = flush_pending_x87_faults(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			/*
			 * Now that we know the fpu is exception safe, we can fetch
			 * operands from it.
			 */
5567 5568
			fetch_possible_mmx_operand(&ctxt->src);
			fetch_possible_mmx_operand(&ctxt->src2);
5569
			if (!(ctxt->d & Mov))
5570
				fetch_possible_mmx_operand(&ctxt->dst);
5571
		}
5572

5573
		if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5574 5575 5576 5577 5578
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_PRE_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}
5579

5580 5581 5582 5583 5584 5585
		/* Instruction can only be executed in protected mode */
		if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
			rc = emulate_ud(ctxt);
			goto done;
		}

5586 5587
		/* Privileged instruction can be executed only in CPL=0 */
		if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5588 5589 5590 5591
			if (ctxt->d & PrivUD)
				rc = emulate_ud(ctxt);
			else
				rc = emulate_gp(ctxt, 0);
5592
			goto done;
5593
		}
5594

5595
		/* Do instruction specific permission checks */
5596
		if (ctxt->d & CheckPerm) {
5597 5598 5599 5600 5601
			rc = ctxt->check_perm(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

5602
		if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5603 5604 5605 5606 5607 5608 5609 5610 5611
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_POST_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

		if (ctxt->rep_prefix && (ctxt->d & String)) {
			/* All REP prefixes have the same first termination condition */
			if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5612
				string_registers_quirk(ctxt);
5613
				ctxt->eip = ctxt->_eip;
5614
				ctxt->eflags &= ~X86_EFLAGS_RF;
5615 5616
				goto done;
			}
5617 5618 5619
		}
	}

5620 5621 5622
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
5623
		if (rc != X86EMUL_CONTINUE)
5624
			goto done;
5625
		ctxt->src.orig_val64 = ctxt->src.val64;
5626 5627
	}

5628 5629 5630
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
5631 5632 5633 5634
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

5635
	if ((ctxt->d & DstMask) == ImplicitOps)
5636 5637 5638
		goto special_insn;


5639
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5640
		/* optimisation - avoid slow emulated read if Mov */
5641 5642
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
5643
		if (rc != X86EMUL_CONTINUE) {
5644 5645
			if (!(ctxt->d & NoWrite) &&
			    rc == X86EMUL_PROPAGATE_FAULT &&
5646 5647
			    ctxt->exception.vector == PF_VECTOR)
				ctxt->exception.error_code |= PFERR_WRITE_MASK;
5648
			goto done;
5649
		}
5650
	}
5651 5652
	/* Copy full 64-bit value for CMPXCHG8B.  */
	ctxt->dst.orig_val64 = ctxt->dst.val64;
5653

5654 5655
special_insn:

5656
	if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5657
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
5658
					      X86_ICPT_POST_MEMACCESS);
5659 5660 5661 5662
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

5663
	if (ctxt->rep_prefix && (ctxt->d & String))
5664
		ctxt->eflags |= X86_EFLAGS_RF;
5665
	else
5666
		ctxt->eflags &= ~X86_EFLAGS_RF;
5667

5668
	if (ctxt->execute) {
5669
		if (ctxt->d & Fastop)
5670
			rc = fastop(ctxt, ctxt->fop);
5671
		else
5672
			rc = ctxt->execute(ctxt);
5673 5674 5675 5676 5677
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

B
Borislav Petkov 已提交
5678
	if (ctxt->opcode_len == 2)
A
Avi Kivity 已提交
5679
		goto twobyte_insn;
5680 5681
	else if (ctxt->opcode_len == 3)
		goto threebyte_insn;
A
Avi Kivity 已提交
5682

5683
	switch (ctxt->b) {
5684
	case 0x70 ... 0x7f: /* jcc (short) */
5685
		if (test_cc(ctxt->b, ctxt->eflags))
5686
			rc = jmp_rel(ctxt, ctxt->src.val);
5687
		break;
N
Nitin A Kamble 已提交
5688
	case 0x8d: /* lea r16/r32, m */
5689
		ctxt->dst.val = ctxt->src.addr.mem.ea;
N
Nitin A Kamble 已提交
5690
		break;
5691
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
5692
		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5693 5694 5695
			ctxt->dst.type = OP_NONE;
		else
			rc = em_xchg(ctxt);
5696
		break;
5697
	case 0x98: /* cbw/cwde/cdqe */
5698 5699 5700 5701
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5702 5703
		}
		break;
5704
	case 0xcc:		/* int3 */
5705 5706
		rc = emulate_int(ctxt, 3);
		break;
5707
	case 0xcd:		/* int n */
5708
		rc = emulate_int(ctxt, ctxt->src.val);
5709 5710
		break;
	case 0xce:		/* into */
5711
		if (ctxt->eflags & X86_EFLAGS_OF)
5712
			rc = emulate_int(ctxt, 4);
5713
		break;
5714
	case 0xe9: /* jmp rel */
5715
	case 0xeb: /* jmp rel short */
5716
		rc = jmp_rel(ctxt, ctxt->src.val);
5717
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
5718
		break;
5719
	case 0xf4:              /* hlt */
5720
		ctxt->ops->halt(ctxt);
5721
		break;
5722 5723
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
5724
		ctxt->eflags ^= X86_EFLAGS_CF;
5725 5726
		break;
	case 0xf8: /* clc */
5727
		ctxt->eflags &= ~X86_EFLAGS_CF;
5728
		break;
5729
	case 0xf9: /* stc */
5730
		ctxt->eflags |= X86_EFLAGS_CF;
5731
		break;
5732
	case 0xfc: /* cld */
5733
		ctxt->eflags &= ~X86_EFLAGS_DF;
5734 5735
		break;
	case 0xfd: /* std */
5736
		ctxt->eflags |= X86_EFLAGS_DF;
5737
		break;
5738 5739
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
5740
	}
5741

5742 5743 5744
	if (rc != X86EMUL_CONTINUE)
		goto done;

5745
writeback:
5746 5747 5748 5749 5750 5751
	if (ctxt->d & SrcWrite) {
		BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
		rc = writeback(ctxt, &ctxt->src);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
5752 5753 5754 5755 5756
	if (!(ctxt->d & NoWrite)) {
		rc = writeback(ctxt, &ctxt->dst);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
5757

5758 5759 5760 5761
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
5762
	ctxt->dst.type = saved_dst_type;
5763

5764
	if ((ctxt->d & SrcMask) == SrcSI)
5765
		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5766

5767
	if ((ctxt->d & DstMask) == DstDI)
5768
		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5769

5770
	if (ctxt->rep_prefix && (ctxt->d & String)) {
5771
		unsigned int count;
5772
		struct read_cache *r = &ctxt->io_read;
5773 5774 5775 5776
		if ((ctxt->d & SrcMask) == SrcSI)
			count = ctxt->src.count;
		else
			count = ctxt->dst.count;
5777
		register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5778

5779 5780 5781 5782 5783
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
5784
			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5785 5786 5787 5788 5789 5790
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
5791
				ctxt->mem_read.end = 0;
5792
				writeback_registers(ctxt);
5793 5794 5795
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
5796
		}
5797
		ctxt->eflags &= ~X86_EFLAGS_RF;
5798
	}
5799

5800
	ctxt->eip = ctxt->_eip;
5801 5802
	if (ctxt->mode != X86EMUL_MODE_PROT64)
		ctxt->eip = (u32)ctxt->_eip;
5803 5804

done:
5805 5806
	if (rc == X86EMUL_PROPAGATE_FAULT) {
		WARN_ON(ctxt->exception.vector > 0x1f);
5807
		ctxt->have_exception = true;
5808
	}
5809 5810 5811
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

5812 5813 5814
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);

5815
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
5816 5817

twobyte_insn:
5818
	switch (ctxt->b) {
5819
	case 0x09:		/* wbinvd */
5820
		(ctxt->ops->wbinvd)(ctxt);
5821 5822
		break;
	case 0x08:		/* invd */
5823 5824
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
P
Paolo Bonzini 已提交
5825
	case 0x1f:		/* nop */
5826 5827
		break;
	case 0x20: /* mov cr, reg */
5828
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5829
		break;
A
Avi Kivity 已提交
5830
	case 0x21: /* mov from dr to reg */
5831
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
A
Avi Kivity 已提交
5832 5833
		break;
	case 0x40 ... 0x4f:	/* cmov */
5834 5835
		if (test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.val = ctxt->src.val;
5836
		else if (ctxt->op_bytes != 4)
5837
			ctxt->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
5838
		break;
5839
	case 0x80 ... 0x8f: /* jnz rel, etc*/
5840
		if (test_cc(ctxt->b, ctxt->eflags))
5841
			rc = jmp_rel(ctxt, ctxt->src.val);
5842
		break;
5843
	case 0x90 ... 0x9f:     /* setcc r/m8 */
5844
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5845
		break;
A
Avi Kivity 已提交
5846
	case 0xb6 ... 0xb7:	/* movzx */
5847
		ctxt->dst.bytes = ctxt->op_bytes;
5848
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5849
						       : (u16) ctxt->src.val;
A
Avi Kivity 已提交
5850 5851
		break;
	case 0xbe ... 0xbf:	/* movsx */
5852
		ctxt->dst.bytes = ctxt->op_bytes;
5853
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5854
							(s16) ctxt->src.val;
A
Avi Kivity 已提交
5855
		break;
5856 5857
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
5858
	}
5859

5860 5861
threebyte_insn:

5862 5863 5864
	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
5865 5866 5867
	goto writeback;

cannot_emulate:
5868
	return EMULATION_FAILED;
A
Avi Kivity 已提交
5869
}
5870 5871 5872 5873 5874 5875 5876 5877 5878 5879

void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
{
	invalidate_registers(ctxt);
}

void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
{
	writeback_registers(ctxt);
}
5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890

bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->rep_prefix && (ctxt->d & String))
		return false;

	if (ctxt->d & TwoMemOp)
		return false;

	return true;
}