emulate.c 129.7 KB
Newer Older
A
Avi Kivity 已提交
1
/******************************************************************************
2
 * emulate.c
A
Avi Kivity 已提交
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
A
Avi Kivity 已提交
10 11
 *
 * Copyright (C) 2006 Qumranet
N
Nicolas Kaiser 已提交
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
A
Avi Kivity 已提交
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
27
#include <linux/stringify.h>
A
Avi Kivity 已提交
28

29
#include "x86.h"
30
#include "tss.h"
31

32 33 34
/*
 * Operand types
 */
35 36 37 38 39 40 41 42 43
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
44 45 46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
47
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
48 49 50 51 52 53 54
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
55 56 57 58 59 60
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
61
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
62
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
P
Paolo Bonzini 已提交
63
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
64 65
#define OpAccLo           29ull  /* Low part of extended acc (AX/AX/EAX/RAX) */
#define OpAccHi           30ull  /* High part of extended acc (-/DX/EDX/RDX) */
66 67

#define OpBits             5  /* Width of operand field */
68
#define OpMask             ((1ull << OpBits) - 1)
69

A
Avi Kivity 已提交
70 71 72 73 74 75 76 77 78 79
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
80
#define ByteOp      (1<<0)	/* 8-bit operands. */
A
Avi Kivity 已提交
81
/* Destination operand type. */
82 83 84 85 86 87 88 89 90
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
91
#define DstAccLo    (OpAccLo << DstShift)
92
#define DstMask     (OpMask << DstShift)
A
Avi Kivity 已提交
93
/* Source operand type. */
94 95 96 97 98 99 100 101 102 103 104 105
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
P
Paolo Bonzini 已提交
106
#define SrcXLat     (OpXLat << SrcShift)
107 108 109 110
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
111
#define SrcImm64    (OpImm64 << SrcShift)
112
#define SrcDX       (OpDX << SrcShift)
113
#define SrcMem8     (OpMem8 << SrcShift)
114
#define SrcAccHi    (OpAccHi << SrcShift)
115
#define SrcMask     (OpMask << SrcShift)
116 117 118 119 120 121 122 123 124
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
125
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
126
#define Sse         (1<<18)     /* SSE Vector instruction */
127 128 129 130
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
131
/* Misc flags */
132
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
133
#define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136
#define Undefined   (1<<25) /* No Such Instruction */
137
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
138
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
139
#define No64	    (1<<28)
140
#define PageTable   (1 << 29)   /* instruction used to write page table */
141
#define NotImpl     (1 << 30)   /* instruction is not implemented */
142
/* Source 2 operand type */
143
#define Src2Shift   (31)
144
#define Src2None    (OpNone << Src2Shift)
145
#define Src2Mem     (OpMem << Src2Shift)
146 147 148 149
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
150 151 152 153 154 155
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
156
#define Src2Mask    (OpMask << Src2Shift)
A
Avi Kivity 已提交
157
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
158 159 160
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
161
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
162
#define NoWrite     ((u64)1 << 45)  /* No writeback */
163
#define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
164
#define NoMod	    ((u64)1 << 47)  /* Mod field is ignored */
165 166
#define Intercept   ((u64)1 << 48)  /* Has valid intercept field */
#define CheckPerm   ((u64)1 << 49)  /* Has valid check_perm field */
167
#define NoBigReal   ((u64)1 << 50)  /* No big real mode */
168
#define PrivUD      ((u64)1 << 51)  /* #UD instead of #GP on CPL > 0 */
169
#define NearBranch  ((u64)1 << 52)  /* Near branches */
170
#define No16	    ((u64)1 << 53)  /* No 16 bit operand */
A
Avi Kivity 已提交
171

172
#define DstXacc     (DstAccLo | SrcAccHi | SrcWrite)
A
Avi Kivity 已提交
173

174 175 176 177 178 179 180 181
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
182

183 184 185 186 187 188
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

/*
 * fastop functions have a special calling convention:
 *
189 190
 * dst:    rax        (in/out)
 * src:    rdx        (in/out)
191 192
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
193
 * ex:     rsi        (in:fastop pointer, out:zero if exception)
194 195 196 197 198 199 200 201 202 203 204
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 *
 * fastop functions are declared as taking a never-defined fastop parameter,
 * so they can't be called from C directly.
 */

struct fastop;

205
struct opcode {
206 207
	u64 flags : 56;
	u64 intercept : 8;
208
	union {
209
		int (*execute)(struct x86_emulate_ctxt *ctxt);
210 211 212
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
213
		const struct escape *esc;
214
		void (*fastop)(struct fastop *fake);
215
	} u;
216
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
217 218 219 220 221
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
222 223
};

224 225 226 227 228 229 230
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

231 232 233 234 235
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

A
Avi Kivity 已提交
236
/* EFLAGS bit definitions. */
237 238 239 240
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
241 242
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
243 244
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
A
Avi Kivity 已提交
245 246
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
247
#define EFLG_IF (1<<9)
248
#define EFLG_TF (1<<8)
A
Avi Kivity 已提交
249 250 251 252 253 254
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

255 256 257
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

A
Avi Kivity 已提交
294 295 296 297 298 299
/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

300 301 302 303 304 305
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

306 307
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));

308 309 310 311 312 313 314 315 316 317 318 319 320
#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
#define FOP_RET   "ret \n\t"

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
            FOP_ALIGN \
	    "em_" #op ": \n\t"

#define FOP_END \
	    ".popsection")

321 322
#define FOPNOP() FOP_ALIGN FOP_RET

323
#define FOP1E(op,  dst) \
324 325 326 327
	FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET

#define FOP1EEX(op,  dst) \
	FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
328 329 330 331 332 333 334 335 336

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

337 338 339 340 341 342 343 344 345
/* 1-operand, using src2 (for MUL/DIV r/m) */
#define FASTOP1SRC2(op, name) \
	FOP_START(name) \
	FOP1E(op, cl) \
	FOP1E(op, cx) \
	FOP1E(op, ecx) \
	ON64(FOP1E(op, rcx)) \
	FOP_END

346 347 348 349 350 351 352 353 354
/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
#define FASTOP1SRC2EX(op, name) \
	FOP_START(name) \
	FOP1EEX(op, cl) \
	FOP1EEX(op, cx) \
	FOP1EEX(op, ecx) \
	ON64(FOP1EEX(op, rcx)) \
	FOP_END

355 356 357 358 359
#define FOP2E(op,  dst, src)	   \
	FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET

#define FASTOP2(op) \
	FOP_START(op) \
360 361 362 363
	FOP2E(op##b, al, dl) \
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
364 365
	FOP_END

366 367 368 369
/* 2 operand, word only */
#define FASTOP2W(op) \
	FOP_START(op) \
	FOPNOP() \
370 371 372
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
373 374
	FOP_END

375 376 377 378 379 380 381 382 383
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

384 385 386 387 388 389 390 391 392
/* 2 operand, src and dest are reversed */
#define FASTOP2R(op, name) \
	FOP_START(name) \
	FOP2E(op##b, dl, al) \
	FOP2E(op##w, dx, ax) \
	FOP2E(op##l, edx, eax) \
	ON64(FOP2E(op##q, rdx, rax)) \
	FOP_END

393 394 395 396 397 398 399
#define FOP3E(op,  dst, src, src2) \
	FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
400 401 402
	FOP3E(op##w, ax, dx, cl) \
	FOP3E(op##l, eax, edx, cl) \
	ON64(FOP3E(op##q, rax, rdx, cl)) \
403 404
	FOP_END

405 406 407
/* Special case for SETcc - 1 instruction per cc */
#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"

408 409 410
asm(".global kvm_fastop_exception \n"
    "kvm_fastop_exception: xor %esi, %esi; ret");

411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

P
Paolo Bonzini 已提交
430 431 432
FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
FOP_END;

433 434 435 436 437 438
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
439 440 441 442 443
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
444
		.dst_val    = ctxt->dst.val64,
445 446 447
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
448 449 450
		.next_rip   = ctxt->eip,
	};

451
	return ctxt->ops->intercept(ctxt, &info, stage);
452 453
}

A
Avi Kivity 已提交
454 455 456 457 458
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

459
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
460
{
461
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
462 463
}

A
Avi Kivity 已提交
464 465 466 467 468 469 470 471 472 473 474
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

A
Avi Kivity 已提交
475 476 477 478 479
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

A
Avi Kivity 已提交
480
/* Access/update address held in a register, based on addressing mode. */
481
static inline unsigned long
482
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
483
{
484
	if (ctxt->ad_bytes == sizeof(unsigned long))
485 486
		return reg;
	else
487
		return reg & ad_mask(ctxt);
488 489 490
}

static inline unsigned long
491
register_address(struct x86_emulate_ctxt *ctxt, int reg)
492
{
493
	return address_mask(ctxt, reg_read(ctxt, reg));
494 495
}

496 497 498 499 500
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

501
static inline void
502
register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
503
{
504 505
	ulong mask;

506
	if (ctxt->ad_bytes == sizeof(unsigned long))
507
		mask = ~0UL;
508
	else
509
		mask = ad_mask(ctxt);
510
	masked_increment(reg_rmw(ctxt, reg), mask, inc);
511 512 513 514
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
515
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
516
}
A
Avi Kivity 已提交
517

518 519 520 521 522 523 524
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

525
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
526 527 528 529
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

530
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
531 532
}

533 534
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
535
{
536
	WARN_ON(vec > 0x1f);
537 538 539
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
540
	return X86EMUL_PROPAGATE_FAULT;
541 542
}

543 544 545 546 547
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

548
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
549
{
550
	return emulate_exception(ctxt, GP_VECTOR, err, true);
551 552
}

553 554 555 556 557
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

558
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
559
{
560
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
561 562
}

563
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
564
{
565
	return emulate_exception(ctxt, TS_VECTOR, err, true);
566 567
}

568 569
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
570
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
571 572
}

A
Avi Kivity 已提交
573 574 575 576 577
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.
 */
static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
{
	if (likely(size < 16))
		return false;

	if (ctxt->d & Aligned)
		return true;
	else if (ctxt->d & Unaligned)
		return false;
	else if (ctxt->d & Avx)
		return false;
	else
		return true;
}

621 622 623 624
static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
				       struct segmented_address addr,
				       unsigned *max_size, unsigned size,
				       bool write, bool fetch,
625
				       enum x86emul_mode mode, ulong *linear)
626
{
627 628
	struct desc_struct desc;
	bool usable;
629
	ulong la;
630
	u32 lim;
631
	u16 sel;
632

633
	la = seg_base(ctxt, addr.seg) + addr.ea;
634
	*max_size = 0;
635
	switch (mode) {
636
	case X86EMUL_MODE_PROT64:
637
		if (is_noncanonical_address(la))
638
			goto bad;
639 640 641 642

		*max_size = min_t(u64, ~0u, (1ull << 48) - la);
		if (size > *max_size)
			goto bad;
643 644
		break;
	default:
645 646
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
647 648
		if (!usable)
			goto bad;
649 650 651
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
652 653
			goto bad;
		/* unreadable code segment */
654
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
655 656
			goto bad;
		lim = desc_limit_scaled(&desc);
657
		if (!(desc.type & 8) && (desc.type & 4)) {
G
Guo Chao 已提交
658
			/* expand-down segment */
659
			if (addr.ea <= lim)
660 661 662
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
		}
663 664 665
		if (addr.ea > lim)
			goto bad;
		*max_size = min_t(u64, ~0u, (u64)lim + 1 - addr.ea);
666 667
		if (size > *max_size)
			goto bad;
668
		la &= (u32)-1;
669 670
		break;
	}
671 672
	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
		return emulate_gp(ctxt, 0);
673 674
	*linear = la;
	return X86EMUL_CONTINUE;
675 676
bad:
	if (addr.seg == VCPU_SREG_SS)
677
		return emulate_ss(ctxt, 0);
678
	else
679
		return emulate_gp(ctxt, 0);
680 681
}

682 683 684 685 686
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
687
	unsigned max_size;
688 689
	return __linearize(ctxt, addr, &max_size, size, write, false,
			   ctxt->mode, linear);
690 691
}

692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736
static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
			     enum x86emul_mode mode)
{
	ulong linear;
	int rc;
	unsigned max_size;
	struct segmented_address addr = { .seg = VCPU_SREG_CS,
					   .ea = dst };

	if (ctxt->op_bytes != sizeof(unsigned long))
		addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
	rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
	if (rc == X86EMUL_CONTINUE)
		ctxt->_eip = addr.ea;
	return rc;
}

static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
{
	return assign_eip(ctxt, dst, ctxt->mode);
}

static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
			  const struct desc_struct *cs_desc)
{
	enum x86emul_mode mode = ctxt->mode;

#ifdef CONFIG_X86_64
	if (ctxt->mode >= X86EMUL_MODE_PROT32 && cs_desc->l) {
		u64 efer = 0;

		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
			mode = X86EMUL_MODE_PROT64;
	}
#endif
	if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
		mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
	return assign_eip(ctxt, dst, mode);
}

static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
{
	return assign_eip_near(ctxt, ctxt->_eip + rel);
}
737

738 739 740 741 742
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
743 744 745
	int rc;
	ulong linear;

746
	rc = linearize(ctxt, addr, size, false, &linear);
747 748
	if (rc != X86EMUL_CONTINUE)
		return rc;
749
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
750 751
}

752
/*
753
 * Prefetch the remaining bytes of the instruction without crossing page
754 755
 * boundary if they are not in fetch_cache yet.
 */
756
static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
757 758
{
	int rc;
759
	unsigned size, max_size;
760
	unsigned long linear;
761
	int cur_size = ctxt->fetch.end - ctxt->fetch.data;
762
	struct segmented_address addr = { .seg = VCPU_SREG_CS,
763 764
					   .ea = ctxt->eip + cur_size };

765 766 767 768 769 770 771 772 773 774
	/*
	 * We do not know exactly how many bytes will be needed, and
	 * __linearize is expensive, so fetch as much as possible.  We
	 * just have to avoid going beyond the 15 byte limit, the end
	 * of the segment, or the end of the page.
	 *
	 * __linearize is called with size 0 so that it does not do any
	 * boundary check itself.  Instead, we use max_size to check
	 * against op_size.
	 */
775 776
	rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
			 &linear);
777 778 779
	if (unlikely(rc != X86EMUL_CONTINUE))
		return rc;

780
	size = min_t(unsigned, 15UL ^ cur_size, max_size);
781
	size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
782 783 784 785 786 787 788 789

	/*
	 * One instruction can only straddle two pages,
	 * and one has been loaded at the beginning of
	 * x86_decode_insn.  So, if not enough bytes
	 * still, we must have hit the 15-byte boundary.
	 */
	if (unlikely(size < op_size))
790 791
		return emulate_gp(ctxt, 0);

792
	rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
793 794 795
			      size, &ctxt->exception);
	if (unlikely(rc != X86EMUL_CONTINUE))
		return rc;
796
	ctxt->fetch.end += size;
797
	return X86EMUL_CONTINUE;
798 799
}

800 801
static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
					       unsigned size)
802
{
803 804 805 806
	unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;

	if (unlikely(done_size < size))
		return __do_insn_fetch_bytes(ctxt, size - done_size);
807 808
	else
		return X86EMUL_CONTINUE;
809 810
}

811
/* Fetch next part of the instruction being emulated. */
812
#define insn_fetch(_type, _ctxt)					\
813 814 815
({	_type _x;							\
									\
	rc = do_insn_fetch_bytes(_ctxt, sizeof(_type));			\
816 817
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
818
	ctxt->_eip += sizeof(_type);					\
819 820
	_x = *(_type __aligned(1) *) ctxt->fetch.ptr;			\
	ctxt->fetch.ptr += sizeof(_type);				\
821
	_x;								\
822 823
})

824
#define insn_fetch_arr(_arr, _size, _ctxt)				\
825 826
({									\
	rc = do_insn_fetch_bytes(_ctxt, _size);				\
827 828
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
829
	ctxt->_eip += (_size);						\
830 831
	memcpy(_arr, ctxt->fetch.ptr, _size);				\
	ctxt->fetch.ptr += (_size);					\
832 833
})

834 835 836 837 838
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
839
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
840
			     int byteop)
A
Avi Kivity 已提交
841 842
{
	void *p;
843
	int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
A
Avi Kivity 已提交
844 845

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
846 847 848
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
A
Avi Kivity 已提交
849 850 851 852
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
853
			   struct segmented_address addr,
A
Avi Kivity 已提交
854 855 856 857 858 859 860
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
861
	rc = segmented_read_std(ctxt, addr, size, 2);
862
	if (rc != X86EMUL_CONTINUE)
A
Avi Kivity 已提交
863
		return rc;
864
	addr.ea += 2;
865
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
A
Avi Kivity 已提交
866 867 868
	return rc;
}

869 870 871 872 873 874 875 876 877 878
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);

879 880
FASTOP1SRC2(mul, mul_ex);
FASTOP1SRC2(imul, imul_ex);
881 882
FASTOP1SRC2EX(div, div_ex);
FASTOP1SRC2EX(idiv, idiv_ex);
883

884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908
FASTOP3WCL(shld);
FASTOP3WCL(shrd);

FASTOP2W(imul);

FASTOP1(not);
FASTOP1(neg);
FASTOP1(inc);
FASTOP1(dec);

FASTOP2CL(rol);
FASTOP2CL(ror);
FASTOP2CL(rcl);
FASTOP2CL(rcr);
FASTOP2CL(shl);
FASTOP2CL(shr);
FASTOP2CL(sar);

FASTOP2W(bsf);
FASTOP2W(bsr);
FASTOP2W(bt);
FASTOP2W(bts);
FASTOP2W(btr);
FASTOP2W(btc);

909 910
FASTOP2(xadd);

911 912
FASTOP2R(cmp, cmp_r);

913
static u8 test_cc(unsigned int condition, unsigned long flags)
914
{
915 916
	u8 rc;
	void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
917

918
	flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
919
	asm("push %[flags]; popf; call *%[fastop]"
920 921
	    : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
	return rc;
922 923
}

924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

A
Avi Kivity 已提交
942 943 944 945
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
946 947 948 949 950 951 952 953
	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
954
#ifdef CONFIG_X86_64
955 956 957 958 959 960 961 962
	case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
A
Avi Kivity 已提交
963 964 965 966 967 968 969 970 971 972 973
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
974 975 976 977 978 979 980 981
	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
A
Avi Kivity 已提交
982
#ifdef CONFIG_X86_64
983 984 985 986 987 988 989 990
	case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
A
Avi Kivity 已提交
991 992 993 994 995 996
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

A
Avi Kivity 已提交
997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
	case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
	case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
	case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
	case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
	case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
	case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
	case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
	case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
	case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
	case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
static int em_fninit(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fninit");
	ctxt->ops->put_fpu(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
{
	u16 fcw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstcw %0": "+m"(fcw));
	ctxt->ops->put_fpu(ctxt);

	/* force 2 byte destination */
	ctxt->dst.bytes = 2;
	ctxt->dst.val = fcw;

	return X86EMUL_CONTINUE;
}

static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
{
	u16 fsw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstsw %0": "+m"(fsw));
	ctxt->ops->put_fpu(ctxt);

	/* force 2 byte destination */
	ctxt->dst.bytes = 2;
	ctxt->dst.val = fsw;

	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1078
static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1079
				    struct operand *op)
1080
{
1081
	unsigned reg = ctxt->modrm_reg;
1082

1083 1084
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
A
Avi Kivity 已提交
1085

1086
	if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1087 1088 1089 1090 1091 1092
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}
A
Avi Kivity 已提交
1093 1094 1095 1096 1097 1098 1099
	if (ctxt->d & Mmx) {
		reg &= 7;
		op->type = OP_MM;
		op->bytes = 8;
		op->addr.mm = reg;
		return;
	}
A
Avi Kivity 已提交
1100

1101
	op->type = OP_REG;
1102 1103 1104
	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);

1105
	fetch_register_operand(op);
1106 1107 1108
	op->orig_val = op->val;
}

1109 1110 1111 1112 1113 1114
static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
{
	if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
		ctxt->modrm_seg = VCPU_SREG_SS;
}

1115
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1116
			struct operand *op)
1117 1118
{
	u8 sib;
B
Bandan Das 已提交
1119
	int index_reg, base_reg, scale;
1120
	int rc = X86EMUL_CONTINUE;
1121
	ulong modrm_ea = 0;
1122

B
Bandan Das 已提交
1123 1124 1125
	ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
	index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
	base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1126

B
Bandan Das 已提交
1127
	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1128
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
B
Bandan Das 已提交
1129
	ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1130
	ctxt->modrm_seg = VCPU_SREG_DS;
1131

1132
	if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1133
		op->type = OP_REG;
1134
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1135
		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1136
				ctxt->d & ByteOp);
1137
		if (ctxt->d & Sse) {
A
Avi Kivity 已提交
1138 1139
			op->type = OP_XMM;
			op->bytes = 16;
1140 1141
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
A
Avi Kivity 已提交
1142 1143
			return rc;
		}
A
Avi Kivity 已提交
1144 1145 1146
		if (ctxt->d & Mmx) {
			op->type = OP_MM;
			op->bytes = 8;
1147
			op->addr.mm = ctxt->modrm_rm & 7;
A
Avi Kivity 已提交
1148 1149
			return rc;
		}
1150
		fetch_register_operand(op);
1151 1152 1153
		return rc;
	}

1154 1155
	op->type = OP_MEM;

1156
	if (ctxt->ad_bytes == 2) {
1157 1158 1159 1160
		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1161 1162

		/* 16-bit ModR/M decode. */
1163
		switch (ctxt->modrm_mod) {
1164
		case 0:
1165
			if (ctxt->modrm_rm == 6)
1166
				modrm_ea += insn_fetch(u16, ctxt);
1167 1168
			break;
		case 1:
1169
			modrm_ea += insn_fetch(s8, ctxt);
1170 1171
			break;
		case 2:
1172
			modrm_ea += insn_fetch(u16, ctxt);
1173 1174
			break;
		}
1175
		switch (ctxt->modrm_rm) {
1176
		case 0:
1177
			modrm_ea += bx + si;
1178 1179
			break;
		case 1:
1180
			modrm_ea += bx + di;
1181 1182
			break;
		case 2:
1183
			modrm_ea += bp + si;
1184 1185
			break;
		case 3:
1186
			modrm_ea += bp + di;
1187 1188
			break;
		case 4:
1189
			modrm_ea += si;
1190 1191
			break;
		case 5:
1192
			modrm_ea += di;
1193 1194
			break;
		case 6:
1195
			if (ctxt->modrm_mod != 0)
1196
				modrm_ea += bp;
1197 1198
			break;
		case 7:
1199
			modrm_ea += bx;
1200 1201
			break;
		}
1202 1203 1204
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
1205
		modrm_ea = (u16)modrm_ea;
1206 1207
	} else {
		/* 32/64-bit ModR/M decode. */
1208
		if ((ctxt->modrm_rm & 7) == 4) {
1209
			sib = insn_fetch(u8, ctxt);
1210 1211 1212 1213
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

1214
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1215
				modrm_ea += insn_fetch(s32, ctxt);
1216
			else {
1217
				modrm_ea += reg_read(ctxt, base_reg);
1218 1219
				adjust_modrm_seg(ctxt, base_reg);
			}
1220
			if (index_reg != 4)
1221
				modrm_ea += reg_read(ctxt, index_reg) << scale;
1222
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1223
			modrm_ea += insn_fetch(s32, ctxt);
1224
			if (ctxt->mode == X86EMUL_MODE_PROT64)
1225
				ctxt->rip_relative = 1;
1226 1227
		} else {
			base_reg = ctxt->modrm_rm;
1228
			modrm_ea += reg_read(ctxt, base_reg);
1229 1230
			adjust_modrm_seg(ctxt, base_reg);
		}
1231
		switch (ctxt->modrm_mod) {
1232
		case 1:
1233
			modrm_ea += insn_fetch(s8, ctxt);
1234 1235
			break;
		case 2:
1236
			modrm_ea += insn_fetch(s32, ctxt);
1237 1238 1239
			break;
		}
	}
1240
	op->addr.mem.ea = modrm_ea;
1241 1242 1243
	if (ctxt->ad_bytes != 8)
		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;

1244 1245 1246 1247 1248
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
1249
		      struct operand *op)
1250
{
1251
	int rc = X86EMUL_CONTINUE;
1252

1253
	op->type = OP_MEM;
1254
	switch (ctxt->ad_bytes) {
1255
	case 2:
1256
		op->addr.mem.ea = insn_fetch(u16, ctxt);
1257 1258
		break;
	case 4:
1259
		op->addr.mem.ea = insn_fetch(u32, ctxt);
1260 1261
		break;
	case 8:
1262
		op->addr.mem.ea = insn_fetch(u64, ctxt);
1263 1264 1265 1266 1267 1268
		break;
	}
done:
	return rc;
}

1269
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1270
{
1271
	long sv = 0, mask;
1272

1273
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1274
		mask = ~((long)ctxt->dst.bytes * 8 - 1);
1275

1276 1277 1278 1279
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1280 1281
		else
			sv = (s64)ctxt->src.val & (s64)mask;
1282

1283 1284
		ctxt->dst.addr.mem.ea = address_mask(ctxt,
					   ctxt->dst.addr.mem.ea + (sv >> 3));
1285
	}
1286 1287

	/* only subword offset */
1288
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1289 1290
}

1291 1292
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
A
Avi Kivity 已提交
1293
{
1294
	int rc;
1295
	struct read_cache *mc = &ctxt->mem_read;
A
Avi Kivity 已提交
1296

1297 1298
	if (mc->pos < mc->end)
		goto read_cached;
A
Avi Kivity 已提交
1299

1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
	WARN_ON((mc->end + size) >= sizeof(mc->data));

	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
				      &ctxt->exception);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	mc->end += size;

read_cached:
	memcpy(dest, mc->data + mc->pos, size);
	mc->pos += size;
1312 1313
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1314

1315 1316 1317 1318 1319
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1320 1321 1322
	int rc;
	ulong linear;

1323
	rc = linearize(ctxt, addr, size, false, &linear);
1324 1325
	if (rc != X86EMUL_CONTINUE)
		return rc;
1326
	return read_emulated(ctxt, linear, data, size);
1327 1328 1329 1330 1331 1332 1333
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1334 1335 1336
	int rc;
	ulong linear;

1337
	rc = linearize(ctxt, addr, size, true, &linear);
1338 1339
	if (rc != X86EMUL_CONTINUE)
		return rc;
1340 1341
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1342 1343 1344 1345 1346 1347 1348
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1349 1350 1351
	int rc;
	ulong linear;

1352
	rc = linearize(ctxt, addr, size, true, &linear);
1353 1354
	if (rc != X86EMUL_CONTINUE)
		return rc;
1355 1356
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1357 1358
}

1359 1360 1361 1362
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1363
	struct read_cache *rc = &ctxt->io_read;
1364

1365 1366
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1367
		unsigned int count = ctxt->rep_prefix ?
1368
			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1369
		in_page = (ctxt->eflags & EFLG_DF) ?
1370 1371
			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1372
		n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1373 1374 1375
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1376
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1377 1378
			return 0;
		rc->end = n * size;
A
Avi Kivity 已提交
1379 1380
	}

1381 1382
	if (ctxt->rep_prefix && (ctxt->d & String) &&
	    !(ctxt->eflags & EFLG_DF)) {
1383 1384 1385 1386 1387 1388 1389 1390
		ctxt->dst.data = rc->data + rc->pos;
		ctxt->dst.type = OP_MEM_STR;
		ctxt->dst.count = (rc->end - rc->pos) / size;
		rc->pos = rc->end;
	} else {
		memcpy(dest, rc->data + rc->pos, size);
		rc->pos += size;
	}
1391 1392
	return 1;
}
A
Avi Kivity 已提交
1393

1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
				     u16 index, struct desc_struct *desc)
{
	struct desc_ptr dt;
	ulong addr;

	ctxt->ops->get_idt(ctxt, &dt);

	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, index << 3 | 0x2);

	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
}

1410 1411 1412
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1413
	const struct x86_emulate_ops *ops = ctxt->ops;
1414
	u32 base3 = 0;
1415

1416 1417
	if (selector & 1 << 2) {
		struct desc_struct desc;
1418 1419
		u16 sel;

1420
		memset (dt, 0, sizeof *dt);
1421 1422
		if (!ops->get_segment(ctxt, &sel, &desc, &base3,
				      VCPU_SREG_LDTR))
1423
			return;
1424

1425
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1426
		dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1427
	} else
1428
		ops->get_gdt(ctxt, dt);
1429
}
1430

1431 1432
/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1433 1434
				   u16 selector, struct desc_struct *desc,
				   ulong *desc_addr_p)
1435 1436 1437 1438
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1439

1440
	get_descriptor_table_ptr(ctxt, selector, &dt);
1441

1442 1443
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1444

1445
	*desc_addr_p = addr = dt.address + index * 8;
1446 1447
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
1448
}
1449

1450 1451 1452 1453 1454 1455 1456
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
A
Avi Kivity 已提交
1457

1458
	get_descriptor_table_ptr(ctxt, selector, &dt);
1459

1460 1461
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
A
Avi Kivity 已提交
1462

1463
	addr = dt.address + index * 8;
1464 1465
	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
				    &ctxt->exception);
1466
}
1467

1468
/* Does not support long mode */
1469
static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1470 1471 1472
				     u16 selector, int seg, u8 cpl,
				     bool in_task_switch,
				     struct desc_struct *desc)
1473
{
1474
	struct desc_struct seg_desc, old_desc;
1475
	u8 dpl, rpl;
1476 1477 1478
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1479
	ulong desc_addr;
1480
	int ret;
1481
	u16 dummy;
1482
	u32 base3 = 0;
1483

1484
	memset(&seg_desc, 0, sizeof seg_desc);
1485

1486 1487 1488
	if (ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor (keep limit etc. for
		 * unreal mode) */
1489
		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1490 1491
		set_desc_base(&seg_desc, selector << 4);
		goto load;
1492 1493 1494 1495 1496 1497 1498 1499 1500
	} else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
		/* VM86 needs a clean new segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
		seg_desc.dpl = 3;
		goto load;
1501 1502
	}

1503 1504 1505 1506 1507 1508 1509
	rpl = selector & 3;

	/* NULL selector is not valid for TR, CS and SS (except for long mode) */
	if ((seg == VCPU_SREG_CS
	     || (seg == VCPU_SREG_SS
		 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
	     || seg == VCPU_SREG_TR)
1510 1511 1512 1513 1514 1515 1516 1517 1518 1519
	    && null_selector)
		goto exception;

	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

	if (null_selector) /* for NULL selector skip all following checks */
		goto load;

1520
	ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1521 1522 1523 1524
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
1525
	err_vec = in_task_switch ? TS_VECTOR : GP_VECTOR;
1526

G
Guo Chao 已提交
1527
	/* can't load system descriptor into segment selector */
1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545
	if (seg <= VCPU_SREG_GS && !seg_desc.s)
		goto exception;

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	dpl = seg_desc.dpl;

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
A
Avi Kivity 已提交
1546
		break;
1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
1560 1561 1562 1563 1564 1565 1566 1567 1568
		/* in long-mode d/b must be clear if l is set */
		if (seg_desc.d && seg_desc.l) {
			u64 efer = 0;

			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
			if (efer & EFER_LMA)
				goto exception;
		}

1569 1570
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
A
Avi Kivity 已提交
1571
		break;
1572 1573 1574
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
1575 1576 1577 1578 1579 1580
		old_desc = seg_desc;
		seg_desc.type |= 2; /* busy */
		ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
						  sizeof(seg_desc), &ctxt->exception);
		if (ret != X86EMUL_CONTINUE)
			return ret;
1581 1582 1583 1584 1585 1586
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1587
		/*
1588 1589 1590
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1591
		 */
1592 1593 1594 1595
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
A
Avi Kivity 已提交
1596
		break;
1597 1598 1599 1600 1601
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
		seg_desc.type |= 1;
1602
		ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1603 1604
		if (ret != X86EMUL_CONTINUE)
			return ret;
1605 1606 1607 1608 1609
	} else if (ctxt->mode == X86EMUL_MODE_PROT64) {
		ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
				sizeof(base3), &ctxt->exception);
		if (ret != X86EMUL_CONTINUE)
			return ret;
1610 1611 1612
		if (is_noncanonical_address(get_desc_base(&seg_desc) |
					     ((u64)base3 << 32)))
			return emulate_gp(ctxt, 0);
1613 1614
	}
load:
1615
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1616 1617
	if (desc)
		*desc = seg_desc;
1618 1619
	return X86EMUL_CONTINUE;
exception:
1620
	return emulate_exception(ctxt, err_vec, err_code, true);
1621 1622
}

1623 1624 1625 1626
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
	u8 cpl = ctxt->ops->cpl(ctxt);
1627
	return __load_segment_descriptor(ctxt, selector, seg, cpl, false, NULL);
1628 1629
}

1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648
static void write_register_operand(struct operand *op)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (op->bytes) {
	case 1:
		*(u8 *)op->addr.reg = (u8)op->val;
		break;
	case 2:
		*(u16 *)op->addr.reg = (u16)op->val;
		break;
	case 4:
		*op->addr.reg = (u32)op->val;
		break;	/* 64b: zero-extend */
	case 8:
		*op->addr.reg = op->val;
		break;
	}
}

1649
static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1650
{
1651
	switch (op->type) {
1652
	case OP_REG:
1653
		write_register_operand(op);
A
Avi Kivity 已提交
1654
		break;
1655
	case OP_MEM:
1656
		if (ctxt->lock_prefix)
P
Paolo Bonzini 已提交
1657 1658 1659 1660 1661 1662 1663
			return segmented_cmpxchg(ctxt,
						 op->addr.mem,
						 &op->orig_val,
						 &op->val,
						 op->bytes);
		else
			return segmented_write(ctxt,
1664 1665 1666
					       op->addr.mem,
					       &op->val,
					       op->bytes);
1667
		break;
1668
	case OP_MEM_STR:
P
Paolo Bonzini 已提交
1669 1670 1671 1672
		return segmented_write(ctxt,
				       op->addr.mem,
				       op->data,
				       op->bytes * op->count);
1673
		break;
A
Avi Kivity 已提交
1674
	case OP_XMM:
1675
		write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
A
Avi Kivity 已提交
1676
		break;
A
Avi Kivity 已提交
1677
	case OP_MM:
1678
		write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
A
Avi Kivity 已提交
1679
		break;
1680 1681
	case OP_NONE:
		/* no writeback */
1682
		break;
1683
	default:
1684
		break;
A
Avi Kivity 已提交
1685
	}
1686 1687
	return X86EMUL_CONTINUE;
}
A
Avi Kivity 已提交
1688

1689
static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1690
{
1691
	struct segmented_address addr;
1692

1693
	rsp_increment(ctxt, -bytes);
1694
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1695 1696
	addr.seg = VCPU_SREG_SS;

1697 1698 1699 1700 1701
	return segmented_write(ctxt, addr, data, bytes);
}

static int em_push(struct x86_emulate_ctxt *ctxt)
{
1702
	/* Disable writeback. */
1703
	ctxt->dst.type = OP_NONE;
1704
	return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1705
}
1706

1707 1708 1709 1710
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1711
	struct segmented_address addr;
1712

1713
	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1714
	addr.seg = VCPU_SREG_SS;
1715
	rc = segmented_read(ctxt, addr, dest, len);
1716 1717 1718
	if (rc != X86EMUL_CONTINUE)
		return rc;

1719
	rsp_increment(ctxt, len);
1720
	return rc;
1721 1722
}

1723 1724
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1725
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1726 1727
}

1728
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1729
			void *dest, int len)
1730 1731
{
	int rc;
1732 1733
	unsigned long val, change_mask;
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1734
	int cpl = ctxt->ops->cpl(ctxt);
1735

1736
	rc = emulate_pop(ctxt, &val, len);
1737 1738
	if (rc != X86EMUL_CONTINUE)
		return rc;
1739

1740
	change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1741
		| EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1742

1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
			change_mask |= EFLG_IOPL;
		if (cpl <= iopl)
			change_mask |= EFLG_IF;
		break;
	case X86EMUL_MODE_VM86:
1753 1754
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1755 1756 1757 1758 1759
		change_mask |= EFLG_IF;
		break;
	default: /* real mode */
		change_mask |= (EFLG_IOPL | EFLG_IF);
		break;
1760
	}
1761 1762 1763 1764 1765

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1766 1767
}

1768 1769
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1770 1771 1772 1773
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1774 1775
}

A
Avi Kivity 已提交
1776 1777 1778 1779 1780
static int em_enter(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned frame_size = ctxt->src.val;
	unsigned nesting_level = ctxt->src2.val & 31;
1781
	ulong rbp;
A
Avi Kivity 已提交
1782 1783 1784 1785

	if (nesting_level)
		return X86EMUL_UNHANDLEABLE;

1786 1787
	rbp = reg_read(ctxt, VCPU_REGS_RBP);
	rc = push(ctxt, &rbp, stack_size(ctxt));
A
Avi Kivity 已提交
1788 1789
	if (rc != X86EMUL_CONTINUE)
		return rc;
1790
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
A
Avi Kivity 已提交
1791
		      stack_mask(ctxt));
1792 1793
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
A
Avi Kivity 已提交
1794 1795 1796 1797
		      stack_mask(ctxt));
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
1798 1799
static int em_leave(struct x86_emulate_ctxt *ctxt)
{
1800
	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
A
Avi Kivity 已提交
1801
		      stack_mask(ctxt));
1802
	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
A
Avi Kivity 已提交
1803 1804
}

1805
static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1806
{
1807 1808
	int seg = ctxt->src2.val;

1809
	ctxt->src.val = get_segment_selector(ctxt, seg);
1810 1811 1812 1813
	if (ctxt->op_bytes == 4) {
		rsp_increment(ctxt, -2);
		ctxt->op_bytes = 2;
	}
1814

1815
	return em_push(ctxt);
1816 1817
}

1818
static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1819
{
1820
	int seg = ctxt->src2.val;
1821 1822
	unsigned long selector;
	int rc;
1823

1824
	rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1825 1826 1827
	if (rc != X86EMUL_CONTINUE)
		return rc;

1828 1829 1830
	if (ctxt->modrm_reg == VCPU_SREG_SS)
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

1831
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1832
	return rc;
1833 1834
}

1835
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1836
{
1837
	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1838 1839
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1840

1841 1842
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
1843
		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1844

1845
		rc = em_push(ctxt);
1846 1847
		if (rc != X86EMUL_CONTINUE)
			return rc;
1848

1849
		++reg;
1850 1851
	}

1852
	return rc;
1853 1854
}

1855 1856
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
1857
	ctxt->src.val =  (unsigned long)ctxt->eflags;
1858 1859 1860
	return em_push(ctxt);
}

1861
static int em_popa(struct x86_emulate_ctxt *ctxt)
1862
{
1863 1864
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
1865

1866 1867
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
1868
			rsp_increment(ctxt, ctxt->op_bytes);
1869 1870
			--reg;
		}
1871

1872
		rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1873 1874 1875
		if (rc != X86EMUL_CONTINUE)
			break;
		--reg;
1876
	}
1877
	return rc;
1878 1879
}

1880
static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1881
{
1882
	const struct x86_emulate_ops *ops = ctxt->ops;
1883
	int rc;
1884 1885 1886 1887 1888 1889
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
1890
	ctxt->src.val = ctxt->eflags;
1891
	rc = em_push(ctxt);
1892 1893
	if (rc != X86EMUL_CONTINUE)
		return rc;
1894 1895 1896

	ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);

1897
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1898
	rc = em_push(ctxt);
1899 1900
	if (rc != X86EMUL_CONTINUE)
		return rc;
1901

1902
	ctxt->src.val = ctxt->_eip;
1903
	rc = em_push(ctxt);
1904 1905 1906
	if (rc != X86EMUL_CONTINUE)
		return rc;

1907
	ops->get_idt(ctxt, &dt);
1908 1909 1910 1911

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

1912
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1913 1914 1915
	if (rc != X86EMUL_CONTINUE)
		return rc;

1916
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1917 1918 1919
	if (rc != X86EMUL_CONTINUE)
		return rc;

1920
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1921 1922 1923
	if (rc != X86EMUL_CONTINUE)
		return rc;

1924
	ctxt->_eip = eip;
1925 1926 1927 1928

	return rc;
}

1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
{
	int rc;

	invalidate_registers(ctxt);
	rc = __emulate_int_real(ctxt, irq);
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);
	return rc;
}

1940
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1941 1942 1943
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1944
		return __emulate_int_real(ctxt, irq);
1945 1946 1947 1948 1949 1950 1951 1952 1953 1954
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

1955
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1956
{
1957 1958 1959 1960 1961 1962 1963 1964
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
	unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
			     EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
			     EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
	unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1965

1966
	/* TODO: Add stack limit check */
1967

1968
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1969

1970 1971
	if (rc != X86EMUL_CONTINUE)
		return rc;
1972

1973 1974
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
1975

1976
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1977

1978 1979
	if (rc != X86EMUL_CONTINUE)
		return rc;
1980

1981
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1982

1983 1984
	if (rc != X86EMUL_CONTINUE)
		return rc;
1985

1986
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1987

1988 1989
	if (rc != X86EMUL_CONTINUE)
		return rc;
1990

1991
	ctxt->_eip = temp_eip;
1992 1993


1994
	if (ctxt->op_bytes == 4)
1995
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1996
	else if (ctxt->op_bytes == 2) {
1997 1998
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
1999
	}
2000 2001 2002 2003 2004

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
	ctxt->eflags |= EFLG_RESERVED_ONE_MASK;

	return rc;
2005 2006
}

2007
static int em_iret(struct x86_emulate_ctxt *ctxt)
2008
{
2009 2010
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
2011
		return emulate_iret_real(ctxt);
2012 2013 2014 2015
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
2016
	default:
2017 2018
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
2019 2020 2021
	}
}

2022 2023 2024
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
2025 2026 2027 2028 2029 2030 2031 2032 2033
	unsigned short sel, old_sel;
	struct desc_struct old_desc, new_desc;
	const struct x86_emulate_ops *ops = ctxt->ops;
	u8 cpl = ctxt->ops->cpl(ctxt);

	/* Assignment of RIP may only fail in 64-bit mode */
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
				 VCPU_SREG_CS);
2034

2035
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2036

2037 2038
	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl, false,
				       &new_desc);
2039 2040 2041
	if (rc != X86EMUL_CONTINUE)
		return rc;

2042
	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2043
	if (rc != X86EMUL_CONTINUE) {
2044
		WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2045 2046 2047 2048 2049
		/* assigning eip failed; restore the old cs */
		ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
		return rc;
	}
	return rc;
2050 2051
}

2052
static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2053
{
2054 2055
	return assign_eip_near(ctxt, ctxt->src.val);
}
2056

2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067
static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	long int old_eip;

	old_eip = ctxt->_eip;
	rc = assign_eip_near(ctxt, ctxt->src.val);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->src.val = old_eip;
	rc = em_push(ctxt);
2068
	return rc;
2069 2070
}

2071
static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2072
{
2073
	u64 old = ctxt->dst.orig_val64;
2074

2075 2076 2077
	if (ctxt->dst.bytes == 16)
		return X86EMUL_UNHANDLEABLE;

2078 2079 2080 2081
	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2082
		ctxt->eflags &= ~EFLG_ZF;
2083
	} else {
2084 2085
		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
			(u32) reg_read(ctxt, VCPU_REGS_RBX);
2086

2087
		ctxt->eflags |= EFLG_ZF;
2088
	}
2089
	return X86EMUL_CONTINUE;
2090 2091
}

2092 2093
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
2094 2095 2096 2097 2098 2099 2100 2101
	int rc;
	unsigned long eip;

	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	return assign_eip_near(ctxt, eip);
2102 2103
}

2104
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2105 2106
{
	int rc;
2107 2108
	unsigned long eip, cs;
	u16 old_cs;
2109
	int cpl = ctxt->ops->cpl(ctxt);
2110 2111 2112 2113 2114 2115
	struct desc_struct old_desc, new_desc;
	const struct x86_emulate_ops *ops = ctxt->ops;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
				 VCPU_SREG_CS);
2116

2117
	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2118
	if (rc != X86EMUL_CONTINUE)
2119
		return rc;
2120
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2121
	if (rc != X86EMUL_CONTINUE)
2122
		return rc;
2123 2124 2125
	/* Outer-privilege level return is not implemented */
	if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
		return X86EMUL_UNHANDLEABLE;
2126 2127 2128 2129
	rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, 0, false,
				       &new_desc);
	if (rc != X86EMUL_CONTINUE)
		return rc;
2130
	rc = assign_eip_far(ctxt, eip, &new_desc);
2131
	if (rc != X86EMUL_CONTINUE) {
2132
		WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2133 2134
		ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
	}
2135 2136 2137
	return rc;
}

2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148
static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
{
        int rc;

        rc = em_ret_far(ctxt);
        if (rc != X86EMUL_CONTINUE)
                return rc;
        rsp_increment(ctxt, ctxt->src.val);
        return X86EMUL_CONTINUE;
}

2149 2150 2151
static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
{
	/* Save real source value, then compare EAX against destination. */
2152 2153
	ctxt->dst.orig_val = ctxt->dst.val;
	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2154
	ctxt->src.orig_val = ctxt->src.val;
2155
	ctxt->src.val = ctxt->dst.orig_val;
2156
	fastop(ctxt, em_cmp);
2157 2158 2159 2160 2161 2162 2163

	if (ctxt->eflags & EFLG_ZF) {
		/* Success: write back to memory. */
		ctxt->dst.val = ctxt->src.orig_val;
	} else {
		/* Failure: write the value we saw to EAX. */
		ctxt->dst.type = OP_REG;
2164
		ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2165
		ctxt->dst.val = ctxt->dst.orig_val;
2166 2167 2168 2169
	}
	return X86EMUL_CONTINUE;
}

2170
static int em_lseg(struct x86_emulate_ctxt *ctxt)
2171
{
2172
	int seg = ctxt->src2.val;
2173 2174 2175
	unsigned short sel;
	int rc;

2176
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2177

2178
	rc = load_segment_descriptor(ctxt, sel, seg);
2179 2180 2181
	if (rc != X86EMUL_CONTINUE)
		return rc;

2182
	ctxt->dst.val = ctxt->src.val;
2183 2184 2185
	return rc;
}

2186
static void
2187
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2188
			struct desc_struct *cs, struct desc_struct *ss)
2189 2190
{
	cs->l = 0;		/* will be adjusted later */
2191
	set_desc_base(cs, 0);	/* flat segment */
2192
	cs->g = 1;		/* 4kb granularity */
2193
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
2194 2195 2196
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
2197 2198
	cs->p = 1;
	cs->d = 1;
2199
	cs->avl = 0;
2200

2201 2202
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
2203 2204 2205
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
2206
	ss->d = 1;		/* 32bit stack segment */
2207
	ss->dpl = 0;
2208
	ss->p = 1;
2209 2210
	ss->l = 0;
	ss->avl = 0;
2211 2212
}

2213 2214 2215 2216 2217
static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

	eax = ecx = 0;
2218 2219
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2220 2221 2222 2223
		&& ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
		&& edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
}

2224 2225
static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
{
2226
	const struct x86_emulate_ops *ops = ctxt->ops;
2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237
	u32 eax, ebx, ecx, edx;

	/*
	 * syscall should always be enabled in longmode - so only become
	 * vendor specific (cpuid) if other modes are active...
	 */
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return true;

	eax = 0x00000000;
	ecx = 0x00000000;
2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262
	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	/*
	 * Intel ("GenuineIntel")
	 * remark: Intel CPUs only support "syscall" in 64bit
	 * longmode. Also an 64bit guest with a
	 * 32bit compat-app running will #UD !! While this
	 * behaviour can be fixed (by emulating) into AMD
	 * response - CPUs of AMD can't behave like Intel.
	 */
	if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
		return false;

	/* AMD ("AuthenticAMD") */
	if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
		return true;

	/* AMD ("AMDisbetter!") */
	if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
	    ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
	    edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
		return true;
2263 2264 2265 2266 2267

	/* default: (not Intel, not AMD), apply Intel's stricter rules... */
	return false;
}

2268
static int em_syscall(struct x86_emulate_ctxt *ctxt)
2269
{
2270
	const struct x86_emulate_ops *ops = ctxt->ops;
2271
	struct desc_struct cs, ss;
2272
	u64 msr_data;
2273
	u16 cs_sel, ss_sel;
2274
	u64 efer = 0;
2275 2276

	/* syscall is not available in real mode */
2277
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2278 2279
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
2280

2281 2282 2283
	if (!(em_syscall_is_enabled(ctxt)))
		return emulate_ud(ctxt);

2284
	ops->get_msr(ctxt, MSR_EFER, &efer);
2285
	setup_syscalls_segments(ctxt, &cs, &ss);
2286

2287 2288 2289
	if (!(efer & EFER_SCE))
		return emulate_ud(ctxt);

2290
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
2291
	msr_data >>= 32;
2292 2293
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
2294

2295
	if (efer & EFER_LMA) {
2296
		cs.d = 0;
2297 2298
		cs.l = 1;
	}
2299 2300
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2301

2302
	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2303
	if (efer & EFER_LMA) {
2304
#ifdef CONFIG_X86_64
2305
		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2306

2307
		ops->get_msr(ctxt,
2308 2309
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
2310
		ctxt->_eip = msr_data;
2311

2312
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2313
		ctxt->eflags &= ~msr_data;
2314
		ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2315 2316 2317
#endif
	} else {
		/* legacy mode */
2318
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
2319
		ctxt->_eip = (u32)msr_data;
2320

2321
		ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2322 2323
	}

2324
	return X86EMUL_CONTINUE;
2325 2326
}

2327
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2328
{
2329
	const struct x86_emulate_ops *ops = ctxt->ops;
2330
	struct desc_struct cs, ss;
2331
	u64 msr_data;
2332
	u16 cs_sel, ss_sel;
2333
	u64 efer = 0;
2334

2335
	ops->get_msr(ctxt, MSR_EFER, &efer);
2336
	/* inject #GP if in real mode */
2337 2338
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
2339

2340 2341 2342 2343 2344 2345 2346 2347
	/*
	 * Not recognized on AMD in compat mode (but is recognized in legacy
	 * mode).
	 */
	if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
	    && !vendor_intel(ctxt))
		return emulate_ud(ctxt);

2348
	/* sysenter/sysexit have not been tested in 64bit mode. */
2349
	if (ctxt->mode == X86EMUL_MODE_PROT64)
2350
		return X86EMUL_UNHANDLEABLE;
2351

2352
	setup_syscalls_segments(ctxt, &cs, &ss);
2353

2354
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2355 2356
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT32:
2357 2358
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2359 2360
		break;
	case X86EMUL_MODE_PROT64:
2361 2362
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2363
		break;
2364 2365
	default:
		break;
2366 2367
	}

2368
	ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2369 2370 2371 2372
	cs_sel = (u16)msr_data;
	cs_sel &= ~SELECTOR_RPL_MASK;
	ss_sel = cs_sel + 8;
	ss_sel &= ~SELECTOR_RPL_MASK;
2373
	if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2374
		cs.d = 0;
2375 2376 2377
		cs.l = 1;
	}

2378 2379
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2380

2381
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2382
	ctxt->_eip = msr_data;
2383

2384
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2385
	*reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2386

2387
	return X86EMUL_CONTINUE;
2388 2389
}

2390
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2391
{
2392
	const struct x86_emulate_ops *ops = ctxt->ops;
2393
	struct desc_struct cs, ss;
2394
	u64 msr_data, rcx, rdx;
2395
	int usermode;
X
Xiao Guangrong 已提交
2396
	u16 cs_sel = 0, ss_sel = 0;
2397

2398 2399
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
2400 2401
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
2402

2403
	setup_syscalls_segments(ctxt, &cs, &ss);
2404

2405
	if ((ctxt->rex_prefix & 0x8) != 0x0)
2406 2407 2408 2409
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

2410 2411 2412
	rcx = reg_read(ctxt, VCPU_REGS_RCX);
	rdx = reg_read(ctxt, VCPU_REGS_RDX);

2413 2414
	cs.dpl = 3;
	ss.dpl = 3;
2415
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2416 2417
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
2418
		cs_sel = (u16)(msr_data + 16);
2419 2420
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2421
		ss_sel = (u16)(msr_data + 24);
2422 2423
		rcx = (u32)rcx;
		rdx = (u32)rdx;
2424 2425
		break;
	case X86EMUL_MODE_PROT64:
2426
		cs_sel = (u16)(msr_data + 32);
2427 2428
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2429 2430
		ss_sel = cs_sel + 8;
		cs.d = 0;
2431
		cs.l = 1;
2432 2433 2434
		if (is_noncanonical_address(rcx) ||
		    is_noncanonical_address(rdx))
			return emulate_gp(ctxt, 0);
2435 2436
		break;
	}
2437 2438
	cs_sel |= SELECTOR_RPL_MASK;
	ss_sel |= SELECTOR_RPL_MASK;
2439

2440 2441
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2442

2443 2444
	ctxt->_eip = rdx;
	*reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2445

2446
	return X86EMUL_CONTINUE;
2447 2448
}

2449
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2450 2451 2452 2453 2454 2455 2456
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2457
	return ctxt->ops->cpl(ctxt) > iopl;
2458 2459 2460 2461 2462
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2463
	const struct x86_emulate_ops *ops = ctxt->ops;
2464
	struct desc_struct tr_seg;
2465
	u32 base3;
2466
	int r;
2467
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2468
	unsigned mask = (1 << len) - 1;
2469
	unsigned long base;
2470

2471
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2472
	if (!tr_seg.p)
2473
		return false;
2474
	if (desc_limit_scaled(&tr_seg) < 103)
2475
		return false;
2476 2477 2478 2479
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2480
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2481 2482
	if (r != X86EMUL_CONTINUE)
		return false;
2483
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2484
		return false;
2485
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2486 2487 2488 2489 2490 2491 2492 2493 2494 2495
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2496 2497 2498
	if (ctxt->perm_ok)
		return true;

2499 2500
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2501
			return false;
2502 2503 2504

	ctxt->perm_ok = true;

2505 2506 2507
	return true;
}

2508 2509 2510
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
2511
	tss->ip = ctxt->_eip;
2512
	tss->flag = ctxt->eflags;
2513 2514 2515 2516 2517 2518 2519 2520
	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->si = reg_read(ctxt, VCPU_REGS_RSI);
	tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2521

2522 2523 2524 2525 2526
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2527 2528 2529 2530 2531 2532
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;
2533
	u8 cpl;
2534

2535
	ctxt->_eip = tss->ip;
2536
	ctxt->eflags = tss->flag | 2;
2537 2538 2539 2540 2541 2542 2543 2544
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2545 2546 2547 2548 2549

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2550 2551 2552 2553 2554
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2555

2556 2557
	cpl = tss->cs & 3;

2558
	/*
G
Guo Chao 已提交
2559
	 * Now load segment descriptors. If fault happens at this stage
2560 2561
	 * it is handled in a context of new task
	 */
2562 2563
	ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
					true, NULL);
2564 2565
	if (ret != X86EMUL_CONTINUE)
		return ret;
2566 2567
	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
					true, NULL);
2568 2569
	if (ret != X86EMUL_CONTINUE)
		return ret;
2570 2571
	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
					true, NULL);
2572 2573
	if (ret != X86EMUL_CONTINUE)
		return ret;
2574 2575
	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
					true, NULL);
2576 2577
	if (ret != X86EMUL_CONTINUE)
		return ret;
2578 2579
	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
					true, NULL);
2580 2581 2582 2583 2584 2585 2586 2587 2588 2589
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2590
	const struct x86_emulate_ops *ops = ctxt->ops;
2591 2592
	struct tss_segment_16 tss_seg;
	int ret;
2593
	u32 new_tss_base = get_desc_base(new_desc);
2594

2595
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2596
			    &ctxt->exception);
2597
	if (ret != X86EMUL_CONTINUE)
2598 2599
		return ret;

2600
	save_state_to_tss16(ctxt, &tss_seg);
2601

2602
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2603
			     &ctxt->exception);
2604
	if (ret != X86EMUL_CONTINUE)
2605 2606
		return ret;

2607
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2608
			    &ctxt->exception);
2609
	if (ret != X86EMUL_CONTINUE)
2610 2611 2612 2613 2614
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2615
		ret = ops->write_std(ctxt, new_tss_base,
2616 2617
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2618
				     &ctxt->exception);
2619
		if (ret != X86EMUL_CONTINUE)
2620 2621 2622
			return ret;
	}

2623
	return load_state_from_tss16(ctxt, &tss_seg);
2624 2625 2626 2627 2628
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
2629
	/* CR3 and ldt selector are not saved intentionally */
2630
	tss->eip = ctxt->_eip;
2631
	tss->eflags = ctxt->eflags;
2632 2633 2634 2635 2636 2637 2638 2639
	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2640

2641 2642 2643 2644 2645 2646
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2647 2648 2649 2650 2651 2652
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;
2653
	u8 cpl;
2654

2655
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2656
		return emulate_gp(ctxt, 0);
2657
	ctxt->_eip = tss->eip;
2658
	ctxt->eflags = tss->eflags | 2;
2659 2660

	/* General purpose registers */
2661 2662 2663 2664 2665 2666 2667 2668
	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2669 2670 2671

	/*
	 * SDM says that segment selectors are loaded before segment
2672 2673
	 * descriptors.  This is important because CPL checks will
	 * use CS.RPL.
2674
	 */
2675 2676 2677 2678 2679 2680 2681
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2682

2683 2684 2685 2686 2687
	/*
	 * If we're switching between Protected Mode and VM86, we need to make
	 * sure to update the mode before loading the segment descriptors so
	 * that the selectors are interpreted correctly.
	 */
2688
	if (ctxt->eflags & X86_EFLAGS_VM) {
2689
		ctxt->mode = X86EMUL_MODE_VM86;
2690 2691
		cpl = 3;
	} else {
2692
		ctxt->mode = X86EMUL_MODE_PROT32;
2693 2694
		cpl = tss->cs & 3;
	}
2695

2696 2697 2698 2699
	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2700 2701
	ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
					cpl, true, NULL);
2702 2703
	if (ret != X86EMUL_CONTINUE)
		return ret;
2704 2705
	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
					true, NULL);
2706 2707
	if (ret != X86EMUL_CONTINUE)
		return ret;
2708 2709
	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
					true, NULL);
2710 2711
	if (ret != X86EMUL_CONTINUE)
		return ret;
2712 2713
	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
					true, NULL);
2714 2715
	if (ret != X86EMUL_CONTINUE)
		return ret;
2716 2717
	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
					true, NULL);
2718 2719
	if (ret != X86EMUL_CONTINUE)
		return ret;
2720 2721
	ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
					true, NULL);
2722 2723
	if (ret != X86EMUL_CONTINUE)
		return ret;
2724 2725
	ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
					true, NULL);
2726 2727 2728 2729 2730 2731 2732 2733 2734 2735
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2736
	const struct x86_emulate_ops *ops = ctxt->ops;
2737 2738
	struct tss_segment_32 tss_seg;
	int ret;
2739
	u32 new_tss_base = get_desc_base(new_desc);
2740 2741
	u32 eip_offset = offsetof(struct tss_segment_32, eip);
	u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2742

2743
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2744
			    &ctxt->exception);
2745
	if (ret != X86EMUL_CONTINUE)
2746 2747 2748
		/* FIXME: need to provide precise fault address */
		return ret;

2749
	save_state_to_tss32(ctxt, &tss_seg);
2750

2751 2752 2753
	/* Only GP registers and segment selectors are saved */
	ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
			     ldt_sel_offset - eip_offset, &ctxt->exception);
2754
	if (ret != X86EMUL_CONTINUE)
2755 2756 2757
		/* FIXME: need to provide precise fault address */
		return ret;

2758
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2759
			    &ctxt->exception);
2760
	if (ret != X86EMUL_CONTINUE)
2761 2762 2763 2764 2765 2766
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2767
		ret = ops->write_std(ctxt, new_tss_base,
2768 2769
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2770
				     &ctxt->exception);
2771
		if (ret != X86EMUL_CONTINUE)
2772 2773 2774 2775
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2776
	return load_state_from_tss32(ctxt, &tss_seg);
2777 2778 2779
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2780
				   u16 tss_selector, int idt_index, int reason,
2781
				   bool has_error_code, u32 error_code)
2782
{
2783
	const struct x86_emulate_ops *ops = ctxt->ops;
2784 2785
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
2786
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2787
	ulong old_tss_base =
2788
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2789
	u32 desc_limit;
2790
	ulong desc_addr;
2791 2792 2793

	/* FIXME: old_tss_base == ~0 ? */

2794
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2795 2796
	if (ret != X86EMUL_CONTINUE)
		return ret;
2797
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2798 2799 2800 2801 2802
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

2803 2804 2805 2806 2807
	/*
	 * Check privileges. The three cases are task switch caused by...
	 *
	 * 1. jmp/call/int to task gate: Check against DPL of the task gate
	 * 2. Exception/IRQ/iret: No check is performed
2808 2809
	 * 3. jmp/call to TSS/task-gate: No check is performed since the
	 *    hardware checks it before exiting.
2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825
	 */
	if (reason == TASK_SWITCH_GATE) {
		if (idt_index != -1) {
			/* Software interrupts */
			struct desc_struct task_gate_desc;
			int dpl;

			ret = read_interrupt_descriptor(ctxt, idt_index,
							&task_gate_desc);
			if (ret != X86EMUL_CONTINUE)
				return ret;

			dpl = task_gate_desc.dpl;
			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
				return emulate_gp(ctxt, (idt_index << 3) | 0x2);
		}
2826 2827
	}

2828 2829 2830 2831
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
2832
		return emulate_ts(ctxt, tss_selector & 0xfffc);
2833 2834 2835 2836
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2837
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2838 2839 2840 2841 2842 2843
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
G
Guo Chao 已提交
2844
	   note that old_tss_sel is not used after this point */
2845 2846 2847 2848
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
2849
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2850 2851
				     old_tss_base, &next_tss_desc);
	else
2852
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2853
				     old_tss_base, &next_tss_desc);
2854 2855
	if (ret != X86EMUL_CONTINUE)
		return ret;
2856 2857 2858 2859 2860 2861

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
2862
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2863 2864
	}

2865
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
2866
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2867

2868
	if (has_error_code) {
2869 2870 2871
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
2872
		ret = em_push(ctxt);
2873 2874
	}

2875 2876 2877 2878
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2879
			 u16 tss_selector, int idt_index, int reason,
2880
			 bool has_error_code, u32 error_code)
2881 2882 2883
{
	int rc;

2884
	invalidate_registers(ctxt);
2885 2886
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
2887

2888
	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2889
				     has_error_code, error_code);
2890

2891
	if (rc == X86EMUL_CONTINUE) {
2892
		ctxt->eip = ctxt->_eip;
2893 2894
		writeback_registers(ctxt);
	}
2895

2896
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2897 2898
}

2899 2900
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
		struct operand *op)
2901
{
2902
	int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2903

2904 2905
	register_address_increment(ctxt, reg, df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, reg);
2906 2907
}

2908 2909 2910 2911 2912 2913
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
2914
	al = ctxt->dst.val;
2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

2932
	ctxt->dst.val = al;
2933
	/* Set PF, ZF, SF */
2934 2935 2936
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
2937
	fastop(ctxt, em_or);
2938 2939 2940 2941 2942 2943 2944 2945
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967
static int em_aam(struct x86_emulate_ctxt *ctxt)
{
	u8 al, ah;

	if (ctxt->src.val == 0)
		return emulate_de(ctxt);

	al = ctxt->dst.val & 0xff;
	ah = al / ctxt->src.val;
	al %= ctxt->src.val;

	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);

	/* Set PF, ZF, SF */
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
	fastop(ctxt, em_or);

	return X86EMUL_CONTINUE;
}

2968 2969 2970 2971 2972 2973 2974 2975 2976
static int em_aad(struct x86_emulate_ctxt *ctxt)
{
	u8 al = ctxt->dst.val & 0xff;
	u8 ah = (ctxt->dst.val >> 8) & 0xff;

	al = (al + (ah * ctxt->src.val)) & 0xff;

	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;

2977 2978 2979 2980 2981
	/* Set PF, ZF, SF */
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
	fastop(ctxt, em_or);
2982 2983 2984 2985

	return X86EMUL_CONTINUE;
}

2986 2987
static int em_call(struct x86_emulate_ctxt *ctxt)
{
2988
	int rc;
2989 2990 2991
	long rel = ctxt->src.val;

	ctxt->src.val = (unsigned long)ctxt->_eip;
2992 2993 2994
	rc = jmp_rel(ctxt, rel);
	if (rc != X86EMUL_CONTINUE)
		return rc;
2995 2996 2997
	return em_push(ctxt);
}

2998 2999 3000 3001 3002
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;
3003 3004 3005
	struct desc_struct old_desc, new_desc;
	const struct x86_emulate_ops *ops = ctxt->ops;
	int cpl = ctxt->ops->cpl(ctxt);
3006

3007
	old_eip = ctxt->_eip;
3008
	ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3009

3010
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3011 3012 3013
	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl, false,
				       &new_desc);
	if (rc != X86EMUL_CONTINUE)
3014 3015
		return X86EMUL_CONTINUE;

3016
	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3017 3018
	if (rc != X86EMUL_CONTINUE)
		goto fail;
3019

3020
	ctxt->src.val = old_cs;
3021
	rc = em_push(ctxt);
3022
	if (rc != X86EMUL_CONTINUE)
3023
		goto fail;
3024

3025
	ctxt->src.val = old_eip;
3026 3027 3028 3029 3030 3031 3032 3033 3034 3035
	rc = em_push(ctxt);
	/* If we failed, we tainted the memory, but the very least we should
	   restore cs */
	if (rc != X86EMUL_CONTINUE)
		goto fail;
	return rc;
fail:
	ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
	return rc;

3036 3037
}

3038 3039 3040
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;
3041
	unsigned long eip;
3042

3043 3044 3045 3046
	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
	if (rc != X86EMUL_CONTINUE)
		return rc;
	rc = assign_eip_near(ctxt, eip);
3047 3048
	if (rc != X86EMUL_CONTINUE)
		return rc;
3049
	rsp_increment(ctxt, ctxt->src.val);
3050 3051 3052
	return X86EMUL_CONTINUE;
}

3053 3054 3055
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
3056 3057
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
3058 3059

	/* Write back the memory destination with implicit LOCK prefix. */
3060 3061
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
3062 3063 3064
	return X86EMUL_CONTINUE;
}

3065 3066
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
3067
	ctxt->dst.val = ctxt->src2.val;
3068
	return fastop(ctxt, em_imul);
3069 3070
}

3071 3072
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
3073 3074
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
3075
	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3076
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3077 3078 3079 3080

	return X86EMUL_CONTINUE;
}

3081 3082 3083 3084
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

3085
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3086 3087
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3088 3089 3090
	return X86EMUL_CONTINUE;
}

3091 3092 3093 3094
static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
{
	u64 pmc;

3095
	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3096
		return emulate_gp(ctxt, 0);
3097 3098
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3099 3100 3101
	return X86EMUL_CONTINUE;
}

3102 3103
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
3104
	memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3105 3106 3107
	return X86EMUL_CONTINUE;
}

B
Borislav Petkov 已提交
3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142
#define FFL(x) bit(X86_FEATURE_##x)

static int em_movbe(struct x86_emulate_ctxt *ctxt)
{
	u32 ebx, ecx, edx, eax = 1;
	u16 tmp;

	/*
	 * Check MOVBE is set in the guest-visible CPUID leaf.
	 */
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
	if (!(ecx & FFL(MOVBE)))
		return emulate_ud(ctxt);

	switch (ctxt->op_bytes) {
	case 2:
		/*
		 * From MOVBE definition: "...When the operand size is 16 bits,
		 * the upper word of the destination register remains unchanged
		 * ..."
		 *
		 * Both casting ->valptr and ->val to u16 breaks strict aliasing
		 * rules so we have to do the operation almost per hand.
		 */
		tmp = (u16)ctxt->src.val;
		ctxt->dst.val &= ~0xffffUL;
		ctxt->dst.val |= (unsigned long)swab16(tmp);
		break;
	case 4:
		ctxt->dst.val = swab32((u32)ctxt->src.val);
		break;
	case 8:
		ctxt->dst.val = swab64(ctxt->src.val);
		break;
	default:
3143
		BUG();
B
Borislav Petkov 已提交
3144 3145 3146 3147
	}
	return X86EMUL_CONTINUE;
}

3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175
static int em_cr_write(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

static int em_dr_write(struct x86_emulate_ctxt *ctxt)
{
	unsigned long val;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		val = ctxt->src.val & ~0ULL;
	else
		val = ctxt->src.val & ~0U;

	/* #UD condition is already handled. */
	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
		return emulate_gp(ctxt, 0);

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3176 3177 3178 3179
static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3180 3181 3182
	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
	if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3183 3184 3185 3186 3187 3188 3189 3190 3191
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
{
	u64 msr_data;

3192
	if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3193 3194
		return emulate_gp(ctxt, 0);

3195 3196
	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
	*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3197 3198 3199
	return X86EMUL_CONTINUE;
}

3200 3201
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
3202
	if (ctxt->modrm_reg > VCPU_SREG_GS)
3203 3204
		return emulate_ud(ctxt);

3205
	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3206 3207
	if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
		ctxt->dst.bytes = 2;
3208 3209 3210 3211 3212
	return X86EMUL_CONTINUE;
}

static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
3213
	u16 sel = ctxt->src.val;
3214

3215
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3216 3217
		return emulate_ud(ctxt);

3218
	if (ctxt->modrm_reg == VCPU_SREG_SS)
3219 3220 3221
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
3222 3223
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3224 3225
}

A
Avi Kivity 已提交
3226 3227 3228 3229 3230 3231 3232 3233 3234
static int em_lldt(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
}

A
Avi Kivity 已提交
3235 3236 3237 3238 3239 3240 3241 3242 3243
static int em_ltr(struct x86_emulate_ctxt *ctxt)
{
	u16 sel = ctxt->src.val;

	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
}

3244 3245
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
3246 3247 3248
	int rc;
	ulong linear;

3249
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3250
	if (rc == X86EMUL_CONTINUE)
3251
		ctxt->ops->invlpg(ctxt, linear);
3252
	/* Disable writeback. */
3253
	ctxt->dst.type = OP_NONE;
3254 3255 3256
	return X86EMUL_CONTINUE;
}

3257 3258 3259 3260 3261 3262 3263 3264 3265 3266
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

3267 3268
static int em_vmcall(struct x86_emulate_ctxt *ctxt)
{
3269
	int rc = ctxt->ops->fix_hypercall(ctxt);
3270 3271 3272 3273 3274

	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
3275
	ctxt->_eip = ctxt->eip;
3276
	/* Disable writeback. */
3277
	ctxt->dst.type = OP_NONE;
3278 3279 3280
	return X86EMUL_CONTINUE;
}

3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309
static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
				  void (*get)(struct x86_emulate_ctxt *ctxt,
					      struct desc_ptr *ptr))
{
	struct desc_ptr desc_ptr;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
	get(ctxt, &desc_ptr);
	if (ctxt->op_bytes == 2) {
		ctxt->op_bytes = 4;
		desc_ptr.address &= 0x00ffffff;
	}
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return segmented_write(ctxt, ctxt->dst.addr.mem,
			       &desc_ptr, 2 + ctxt->op_bytes);
}

static int em_sgdt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
}

static int em_sidt(struct x86_emulate_ctxt *ctxt)
{
	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
}

3310
static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3311 3312 3313 3314
{
	struct desc_ptr desc_ptr;
	int rc;

3315 3316
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		ctxt->op_bytes = 8;
3317
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3318
			     &desc_ptr.size, &desc_ptr.address,
3319
			     ctxt->op_bytes);
3320 3321
	if (rc != X86EMUL_CONTINUE)
		return rc;
3322 3323 3324
	if (ctxt->mode == X86EMUL_MODE_PROT64 &&
	    is_noncanonical_address(desc_ptr.address))
		return emulate_gp(ctxt, 0);
3325 3326 3327 3328
	if (lgdt)
		ctxt->ops->set_gdt(ctxt, &desc_ptr);
	else
		ctxt->ops->set_idt(ctxt, &desc_ptr);
3329
	/* Disable writeback. */
3330
	ctxt->dst.type = OP_NONE;
3331 3332 3333
	return X86EMUL_CONTINUE;
}

3334 3335 3336 3337 3338
static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	return em_lgdt_lidt(ctxt, true);
}

3339
static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3340 3341 3342
{
	int rc;

3343 3344
	rc = ctxt->ops->fix_hypercall(ctxt);

3345
	/* Disable writeback. */
3346
	ctxt->dst.type = OP_NONE;
3347 3348 3349 3350 3351
	return rc;
}

static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
3352
	return em_lgdt_lidt(ctxt, false);
3353 3354 3355 3356
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
3357 3358
	if (ctxt->dst.type == OP_MEM)
		ctxt->dst.bytes = 2;
3359
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3360 3361 3362 3363 3364 3365
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3366 3367
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
3368 3369 3370
	return X86EMUL_CONTINUE;
}

3371 3372
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
3373 3374
	int rc = X86EMUL_CONTINUE;

3375
	register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3376
	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3377
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3378
		rc = jmp_rel(ctxt, ctxt->src.val);
3379

3380
	return rc;
3381 3382 3383 3384
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
3385 3386
	int rc = X86EMUL_CONTINUE;

3387
	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3388
		rc = jmp_rel(ctxt, ctxt->src.val);
3389

3390
	return rc;
3391 3392
}

3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410
static int em_in(struct x86_emulate_ctxt *ctxt)
{
	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
			     &ctxt->dst.val))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
}

static int em_out(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				    &ctxt->src.val, 1);
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
	return X86EMUL_CONTINUE;
}

3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3430 3431 3432 3433
static int em_cpuid(struct x86_emulate_ctxt *ctxt)
{
	u32 eax, ebx, ecx, edx;

3434 3435
	eax = reg_read(ctxt, VCPU_REGS_RAX);
	ecx = reg_read(ctxt, VCPU_REGS_RCX);
A
Avi Kivity 已提交
3436
	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3437 3438 3439 3440
	*reg_write(ctxt, VCPU_REGS_RAX) = eax;
	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;
	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;
	*reg_write(ctxt, VCPU_REGS_RDX) = edx;
A
Avi Kivity 已提交
3441 3442 3443
	return X86EMUL_CONTINUE;
}

P
Paolo Bonzini 已提交
3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455
static int em_sahf(struct x86_emulate_ctxt *ctxt)
{
	u32 flags;

	flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
	flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;

	ctxt->eflags &= ~0xffUL;
	ctxt->eflags |= flags | X86_EFLAGS_FIXED;
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3456 3457
static int em_lahf(struct x86_emulate_ctxt *ctxt)
{
3458 3459
	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
A
Avi Kivity 已提交
3460 3461 3462
	return X86EMUL_CONTINUE;
}

A
Avi Kivity 已提交
3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477
static int em_bswap(struct x86_emulate_ctxt *ctxt)
{
	switch (ctxt->op_bytes) {
#ifdef CONFIG_X86_64
	case 8:
		asm("bswap %0" : "+r"(ctxt->dst.val));
		break;
#endif
	default:
		asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
		break;
	}
	return X86EMUL_CONTINUE;
}

3478 3479 3480 3481 3482 3483
static int em_clflush(struct x86_emulate_ctxt *ctxt)
{
	/* emulating clflush regardless of cpuid */
	return X86EMUL_CONTINUE;
}

3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
3498
	if (!valid_cr(ctxt->modrm_reg))
3499 3500 3501 3502 3503 3504 3505
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
3506 3507
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
3508
	u64 efer = 0;
3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
3526
		u64 cr4;
3527 3528 3529 3530
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

3531 3532
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3533 3534 3535 3536 3537 3538 3539 3540 3541 3542

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

3543 3544
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
N
Nadav Amit 已提交
3545
			rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3546 3547 3548 3549 3550 3551 3552

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
3553
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

3565 3566 3567 3568
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

3569
	ctxt->ops->get_dr(ctxt, 7, &dr7);
3570 3571 3572 3573 3574 3575 3576

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
3577
	int dr = ctxt->modrm_reg;
3578 3579 3580 3581 3582
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

3583
	cr4 = ctxt->ops->get_cr(ctxt, 4);
3584 3585 3586
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

3587 3588 3589 3590 3591 3592 3593
	if (check_dr7_gd(ctxt)) {
		ulong dr6;

		ctxt->ops->get_dr(ctxt, 6, &dr6);
		dr6 &= ~15;
		dr6 |= DR6_BD | DR6_RTM;
		ctxt->ops->set_dr(ctxt, 6, dr6);
3594
		return emulate_db(ctxt);
3595
	}
3596 3597 3598 3599 3600 3601

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
3602 3603
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
3604 3605 3606 3607 3608 3609 3610

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

3611 3612 3613 3614
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
	u64 efer;

3615
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3616 3617 3618 3619 3620 3621 3622 3623 3624

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
3625
	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3626 3627

	/* Valid physical address? */
3628
	if (rax & 0xffff000000000000ULL)
3629 3630 3631 3632 3633
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

3634 3635
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
3636
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3637

3638
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3639 3640 3641 3642 3643
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

3644 3645
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
3646
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3647
	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3648

3649
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3650
	    ctxt->ops->check_pmc(ctxt, rcx))
3651 3652 3653 3654 3655
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3656 3657
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
3658 3659
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3660 3661 3662 3663 3664 3665 3666
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
3667 3668
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3669 3670 3671 3672 3673
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

3674
#define D(_y) { .flags = (_y) }
3675 3676 3677
#define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
#define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
		      .intercept = x86_intercept_##_i, .check_perm = (_p) }
3678
#define N    D(NotImpl)
3679
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3680 3681
#define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
#define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3682
#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3683
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3684
#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3685
#define II(_f, _e, _i) \
3686
	{ .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3687
#define IIP(_f, _e, _i, _p) \
3688 3689
	{ .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
	  .intercept = x86_intercept_##_i, .check_perm = (_p) }
3690
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3691

3692
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
3693
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3694
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
3695
#define F2bv(_f, _e)  F((_f) | ByteOp, _e), F(_f, _e)
3696 3697
#define I2bvIP(_f, _e, _i, _p) \
	IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3698

3699 3700 3701
#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3702

3703 3704 3705 3706 3707 3708
static const struct opcode group7_rm0[] = {
	N,
	I(SrcNone | Priv | EmulateOnUD,	em_vmcall),
	N, N, N, N, N, N,
};

3709
static const struct opcode group7_rm1[] = {
3710 3711
	DI(SrcNone | Priv, monitor),
	DI(SrcNone | Priv, mwait),
3712 3713 3714
	N, N, N, N, N, N,
};

3715
static const struct opcode group7_rm3[] = {
3716
	DIP(SrcNone | Prot | Priv,		vmrun,		check_svme_pa),
3717
	II(SrcNone  | Prot | EmulateOnUD,	em_vmmcall,	vmmcall),
3718 3719 3720 3721 3722 3723
	DIP(SrcNone | Prot | Priv,		vmload,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		vmsave,		check_svme_pa),
	DIP(SrcNone | Prot | Priv,		stgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		clgi,		check_svme),
	DIP(SrcNone | Prot | Priv,		skinit,		check_svme),
	DIP(SrcNone | Prot | Priv,		invlpga,	check_svme),
3724
};
3725

3726
static const struct opcode group7_rm7[] = {
3727
	N,
3728
	DIP(SrcNone, rdtscp, check_rdtsc),
3729 3730
	N, N, N, N, N, N,
};
3731

3732
static const struct opcode group1[] = {
3733 3734 3735 3736 3737 3738 3739 3740
	F(Lock, em_add),
	F(Lock | PageTable, em_or),
	F(Lock, em_adc),
	F(Lock, em_sbb),
	F(Lock | PageTable, em_and),
	F(Lock, em_sub),
	F(Lock, em_xor),
	F(NoWrite, em_cmp),
3741 3742
};

3743
static const struct opcode group1A[] = {
3744
	I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3745 3746
};

3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757
static const struct opcode group2[] = {
	F(DstMem | ModRM, em_rol),
	F(DstMem | ModRM, em_ror),
	F(DstMem | ModRM, em_rcl),
	F(DstMem | ModRM, em_rcr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_shr),
	F(DstMem | ModRM, em_shl),
	F(DstMem | ModRM, em_sar),
};

3758
static const struct opcode group3[] = {
3759 3760
	F(DstMem | SrcImm | NoWrite, em_test),
	F(DstMem | SrcImm | NoWrite, em_test),
3761 3762
	F(DstMem | SrcNone | Lock, em_not),
	F(DstMem | SrcNone | Lock, em_neg),
3763 3764
	F(DstXacc | Src2Mem, em_mul_ex),
	F(DstXacc | Src2Mem, em_imul_ex),
3765 3766
	F(DstXacc | Src2Mem, em_div_ex),
	F(DstXacc | Src2Mem, em_idiv_ex),
3767 3768
};

3769
static const struct opcode group4[] = {
3770 3771
	F(ByteOp | DstMem | SrcNone | Lock, em_inc),
	F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3772 3773 3774
	N, N, N, N, N, N,
};

3775
static const struct opcode group5[] = {
3776 3777
	F(DstMem | SrcNone | Lock,		em_inc),
	F(DstMem | SrcNone | Lock,		em_dec),
3778
	I(SrcMem | NearBranch,			em_call_near_abs),
3779
	I(SrcMemFAddr | ImplicitOps | Stack,	em_call_far),
3780
	I(SrcMem | NearBranch,			em_jmp_abs),
3781 3782
	I(SrcMemFAddr | ImplicitOps,		em_jmp_far),
	I(SrcMem | Stack,			em_push), D(Undefined),
3783 3784
};

3785
static const struct opcode group6[] = {
3786 3787
	DI(Prot,	sldt),
	DI(Prot,	str),
A
Avi Kivity 已提交
3788
	II(Prot | Priv | SrcMem16, em_lldt, lldt),
A
Avi Kivity 已提交
3789
	II(Prot | Priv | SrcMem16, em_ltr, ltr),
3790 3791 3792
	N, N, N, N,
};

3793
static const struct group_dual group7 = { {
3794 3795
	II(Mov | DstMem,			em_sgdt, sgdt),
	II(Mov | DstMem,			em_sidt, sidt),
3796 3797 3798 3799 3800
	II(SrcMem | Priv,			em_lgdt, lgdt),
	II(SrcMem | Priv,			em_lidt, lidt),
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	II(SrcMem | ByteOp | Priv | NoAccess,	em_invlpg, invlpg),
3801
}, {
3802
	EXT(0, group7_rm0),
3803
	EXT(0, group7_rm1),
3804
	N, EXT(0, group7_rm3),
3805 3806 3807
	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
	EXT(0, group7_rm7),
3808 3809
} };

3810
static const struct opcode group8[] = {
3811
	N, N, N, N,
3812 3813 3814 3815
	F(DstMem | SrcImmByte | NoWrite,		em_bt),
	F(DstMem | SrcImmByte | Lock | PageTable,	em_bts),
	F(DstMem | SrcImmByte | Lock,			em_btr),
	F(DstMem | SrcImmByte | Lock | PageTable,	em_btc),
3816 3817
};

3818
static const struct group_dual group9 = { {
3819
	N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3820 3821 3822 3823
}, {
	N, N, N, N, N, N, N, N,
} };

3824
static const struct opcode group11[] = {
3825
	I(DstMem | SrcImm | Mov | PageTable, em_mov),
3826
	X7(D(Undefined)),
3827 3828
};

3829
static const struct gprefix pfx_0f_ae_7 = {
3830
	I(SrcMem | ByteOp, em_clflush), N, N, N,
3831 3832 3833 3834 3835 3836 3837 3838
};

static const struct group_dual group15 = { {
	N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
}, {
	N, N, N, N, N, N, N, N,
} };

3839
static const struct gprefix pfx_0f_6f_0f_7f = {
3840
	I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3841 3842
};

3843 3844
static const struct gprefix pfx_0f_2b = {
	I(0, em_mov), I(0, em_mov), N, N,
3845 3846
};

3847
static const struct gprefix pfx_0f_28_0f_29 = {
3848
	I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3849 3850
};

3851 3852 3853 3854
static const struct gprefix pfx_0f_e7 = {
	N, I(Sse, em_mov), N, N,
};

3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917
static const struct escape escape_d9 = { {
	N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_db = { {
	N, N, N, N, N, N, N, N,
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

static const struct escape escape_dd = { {
	N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
}, {
	/* 0xC0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xC8 - 0xCF */
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xC7 */
	N, N, N, N, N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
	N, N, N, N, N, N, N, N,
	/* 0xE8 - 0xEF */
	N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xF7 */
	N, N, N, N, N, N, N, N,
	/* 0xF8 - 0xFF */
	N, N, N, N, N, N, N, N,
} };

3918
static const struct opcode opcode_table[256] = {
3919
	/* 0x00 - 0x07 */
3920
	F6ALU(Lock, em_add),
3921 3922
	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3923
	/* 0x08 - 0x0F */
3924
	F6ALU(Lock | PageTable, em_or),
3925 3926
	I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
	N,
3927
	/* 0x10 - 0x17 */
3928
	F6ALU(Lock, em_adc),
3929 3930
	I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3931
	/* 0x18 - 0x1F */
3932
	F6ALU(Lock, em_sbb),
3933 3934
	I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3935
	/* 0x20 - 0x27 */
3936
	F6ALU(Lock | PageTable, em_and), N, N,
3937
	/* 0x28 - 0x2F */
3938
	F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3939
	/* 0x30 - 0x37 */
3940
	F6ALU(Lock, em_xor), N, N,
3941
	/* 0x38 - 0x3F */
3942
	F6ALU(NoWrite, em_cmp), N, N,
3943
	/* 0x40 - 0x4F */
3944
	X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3945
	/* 0x50 - 0x57 */
3946
	X8(I(SrcReg | Stack, em_push)),
3947
	/* 0x58 - 0x5F */
3948
	X8(I(DstReg | Stack, em_pop)),
3949
	/* 0x60 - 0x67 */
3950 3951
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
3952 3953 3954
	N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
	N, N, N, N,
	/* 0x68 - 0x6F */
3955 3956
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3957 3958
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3959
	I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3960
	I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3961
	/* 0x70 - 0x7F */
3962
	X16(D(SrcImmByte | NearBranch)),
3963
	/* 0x80 - 0x87 */
3964 3965 3966 3967
	G(ByteOp | DstMem | SrcImm, group1),
	G(DstMem | SrcImm, group1),
	G(ByteOp | DstMem | SrcImm | No64, group1),
	G(DstMem | SrcImmByte, group1),
3968
	F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3969
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3970
	/* 0x88 - 0x8F */
3971
	I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3972
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3973
	I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3974 3975 3976
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
3977
	/* 0x90 - 0x97 */
3978
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3979
	/* 0x98 - 0x9F */
3980
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3981
	I(SrcImmFAddr | No64, em_call_far), N,
3982
	II(ImplicitOps | Stack, em_pushf, pushf),
P
Paolo Bonzini 已提交
3983 3984
	II(ImplicitOps | Stack, em_popf, popf),
	I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3985
	/* 0xA0 - 0xA7 */
3986
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3987
	I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3988
	I2bv(SrcSI | DstDI | Mov | String, em_mov),
3989
	F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
3990
	/* 0xA8 - 0xAF */
3991
	F2bv(DstAcc | SrcImm | NoWrite, em_test),
3992 3993
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3994
	F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
3995
	/* 0xB0 - 0xB7 */
3996
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3997
	/* 0xB8 - 0xBF */
3998
	X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3999
	/* 0xC0 - 0xC7 */
4000
	G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4001 4002
	I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
	I(ImplicitOps | NearBranch, em_ret),
4003 4004
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4005
	G(ByteOp, group11), G(0, group11),
4006
	/* 0xC8 - 0xCF */
A
Avi Kivity 已提交
4007
	I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4008 4009
	I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
	I(ImplicitOps | Stack, em_ret_far),
4010
	D(ImplicitOps), DI(SrcImmByte, intn),
4011
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4012
	/* 0xD0 - 0xD7 */
4013 4014
	G(Src2One | ByteOp, group2), G(Src2One, group2),
	G(Src2CL | ByteOp, group2), G(Src2CL, group2),
P
Paolo Bonzini 已提交
4015
	I(DstAcc | SrcImmUByte | No64, em_aam),
P
Paolo Bonzini 已提交
4016 4017
	I(DstAcc | SrcImmUByte | No64, em_aad),
	F(DstAcc | ByteOp | No64, em_salc),
P
Paolo Bonzini 已提交
4018
	I(DstAcc | SrcXLat | ByteOp, em_mov),
4019
	/* 0xD8 - 0xDF */
4020
	N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4021
	/* 0xE0 - 0xE7 */
4022 4023
	X3(I(SrcImmByte | NearBranch, em_loop)),
	I(SrcImmByte | NearBranch, em_jcxz),
4024 4025
	I2bvIP(SrcImmUByte | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4026
	/* 0xE8 - 0xEF */
4027 4028 4029
	I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
	I(SrcImmFAddr | No64, em_jmp_far),
	D(SrcImmByte | ImplicitOps | NearBranch),
4030 4031
	I2bvIP(SrcDX | DstAcc, em_in,  in,  check_perm_in),
	I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4032
	/* 0xF0 - 0xF7 */
4033
	N, DI(ImplicitOps, icebp), N, N,
4034 4035
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
4036
	/* 0xF8 - 0xFF */
4037 4038
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4039 4040 4041
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

4042
static const struct opcode twobyte_table[256] = {
4043
	/* 0x00 - 0x0F */
4044
	G(0, group6), GD(0, &group7), N, N,
4045
	N, I(ImplicitOps | EmulateOnUD, em_syscall),
4046
	II(ImplicitOps | Priv, em_clts, clts), N,
4047
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4048
	N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4049
	/* 0x10 - 0x1F */
P
Paolo Bonzini 已提交
4050
	N, N, N, N, N, N, N, N,
4051 4052
	D(ImplicitOps | ModRM | SrcMem | NoAccess),
	N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4053
	/* 0x20 - 0x2F */
4054 4055 4056 4057 4058 4059
	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
						check_cr_write),
	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
						check_dr_write),
4060
	N, N, N, N,
4061 4062
	GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
	GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4063
	N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4064
	N, N, N, N,
4065
	/* 0x30 - 0x3F */
4066
	II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4067
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4068
	II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4069
	IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4070 4071
	I(ImplicitOps | EmulateOnUD, em_sysenter),
	I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4072
	N, N,
4073 4074
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
4075
	X16(D(DstReg | SrcMem | ModRM)),
4076 4077 4078
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
4079 4080 4081 4082
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4083
	/* 0x70 - 0x7F */
4084 4085 4086 4087
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4088
	/* 0x80 - 0x8F */
4089
	X16(D(SrcImm | NearBranch)),
4090
	/* 0x90 - 0x9F */
4091
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4092
	/* 0xA0 - 0xA7 */
4093
	I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4094 4095
	II(ImplicitOps, em_cpuid, cpuid),
	F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4096 4097
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4098
	/* 0xA8 - 0xAF */
4099
	I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4100
	DI(ImplicitOps, rsm),
4101
	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4102 4103
	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
	F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4104
	GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4105
	/* 0xB0 - 0xB7 */
4106
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4107
	I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4108
	F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4109 4110
	I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4111
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4112 4113
	/* 0xB8 - 0xBF */
	N, N,
4114
	G(BitOp, group8),
4115 4116
	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
	F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4117
	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
A
Avi Kivity 已提交
4118
	/* 0xC0 - 0xC7 */
4119
	F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4120
	N, I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov),
4121
	N, N, N, GD(0, &group9),
A
Avi Kivity 已提交
4122 4123
	/* 0xC8 - 0xCF */
	X8(I(DstReg, em_bswap)),
4124 4125 4126
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
4127 4128
	N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
	N, N, N, N, N, N, N, N,
4129 4130 4131 4132
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

4133
static const struct gprefix three_byte_0f_38_f0 = {
B
Borislav Petkov 已提交
4134
	I(DstReg | SrcMem | Mov, em_movbe), N, N, N
4135 4136 4137
};

static const struct gprefix three_byte_0f_38_f1 = {
B
Borislav Petkov 已提交
4138
	I(DstMem | SrcReg | Mov, em_movbe), N, N, N
4139 4140 4141 4142 4143 4144 4145 4146 4147
};

/*
 * Insns below are selected by the prefix which indexed by the third opcode
 * byte.
 */
static const struct opcode opcode_map_0f_38[256] = {
	/* 0x00 - 0x7f */
	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
B
Borislav Petkov 已提交
4148 4149 4150 4151 4152 4153 4154
	/* 0x80 - 0xef */
	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
	/* 0xf0 - 0xf1 */
	GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
	GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
	/* 0xf2 - 0xff */
	N, N, X4(N), X8(N)
4155 4156
};

4157 4158 4159 4160 4161
#undef D
#undef N
#undef G
#undef GD
#undef I
4162
#undef GP
4163
#undef EXT
4164

4165
#undef D2bv
4166
#undef D2bvIP
4167
#undef I2bv
4168
#undef I2bvIP
4169
#undef I6ALU
4170

4171
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4172 4173 4174
{
	unsigned size;

4175
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
4188
	op->addr.mem.ea = ctxt->_eip;
4189 4190 4191
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
4192
		op->val = insn_fetch(s8, ctxt);
4193 4194
		break;
	case 2:
4195
		op->val = insn_fetch(s16, ctxt);
4196 4197
		break;
	case 4:
4198
		op->val = insn_fetch(s32, ctxt);
4199
		break;
4200 4201 4202
	case 8:
		op->val = insn_fetch(s64, ctxt);
		break;
4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

4221 4222 4223 4224 4225 4226 4227
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
			  unsigned d)
{
	int rc = X86EMUL_CONTINUE;

	switch (d) {
	case OpReg:
4228
		decode_register_operand(ctxt, op);
4229 4230
		break;
	case OpImmUByte:
4231
		rc = decode_imm(ctxt, op, 1, false);
4232 4233
		break;
	case OpMem:
4234
		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4235 4236 4237
	mem_common:
		*op = ctxt->memop;
		ctxt->memopp = op;
4238
		if (ctxt->d & BitOp)
4239 4240 4241
			fetch_bit_operand(ctxt);
		op->orig_val = op->val;
		break;
4242
	case OpMem64:
4243
		ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4244
		goto mem_common;
4245 4246 4247
	case OpAcc:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4248
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4249 4250 4251
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269
	case OpAccLo:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
	case OpAccHi:
		if (ctxt->d & ByteOp) {
			op->type = OP_NONE;
			break;
		}
		op->type = OP_REG;
		op->bytes = ctxt->op_bytes;
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
4270 4271 4272 4273
	case OpDI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4274
			register_address(ctxt, VCPU_REGS_RDI);
4275 4276
		op->addr.mem.seg = VCPU_SREG_ES;
		op->val = 0;
4277
		op->count = 1;
4278 4279 4280 4281
		break;
	case OpDX:
		op->type = OP_REG;
		op->bytes = 2;
4282
		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4283 4284
		fetch_register_operand(op);
		break;
4285 4286
	case OpCL:
		op->bytes = 1;
4287
		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298
		break;
	case OpImmByte:
		rc = decode_imm(ctxt, op, 1, true);
		break;
	case OpOne:
		op->bytes = 1;
		op->val = 1;
		break;
	case OpImm:
		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
		break;
4299 4300 4301
	case OpImm64:
		rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
		break;
4302 4303
	case OpMem8:
		ctxt->memop.bytes = 1;
4304
		if (ctxt->memop.type == OP_REG) {
4305 4306
			ctxt->memop.addr.reg = decode_register(ctxt,
					ctxt->modrm_rm, true);
4307 4308
			fetch_register_operand(&ctxt->memop);
		}
4309
		goto mem_common;
4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325
	case OpMem16:
		ctxt->memop.bytes = 2;
		goto mem_common;
	case OpMem32:
		ctxt->memop.bytes = 4;
		goto mem_common;
	case OpImmU16:
		rc = decode_imm(ctxt, op, 2, false);
		break;
	case OpImmU:
		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
		break;
	case OpSI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4326
			register_address(ctxt, VCPU_REGS_RSI);
B
Bandan Das 已提交
4327
		op->addr.mem.seg = ctxt->seg_override;
4328
		op->val = 0;
4329
		op->count = 1;
4330
		break;
P
Paolo Bonzini 已提交
4331 4332 4333 4334
	case OpXLat:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
4335
			address_mask(ctxt,
P
Paolo Bonzini 已提交
4336 4337
				reg_read(ctxt, VCPU_REGS_RBX) +
				(reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
B
Bandan Das 已提交
4338
		op->addr.mem.seg = ctxt->seg_override;
P
Paolo Bonzini 已提交
4339 4340
		op->val = 0;
		break;
4341 4342 4343 4344 4345 4346 4347 4348 4349
	case OpImmFAddr:
		op->type = OP_IMM;
		op->addr.mem.ea = ctxt->_eip;
		op->bytes = ctxt->op_bytes + 2;
		insn_fetch_arr(op->valptr, op->bytes, ctxt);
		break;
	case OpMemFAddr:
		ctxt->memop.bytes = ctxt->op_bytes + 2;
		goto mem_common;
4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367
	case OpES:
		op->val = VCPU_SREG_ES;
		break;
	case OpCS:
		op->val = VCPU_SREG_CS;
		break;
	case OpSS:
		op->val = VCPU_SREG_SS;
		break;
	case OpDS:
		op->val = VCPU_SREG_DS;
		break;
	case OpFS:
		op->val = VCPU_SREG_FS;
		break;
	case OpGS:
		op->val = VCPU_SREG_GS;
		break;
4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378
	case OpImplicit:
		/* Special instructions do their own operand decoding. */
	default:
		op->type = OP_NONE; /* Disable writeback. */
		break;
	}

done:
	return rc;
}

4379
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4380 4381 4382
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
4383
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4384
	bool op_prefix = false;
B
Bandan Das 已提交
4385
	bool has_seg_override = false;
4386
	struct opcode opcode;
4387

4388 4389
	ctxt->memop.type = OP_NONE;
	ctxt->memopp = NULL;
4390
	ctxt->_eip = ctxt->eip;
4391 4392
	ctxt->fetch.ptr = ctxt->fetch.data;
	ctxt->fetch.end = ctxt->fetch.data + insn_len;
B
Borislav Petkov 已提交
4393
	ctxt->opcode_len = 1;
4394
	if (insn_len > 0)
4395
		memcpy(ctxt->fetch.data, insn, insn_len);
4396
	else {
4397
		rc = __do_insn_fetch_bytes(ctxt, 1);
4398 4399 4400
		if (rc != X86EMUL_CONTINUE)
			return rc;
	}
4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
4418
		return EMULATION_FAILED;
4419 4420
	}

4421 4422
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
4423 4424 4425

	/* Legacy prefixes. */
	for (;;) {
4426
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
4427
		case 0x66:	/* operand-size override */
4428
			op_prefix = true;
4429
			/* switch between 2/4 bytes */
4430
			ctxt->op_bytes = def_op_bytes ^ 6;
4431 4432 4433 4434
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
4435
				ctxt->ad_bytes = def_ad_bytes ^ 12;
4436 4437
			else
				/* switch between 2/4 bytes */
4438
				ctxt->ad_bytes = def_ad_bytes ^ 6;
4439 4440 4441 4442 4443
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
B
Bandan Das 已提交
4444 4445
			has_seg_override = true;
			ctxt->seg_override = (ctxt->b >> 3) & 3;
4446 4447 4448
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
B
Bandan Das 已提交
4449 4450
			has_seg_override = true;
			ctxt->seg_override = ctxt->b & 7;
4451 4452 4453 4454
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
4455
			ctxt->rex_prefix = ctxt->b;
4456 4457
			continue;
		case 0xf0:	/* LOCK */
4458
			ctxt->lock_prefix = 1;
4459 4460 4461
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
4462
			ctxt->rep_prefix = ctxt->b;
4463 4464 4465 4466 4467 4468 4469
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

4470
		ctxt->rex_prefix = 0;
4471 4472 4473 4474 4475
	}

done_prefixes:

	/* REX prefix. */
4476 4477
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
4478 4479

	/* Opcode byte(s). */
4480
	opcode = opcode_table[ctxt->b];
4481
	/* Two-byte opcode? */
4482
	if (ctxt->b == 0x0f) {
B
Borislav Petkov 已提交
4483
		ctxt->opcode_len = 2;
4484
		ctxt->b = insn_fetch(u8, ctxt);
4485
		opcode = twobyte_table[ctxt->b];
4486 4487 4488 4489 4490 4491 4492

		/* 0F_38 opcode map */
		if (ctxt->b == 0x38) {
			ctxt->opcode_len = 3;
			ctxt->b = insn_fetch(u8, ctxt);
			opcode = opcode_map_0f_38[ctxt->b];
		}
4493
	}
4494
	ctxt->d = opcode.flags;
4495

4496 4497 4498
	if (ctxt->d & ModRM)
		ctxt->modrm = insn_fetch(u8, ctxt);

4499 4500
	/* vex-prefix instructions are not implemented */
	if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4501
	    (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4502 4503 4504
		ctxt->d = NotImpl;
	}

4505 4506
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
4507
		case Group:
4508
			goffset = (ctxt->modrm >> 3) & 7;
4509 4510 4511
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
4512 4513
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
4514 4515 4516 4517 4518
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
4519
			goffset = ctxt->modrm & 7;
4520
			opcode = opcode.u.group[goffset];
4521 4522
			break;
		case Prefix:
4523
			if (ctxt->rep_prefix && op_prefix)
4524
				return EMULATION_FAILED;
4525
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4526 4527 4528 4529 4530 4531 4532
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
4533 4534 4535 4536 4537 4538
		case Escape:
			if (ctxt->modrm > 0xbf)
				opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
			else
				opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
			break;
4539
		default:
4540
			return EMULATION_FAILED;
4541
		}
4542

4543
		ctxt->d &= ~(u64)GroupMask;
4544
		ctxt->d |= opcode.flags;
4545 4546
	}

4547 4548 4549 4550
	/* Unrecognised? */
	if (ctxt->d == 0)
		return EMULATION_FAILED;

4551
	ctxt->execute = opcode.u.execute;
4552

4553 4554 4555
	if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
		return EMULATION_FAILED;

4556
	if (unlikely(ctxt->d &
4557 4558
	    (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
	     No16))) {
4559 4560 4561 4562 4563 4564
		/*
		 * These are copied unconditionally here, and checked unconditionally
		 * in x86_emulate_insn.
		 */
		ctxt->check_perm = opcode.check_perm;
		ctxt->intercept = opcode.intercept;
4565

4566 4567
		if (ctxt->d & NotImpl)
			return EMULATION_FAILED;
4568

4569 4570 4571 4572 4573 4574
		if (mode == X86EMUL_MODE_PROT64) {
			if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
				ctxt->op_bytes = 8;
			else if (ctxt->d & NearBranch)
				ctxt->op_bytes = 8;
		}
4575

4576 4577 4578 4579 4580 4581 4582
		if (ctxt->d & Op3264) {
			if (mode == X86EMUL_MODE_PROT64)
				ctxt->op_bytes = 8;
			else
				ctxt->op_bytes = 4;
		}

4583 4584 4585
		if ((ctxt->d & No16) && ctxt->op_bytes == 2)
			ctxt->op_bytes = 4;

4586 4587 4588 4589 4590
		if (ctxt->d & Sse)
			ctxt->op_bytes = 16;
		else if (ctxt->d & Mmx)
			ctxt->op_bytes = 8;
	}
A
Avi Kivity 已提交
4591

4592
	/* ModRM and SIB bytes. */
4593
	if (ctxt->d & ModRM) {
4594
		rc = decode_modrm(ctxt, &ctxt->memop);
B
Bandan Das 已提交
4595 4596 4597 4598
		if (!has_seg_override) {
			has_seg_override = true;
			ctxt->seg_override = ctxt->modrm_seg;
		}
4599
	} else if (ctxt->d & MemAbs)
4600
		rc = decode_abs(ctxt, &ctxt->memop);
4601 4602 4603
	if (rc != X86EMUL_CONTINUE)
		goto done;

B
Bandan Das 已提交
4604 4605
	if (!has_seg_override)
		ctxt->seg_override = VCPU_SREG_DS;
4606

B
Bandan Das 已提交
4607
	ctxt->memop.addr.mem.seg = ctxt->seg_override;
4608 4609 4610 4611 4612

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
4613
	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4614 4615 4616
	if (rc != X86EMUL_CONTINUE)
		goto done;

4617 4618 4619 4620
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
4621
	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4622 4623 4624
	if (rc != X86EMUL_CONTINUE)
		goto done;

4625
	/* Decode and fetch the destination operand: register or memory. */
4626
	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4627

4628
	if (ctxt->rip_relative)
4629 4630
		ctxt->memopp->addr.mem.ea = address_mask(ctxt,
					ctxt->memopp->addr.mem.ea + ctxt->_eip);
4631

4632
done:
4633
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4634 4635
}

4636 4637 4638 4639 4640
bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
{
	return ctxt->d & PageTable;
}

4641 4642 4643 4644 4645 4646 4647 4648 4649
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
4650 4651 4652
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
4653
		 ((ctxt->eflags & EFLG_ZF) == 0))
4654
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
4655 4656 4657 4658 4659 4660
		    ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
		return true;

	return false;
}

A
Avi Kivity 已提交
4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673
static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
{
	bool fault = false;

	ctxt->ops->get_fpu(ctxt);
	asm volatile("1: fwait \n\t"
		     "2: \n\t"
		     ".pushsection .fixup,\"ax\" \n\t"
		     "3: \n\t"
		     "movb $1, %[fault] \n\t"
		     "jmp 2b \n\t"
		     ".popsection \n\t"
		     _ASM_EXTABLE(1b, 3b)
4674
		     : [fault]"+qm"(fault));
A
Avi Kivity 已提交
4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689
	ctxt->ops->put_fpu(ctxt);

	if (unlikely(fault))
		return emulate_exception(ctxt, MF_VECTOR, 0, false);

	return X86EMUL_CONTINUE;
}

static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
				       struct operand *op)
{
	if (op->type == OP_MM)
		read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
}

4690 4691 4692
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
{
	ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4693 4694
	if (!(ctxt->d & ByteOp))
		fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4695
	asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4696 4697 4698
	    : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
	      [fastop]"+S"(fop)
	    : "c"(ctxt->src2.val));
4699
	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4700 4701
	if (!fop) /* exception is returned in fop variable */
		return emulate_de(ctxt);
4702 4703
	return X86EMUL_CONTINUE;
}
4704

4705 4706
void init_decode_cache(struct x86_emulate_ctxt *ctxt)
{
B
Bandan Das 已提交
4707 4708
	memset(&ctxt->rip_relative, 0,
	       (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4709 4710 4711 4712 4713 4714

	ctxt->io_read.pos = 0;
	ctxt->io_read.end = 0;
	ctxt->mem_read.end = 0;
}

4715
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4716
{
4717
	const struct x86_emulate_ops *ops = ctxt->ops;
4718
	int rc = X86EMUL_CONTINUE;
4719
	int saved_dst_type = ctxt->dst.type;
4720

4721
	ctxt->mem_read.pos = 0;
4722

4723 4724
	/* LOCK prefix is allowed only with some instructions */
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4725
		rc = emulate_ud(ctxt);
4726 4727 4728
		goto done;
	}

4729
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4730
		rc = emulate_ud(ctxt);
4731 4732 4733
		goto done;
	}

4734 4735 4736 4737 4738 4739 4740
	if (unlikely(ctxt->d &
		     (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
		if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
				(ctxt->d & Undefined)) {
			rc = emulate_ud(ctxt);
			goto done;
		}
A
Avi Kivity 已提交
4741

4742 4743 4744
		if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
		    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
			rc = emulate_ud(ctxt);
A
Avi Kivity 已提交
4745
			goto done;
4746
		}
A
Avi Kivity 已提交
4747

4748 4749
		if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
			rc = emulate_nm(ctxt);
4750
			goto done;
4751
		}
4752

4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765
		if (ctxt->d & Mmx) {
			rc = flush_pending_x87_faults(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			/*
			 * Now that we know the fpu is exception safe, we can fetch
			 * operands from it.
			 */
			fetch_possible_mmx_operand(ctxt, &ctxt->src);
			fetch_possible_mmx_operand(ctxt, &ctxt->src2);
			if (!(ctxt->d & Mov))
				fetch_possible_mmx_operand(ctxt, &ctxt->dst);
		}
4766

4767
		if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4768 4769 4770 4771 4772
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_PRE_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}
4773

4774 4775
		/* Privileged instruction can be executed only in CPL=0 */
		if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4776 4777 4778 4779
			if (ctxt->d & PrivUD)
				rc = emulate_ud(ctxt);
			else
				rc = emulate_gp(ctxt, 0);
4780
			goto done;
4781
		}
4782

4783 4784 4785
		/* Instruction can only be executed in protected mode */
		if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
			rc = emulate_ud(ctxt);
4786
			goto done;
4787
		}
4788

4789
		/* Do instruction specific permission checks */
4790
		if (ctxt->d & CheckPerm) {
4791 4792 4793 4794 4795
			rc = ctxt->check_perm(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

4796
		if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4797 4798 4799 4800 4801 4802 4803 4804 4805 4806
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_POST_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

		if (ctxt->rep_prefix && (ctxt->d & String)) {
			/* All REP prefixes have the same first termination condition */
			if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
				ctxt->eip = ctxt->_eip;
4807
				ctxt->eflags &= ~EFLG_RF;
4808 4809
				goto done;
			}
4810 4811 4812
		}
	}

4813 4814 4815
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
4816
		if (rc != X86EMUL_CONTINUE)
4817
			goto done;
4818
		ctxt->src.orig_val64 = ctxt->src.val64;
4819 4820
	}

4821 4822 4823
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
4824 4825 4826 4827
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4828
	if ((ctxt->d & DstMask) == ImplicitOps)
4829 4830 4831
		goto special_insn;


4832
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4833
		/* optimisation - avoid slow emulated read if Mov */
4834 4835
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
4836 4837
		if (rc != X86EMUL_CONTINUE)
			goto done;
4838
	}
4839
	ctxt->dst.orig_val = ctxt->dst.val;
4840

4841 4842
special_insn:

4843
	if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4844
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
4845
					      X86_ICPT_POST_MEMACCESS);
4846 4847 4848 4849
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

4850 4851 4852 4853
	if (ctxt->rep_prefix && (ctxt->d & String))
		ctxt->eflags |= EFLG_RF;
	else
		ctxt->eflags &= ~EFLG_RF;
4854

4855
	if (ctxt->execute) {
4856 4857 4858 4859 4860 4861 4862
		if (ctxt->d & Fastop) {
			void (*fop)(struct fastop *) = (void *)ctxt->execute;
			rc = fastop(ctxt, fop);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			goto writeback;
		}
4863
		rc = ctxt->execute(ctxt);
4864 4865 4866 4867 4868
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

B
Borislav Petkov 已提交
4869
	if (ctxt->opcode_len == 2)
A
Avi Kivity 已提交
4870
		goto twobyte_insn;
4871 4872
	else if (ctxt->opcode_len == 3)
		goto threebyte_insn;
A
Avi Kivity 已提交
4873

4874
	switch (ctxt->b) {
A
Avi Kivity 已提交
4875
	case 0x63:		/* movsxd */
4876
		if (ctxt->mode != X86EMUL_MODE_PROT64)
A
Avi Kivity 已提交
4877
			goto cannot_emulate;
4878
		ctxt->dst.val = (s32) ctxt->src.val;
A
Avi Kivity 已提交
4879
		break;
4880
	case 0x70 ... 0x7f: /* jcc (short) */
4881
		if (test_cc(ctxt->b, ctxt->eflags))
4882
			rc = jmp_rel(ctxt, ctxt->src.val);
4883
		break;
N
Nitin A Kamble 已提交
4884
	case 0x8d: /* lea r16/r32, m */
4885
		ctxt->dst.val = ctxt->src.addr.mem.ea;
N
Nitin A Kamble 已提交
4886
		break;
4887
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
4888
		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4889 4890 4891
			ctxt->dst.type = OP_NONE;
		else
			rc = em_xchg(ctxt);
4892
		break;
4893
	case 0x98: /* cbw/cwde/cdqe */
4894 4895 4896 4897
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4898 4899
		}
		break;
4900
	case 0xcc:		/* int3 */
4901 4902
		rc = emulate_int(ctxt, 3);
		break;
4903
	case 0xcd:		/* int n */
4904
		rc = emulate_int(ctxt, ctxt->src.val);
4905 4906
		break;
	case 0xce:		/* into */
4907 4908
		if (ctxt->eflags & EFLG_OF)
			rc = emulate_int(ctxt, 4);
4909
		break;
4910
	case 0xe9: /* jmp rel */
4911
	case 0xeb: /* jmp rel short */
4912
		rc = jmp_rel(ctxt, ctxt->src.val);
4913
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
4914
		break;
4915
	case 0xf4:              /* hlt */
4916
		ctxt->ops->halt(ctxt);
4917
		break;
4918 4919 4920 4921 4922 4923 4924
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
		ctxt->eflags ^= EFLG_CF;
		break;
	case 0xf8: /* clc */
		ctxt->eflags &= ~EFLG_CF;
		break;
4925 4926 4927
	case 0xf9: /* stc */
		ctxt->eflags |= EFLG_CF;
		break;
4928 4929 4930 4931 4932 4933
	case 0xfc: /* cld */
		ctxt->eflags &= ~EFLG_DF;
		break;
	case 0xfd: /* std */
		ctxt->eflags |= EFLG_DF;
		break;
4934 4935
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
4936
	}
4937

4938 4939 4940
	if (rc != X86EMUL_CONTINUE)
		goto done;

4941
writeback:
4942 4943 4944 4945 4946 4947
	if (ctxt->d & SrcWrite) {
		BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
		rc = writeback(ctxt, &ctxt->src);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
4948 4949 4950 4951 4952
	if (!(ctxt->d & NoWrite)) {
		rc = writeback(ctxt, &ctxt->dst);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
4953

4954 4955 4956 4957
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
4958
	ctxt->dst.type = saved_dst_type;
4959

4960
	if ((ctxt->d & SrcMask) == SrcSI)
4961
		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4962

4963
	if ((ctxt->d & DstMask) == DstDI)
4964
		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4965

4966
	if (ctxt->rep_prefix && (ctxt->d & String)) {
4967
		unsigned int count;
4968
		struct read_cache *r = &ctxt->io_read;
4969 4970 4971 4972
		if ((ctxt->d & SrcMask) == SrcSI)
			count = ctxt->src.count;
		else
			count = ctxt->dst.count;
4973
		register_address_increment(ctxt, VCPU_REGS_RCX, -count);
4974

4975 4976 4977 4978 4979
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
4980
			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4981 4982 4983 4984 4985 4986
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
4987
				ctxt->mem_read.end = 0;
4988
				writeback_registers(ctxt);
4989 4990 4991
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
4992
		}
4993
		ctxt->eflags &= ~EFLG_RF;
4994
	}
4995

4996
	ctxt->eip = ctxt->_eip;
4997 4998

done:
4999 5000
	if (rc == X86EMUL_PROPAGATE_FAULT) {
		WARN_ON(ctxt->exception.vector > 0x1f);
5001
		ctxt->have_exception = true;
5002
	}
5003 5004 5005
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

5006 5007 5008
	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);

5009
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
A
Avi Kivity 已提交
5010 5011

twobyte_insn:
5012
	switch (ctxt->b) {
5013
	case 0x09:		/* wbinvd */
5014
		(ctxt->ops->wbinvd)(ctxt);
5015 5016
		break;
	case 0x08:		/* invd */
5017 5018
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
P
Paolo Bonzini 已提交
5019
	case 0x1f:		/* nop */
5020 5021
		break;
	case 0x20: /* mov cr, reg */
5022
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5023
		break;
A
Avi Kivity 已提交
5024
	case 0x21: /* mov from dr to reg */
5025
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
A
Avi Kivity 已提交
5026 5027
		break;
	case 0x40 ... 0x4f:	/* cmov */
5028 5029 5030 5031
		if (test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.val = ctxt->src.val;
		else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
			 ctxt->op_bytes != 4)
5032
			ctxt->dst.type = OP_NONE; /* no writeback */
A
Avi Kivity 已提交
5033
		break;
5034
	case 0x80 ... 0x8f: /* jnz rel, etc*/
5035
		if (test_cc(ctxt->b, ctxt->eflags))
5036
			rc = jmp_rel(ctxt, ctxt->src.val);
5037
		break;
5038
	case 0x90 ... 0x9f:     /* setcc r/m8 */
5039
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5040
		break;
A
Avi Kivity 已提交
5041
	case 0xb6 ... 0xb7:	/* movzx */
5042
		ctxt->dst.bytes = ctxt->op_bytes;
5043
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5044
						       : (u16) ctxt->src.val;
A
Avi Kivity 已提交
5045 5046
		break;
	case 0xbe ... 0xbf:	/* movsx */
5047
		ctxt->dst.bytes = ctxt->op_bytes;
5048
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5049
							(s16) ctxt->src.val;
A
Avi Kivity 已提交
5050
		break;
5051 5052
	default:
		goto cannot_emulate;
A
Avi Kivity 已提交
5053
	}
5054

5055 5056
threebyte_insn:

5057 5058 5059
	if (rc != X86EMUL_CONTINUE)
		goto done;

A
Avi Kivity 已提交
5060 5061 5062
	goto writeback;

cannot_emulate:
5063
	return EMULATION_FAILED;
A
Avi Kivity 已提交
5064
}
5065 5066 5067 5068 5069 5070 5071 5072 5073 5074

void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
{
	invalidate_registers(ctxt);
}

void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
{
	writeback_registers(ctxt);
}