198 		int (*execute)(struct x86_emulate_ctxt *ctxt);
207 	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
246 static void writeback_registers(struct x86_emulate_ctxt *ctxt)  in writeback_registers()  argument
248 	unsigned long dirty = ctxt->regs_dirty;  in writeback_registers()
252 		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);  in writeback_registers()
255 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)  in invalidate_registers()  argument
257 	ctxt->regs_dirty = 0;  in invalidate_registers()
258 	ctxt->regs_valid = 0;  in invalidate_registers()
291 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
466 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,  in emulator_check_intercept()  argument
472 		.rep_prefix = ctxt->rep_prefix,  in emulator_check_intercept()
473 		.modrm_mod  = ctxt->modrm_mod,  in emulator_check_intercept()
474 		.modrm_reg  = ctxt->modrm_reg,  in emulator_check_intercept()
475 		.modrm_rm   = ctxt->modrm_rm,  in emulator_check_intercept()
476 		.src_val    = ctxt->src.val64,  in emulator_check_intercept()
477 		.dst_val    = ctxt->dst.val64,  in emulator_check_intercept()
478 		.src_bytes  = ctxt->src.bytes,  in emulator_check_intercept()
479 		.dst_bytes  = ctxt->dst.bytes,  in emulator_check_intercept()
480 		.ad_bytes   = ctxt->ad_bytes,  in emulator_check_intercept()
481 		.next_rip   = ctxt->eip,  in emulator_check_intercept()
484 	return ctxt->ops->intercept(ctxt, &info, stage);  in emulator_check_intercept()
511 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)  in ad_mask()  argument
513 	return (1UL << (ctxt->ad_bytes << 3)) - 1;  in ad_mask()
516 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)  in stack_mask()  argument
521 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in stack_mask()
523 	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);  in stack_mask()
527 static int stack_size(struct x86_emulate_ctxt *ctxt)  in stack_size()  argument
529 	return (__fls(stack_mask(ctxt)) + 1) >> 3;  in stack_size()
534 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)  in address_mask()  argument
536 	if (ctxt->ad_bytes == sizeof(unsigned long))  in address_mask()
539 		return reg & ad_mask(ctxt);  in address_mask()
543 register_address(struct x86_emulate_ctxt *ctxt, int reg)  in register_address()  argument
545 	return address_mask(ctxt, reg_read(ctxt, reg));  in register_address()
554 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)  in register_address_increment()  argument
556 	ulong *preg = reg_rmw(ctxt, reg);  in register_address_increment()
558 	assign_register(preg, *preg + inc, ctxt->ad_bytes);  in register_address_increment()
561 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)  in rsp_increment()  argument
563 	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);  in rsp_increment()
573 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)  in seg_base()  argument
575 	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)  in seg_base()
578 	return ctxt->ops->get_cached_segment_base(ctxt, seg);  in seg_base()
581 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,  in emulate_exception()  argument
584 	if (KVM_EMULATOR_BUG_ON(vec > 0x1f, ctxt))  in emulate_exception()
587 	ctxt->exception.vector = vec;  in emulate_exception()
588 	ctxt->exception.error_code = error;  in emulate_exception()
589 	ctxt->exception.error_code_valid = valid;  in emulate_exception()
593 static int emulate_db(struct x86_emulate_ctxt *ctxt)  in emulate_db()  argument
595 	return emulate_exception(ctxt, DB_VECTOR, 0, false);  in emulate_db()
598 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)  in emulate_gp()  argument
600 	return emulate_exception(ctxt, GP_VECTOR, err, true);  in emulate_gp()
603 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)  in emulate_ss()  argument
605 	return emulate_exception(ctxt, SS_VECTOR, err, true);  in emulate_ss()
608 static int emulate_ud(struct x86_emulate_ctxt *ctxt)  in emulate_ud()  argument
610 	return emulate_exception(ctxt, UD_VECTOR, 0, false);  in emulate_ud()
613 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)  in emulate_ts()  argument
615 	return emulate_exception(ctxt, TS_VECTOR, err, true);  in emulate_ts()
618 static int emulate_de(struct x86_emulate_ctxt *ctxt)  in emulate_de()  argument
620 	return emulate_exception(ctxt, DE_VECTOR, 0, false);  in emulate_de()
623 static int emulate_nm(struct x86_emulate_ctxt *ctxt)  in emulate_nm()  argument
625 	return emulate_exception(ctxt, NM_VECTOR, 0, false);  in emulate_nm()
628 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)  in get_segment_selector()  argument
633 	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);  in get_segment_selector()
637 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,  in set_segment_selector()  argument
644 	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);  in set_segment_selector()
645 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);  in set_segment_selector()
648 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)  in ctxt_virt_addr_bits()  argument
650 	return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;  in ctxt_virt_addr_bits()
654 						struct x86_emulate_ctxt *ctxt)  in emul_is_noncanonical_address()  argument
656 	return !__is_canonical_address(la, ctxt_virt_addr_bits(ctxt));  in emul_is_noncanonical_address()
668 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)  in insn_alignment()  argument
670 	u64 alignment = ctxt->d & AlignMask;  in insn_alignment()
687 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,  in __linearize()  argument
700 	la = seg_base(ctxt, addr.seg) + addr.ea;  in __linearize()
705 		va_bits = ctxt_virt_addr_bits(ctxt);  in __linearize()
715 		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,  in __linearize()
720 		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))  in __linearize()
744 	if (la & (insn_alignment(ctxt, size) - 1))  in __linearize()
745 		return emulate_gp(ctxt, 0);  in __linearize()
749 		return emulate_ss(ctxt, 0);  in __linearize()
751 		return emulate_gp(ctxt, 0);  in __linearize()
754 static int linearize(struct x86_emulate_ctxt *ctxt,  in linearize()  argument
760 	return __linearize(ctxt, addr, &max_size, size, write, false,  in linearize()
761 			   ctxt->mode, linear);  in linearize()
764 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst)  in assign_eip()  argument
772 	if (ctxt->op_bytes != sizeof(unsigned long))  in assign_eip()
773 		addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);  in assign_eip()
774 	rc = __linearize(ctxt, addr, &max_size, 1, false, true, ctxt->mode, &linear);  in assign_eip()
776 		ctxt->_eip = addr.ea;  in assign_eip()
780 static inline int emulator_recalc_and_set_mode(struct x86_emulate_ctxt *ctxt)  in emulator_recalc_and_set_mode()  argument
787 	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in emulator_recalc_and_set_mode()
789 	if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) {  in emulator_recalc_and_set_mode()
793 		ctxt->mode = X86EMUL_MODE_REAL;  in emulator_recalc_and_set_mode()
797 	if (ctxt->eflags & X86_EFLAGS_VM) {  in emulator_recalc_and_set_mode()
801 		ctxt->mode = X86EMUL_MODE_VM86;  in emulator_recalc_and_set_mode()
805 	if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS))  in emulator_recalc_and_set_mode()
811 			ctxt->mode = X86EMUL_MODE_PROT64;  in emulator_recalc_and_set_mode()
814 			ctxt->mode = X86EMUL_MODE_PROT32;  in emulator_recalc_and_set_mode()
816 			ctxt->mode = X86EMUL_MODE_PROT16;  in emulator_recalc_and_set_mode()
820 		ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;  in emulator_recalc_and_set_mode()
826 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)  in assign_eip_near()  argument
828 	return assign_eip(ctxt, dst);  in assign_eip_near()
831 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst)  in assign_eip_far()  argument
833 	int rc = emulator_recalc_and_set_mode(ctxt);  in assign_eip_far()
838 	return assign_eip(ctxt, dst);  in assign_eip_far()
841 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)  in jmp_rel()  argument
843 	return assign_eip_near(ctxt, ctxt->_eip + rel);  in jmp_rel()
846 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,  in linear_read_system()  argument
849 	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);  in linear_read_system()
852 static int linear_write_system(struct x86_emulate_ctxt *ctxt,  in linear_write_system()  argument
856 	return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);  in linear_write_system()
859 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,  in segmented_read_std()  argument
867 	rc = linearize(ctxt, addr, size, false, &linear);  in segmented_read_std()
870 	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);  in segmented_read_std()
873 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,  in segmented_write_std()  argument
881 	rc = linearize(ctxt, addr, size, true, &linear);  in segmented_write_std()
884 	return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);  in segmented_write_std()
891 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)  in __do_insn_fetch_bytes()  argument
896 	int cur_size = ctxt->fetch.end - ctxt->fetch.data;  in __do_insn_fetch_bytes()
898 					   .ea = ctxt->eip + cur_size };  in __do_insn_fetch_bytes()
910 	rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,  in __do_insn_fetch_bytes()
925 		return emulate_gp(ctxt, 0);  in __do_insn_fetch_bytes()
927 	rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,  in __do_insn_fetch_bytes()
928 			      size, &ctxt->exception);  in __do_insn_fetch_bytes()
931 	ctxt->fetch.end += size;  in __do_insn_fetch_bytes()
935 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,  in do_insn_fetch_bytes()  argument
938 	unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;  in do_insn_fetch_bytes()
941 		return __do_insn_fetch_bytes(ctxt, size - done_size);  in do_insn_fetch_bytes()
953 	ctxt->_eip += sizeof(_type);					\
954 	memcpy(&_x, ctxt->fetch.ptr, sizeof(_type));			\
955 	ctxt->fetch.ptr += sizeof(_type);				\
964 	ctxt->_eip += (_size);						\
965 	memcpy(_arr, ctxt->fetch.ptr, _size);				\
966 	ctxt->fetch.ptr += (_size);					\
974 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,  in decode_register()  argument
978 	int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;  in decode_register()
981 		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;  in decode_register()
983 		p = reg_rmw(ctxt, modrm_reg);  in decode_register()
987 static int read_descriptor(struct x86_emulate_ctxt *ctxt,  in read_descriptor()  argument
996 	rc = segmented_read_std(ctxt, addr, size, 2);  in read_descriptor()
1000 	rc = segmented_read_std(ctxt, addr, address, op_bytes);  in read_descriptor()
1048 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)  in em_bsf_c()  argument
1051 	if (ctxt->src.val == 0)  in em_bsf_c()
1052 		ctxt->dst.type = OP_NONE;  in em_bsf_c()
1053 	return fastop(ctxt, em_bsf);  in em_bsf_c()
1056 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)  in em_bsr_c()  argument
1059 	if (ctxt->src.val == 0)  in em_bsr_c()
1060 		ctxt->dst.type = OP_NONE;  in em_bsr_c()
1061 	return fastop(ctxt, em_bsr);  in em_bsr_c()
1093 static int em_fninit(struct x86_emulate_ctxt *ctxt)  in em_fninit()  argument
1095 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in em_fninit()
1096 		return emulate_nm(ctxt);  in em_fninit()
1104 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)  in em_fnstcw()  argument
1108 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in em_fnstcw()
1109 		return emulate_nm(ctxt);  in em_fnstcw()
1115 	ctxt->dst.val = fcw;  in em_fnstcw()
1120 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)  in em_fnstsw()  argument
1124 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in em_fnstsw()
1125 		return emulate_nm(ctxt);  in em_fnstsw()
1131 	ctxt->dst.val = fsw;  in em_fnstsw()
1136 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,  in decode_register_operand()  argument
1141 	if (ctxt->d & ModRM)  in decode_register_operand()
1142 		reg = ctxt->modrm_reg;  in decode_register_operand()
1144 		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);  in decode_register_operand()
1146 	if (ctxt->d & Sse) {  in decode_register_operand()
1153 	if (ctxt->d & Mmx) {  in decode_register_operand()
1162 	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_register_operand()
1163 	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);  in decode_register_operand()
1169 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)  in adjust_modrm_seg()  argument
1172 		ctxt->modrm_seg = VCPU_SREG_SS;  in adjust_modrm_seg()
1175 static int decode_modrm(struct x86_emulate_ctxt *ctxt,  in decode_modrm()  argument
1183 	ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */  in decode_modrm()
1184 	index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */  in decode_modrm()
1185 	base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */  in decode_modrm()
1187 	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;  in decode_modrm()
1188 	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;  in decode_modrm()
1189 	ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);  in decode_modrm()
1190 	ctxt->modrm_seg = VCPU_SREG_DS;  in decode_modrm()
1192 	if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {  in decode_modrm()
1194 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_modrm()
1195 		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,  in decode_modrm()
1196 				ctxt->d & ByteOp);  in decode_modrm()
1197 		if (ctxt->d & Sse) {  in decode_modrm()
1200 			op->addr.xmm = ctxt->modrm_rm;  in decode_modrm()
1201 			kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val);  in decode_modrm()
1204 		if (ctxt->d & Mmx) {  in decode_modrm()
1207 			op->addr.mm = ctxt->modrm_rm & 7;  in decode_modrm()
1216 	if (ctxt->ad_bytes == 2) {  in decode_modrm()
1217 		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);  in decode_modrm()
1218 		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);  in decode_modrm()
1219 		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);  in decode_modrm()
1220 		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);  in decode_modrm()
1223 		switch (ctxt->modrm_mod) {  in decode_modrm()
1225 			if (ctxt->modrm_rm == 6)  in decode_modrm()
1226 				modrm_ea += insn_fetch(u16, ctxt);  in decode_modrm()
1229 			modrm_ea += insn_fetch(s8, ctxt);  in decode_modrm()
1232 			modrm_ea += insn_fetch(u16, ctxt);  in decode_modrm()
1235 		switch (ctxt->modrm_rm) {  in decode_modrm()
1255 			if (ctxt->modrm_mod != 0)  in decode_modrm()
1262 		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||  in decode_modrm()
1263 		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))  in decode_modrm()
1264 			ctxt->modrm_seg = VCPU_SREG_SS;  in decode_modrm()
1268 		if ((ctxt->modrm_rm & 7) == 4) {  in decode_modrm()
1269 			sib = insn_fetch(u8, ctxt);  in decode_modrm()
1274 			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)  in decode_modrm()
1275 				modrm_ea += insn_fetch(s32, ctxt);  in decode_modrm()
1277 				modrm_ea += reg_read(ctxt, base_reg);  in decode_modrm()
1278 				adjust_modrm_seg(ctxt, base_reg);  in decode_modrm()
1280 				if ((ctxt->d & IncSP) &&  in decode_modrm()
1282 					modrm_ea += ctxt->op_bytes;  in decode_modrm()
1285 				modrm_ea += reg_read(ctxt, index_reg) << scale;  in decode_modrm()
1286 		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {  in decode_modrm()
1287 			modrm_ea += insn_fetch(s32, ctxt);  in decode_modrm()
1288 			if (ctxt->mode == X86EMUL_MODE_PROT64)  in decode_modrm()
1289 				ctxt->rip_relative = 1;  in decode_modrm()
1291 			base_reg = ctxt->modrm_rm;  in decode_modrm()
1292 			modrm_ea += reg_read(ctxt, base_reg);  in decode_modrm()
1293 			adjust_modrm_seg(ctxt, base_reg);  in decode_modrm()
1295 		switch (ctxt->modrm_mod) {  in decode_modrm()
1297 			modrm_ea += insn_fetch(s8, ctxt);  in decode_modrm()
1300 			modrm_ea += insn_fetch(s32, ctxt);  in decode_modrm()
1305 	if (ctxt->ad_bytes != 8)  in decode_modrm()
1306 		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;  in decode_modrm()
1312 static int decode_abs(struct x86_emulate_ctxt *ctxt,  in decode_abs()  argument
1318 	switch (ctxt->ad_bytes) {  in decode_abs()
1320 		op->addr.mem.ea = insn_fetch(u16, ctxt);  in decode_abs()
1323 		op->addr.mem.ea = insn_fetch(u32, ctxt);  in decode_abs()
1326 		op->addr.mem.ea = insn_fetch(u64, ctxt);  in decode_abs()
1333 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)  in fetch_bit_operand()  argument
1337 	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {  in fetch_bit_operand()
1338 		mask = ~((long)ctxt->dst.bytes * 8 - 1);  in fetch_bit_operand()
1340 		if (ctxt->src.bytes == 2)  in fetch_bit_operand()
1341 			sv = (s16)ctxt->src.val & (s16)mask;  in fetch_bit_operand()
1342 		else if (ctxt->src.bytes == 4)  in fetch_bit_operand()
1343 			sv = (s32)ctxt->src.val & (s32)mask;  in fetch_bit_operand()
1345 			sv = (s64)ctxt->src.val & (s64)mask;  in fetch_bit_operand()
1347 		ctxt->dst.addr.mem.ea = address_mask(ctxt,  in fetch_bit_operand()
1348 					   ctxt->dst.addr.mem.ea + (sv >> 3));  in fetch_bit_operand()
1352 	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;  in fetch_bit_operand()
1355 static int read_emulated(struct x86_emulate_ctxt *ctxt,  in read_emulated()  argument
1359 	struct read_cache *mc = &ctxt->mem_read;  in read_emulated()
1364 	if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt))  in read_emulated()
1367 	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,  in read_emulated()
1368 				      &ctxt->exception);  in read_emulated()
1380 static int segmented_read(struct x86_emulate_ctxt *ctxt,  in segmented_read()  argument
1388 	rc = linearize(ctxt, addr, size, false, &linear);  in segmented_read()
1391 	return read_emulated(ctxt, linear, data, size);  in segmented_read()
1394 static int segmented_write(struct x86_emulate_ctxt *ctxt,  in segmented_write()  argument
1402 	rc = linearize(ctxt, addr, size, true, &linear);  in segmented_write()
1405 	return ctxt->ops->write_emulated(ctxt, linear, data, size,  in segmented_write()
1406 					 &ctxt->exception);  in segmented_write()
1409 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,  in segmented_cmpxchg()  argument
1417 	rc = linearize(ctxt, addr, size, true, &linear);  in segmented_cmpxchg()
1420 	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,  in segmented_cmpxchg()
1421 					   size, &ctxt->exception);  in segmented_cmpxchg()
1424 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,  in pio_in_emulated()  argument
1428 	struct read_cache *rc = &ctxt->io_read;  in pio_in_emulated()
1432 		unsigned int count = ctxt->rep_prefix ?  in pio_in_emulated()
1433 			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;  in pio_in_emulated()
1434 		in_page = (ctxt->eflags & X86_EFLAGS_DF) ?  in pio_in_emulated()
1435 			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :  in pio_in_emulated()
1436 			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));  in pio_in_emulated()
1441 		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))  in pio_in_emulated()
1446 	if (ctxt->rep_prefix && (ctxt->d & String) &&  in pio_in_emulated()
1447 	    !(ctxt->eflags & X86_EFLAGS_DF)) {  in pio_in_emulated()
1448 		ctxt->dst.data = rc->data + rc->pos;  in pio_in_emulated()
1449 		ctxt->dst.type = OP_MEM_STR;  in pio_in_emulated()
1450 		ctxt->dst.count = (rc->end - rc->pos) / size;  in pio_in_emulated()
1459 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,  in read_interrupt_descriptor()  argument
1465 	ctxt->ops->get_idt(ctxt, &dt);  in read_interrupt_descriptor()
1468 		return emulate_gp(ctxt, index << 3 | 0x2);  in read_interrupt_descriptor()
1471 	return linear_read_system(ctxt, addr, desc, sizeof(*desc));  in read_interrupt_descriptor()
1474 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,  in get_descriptor_table_ptr()  argument
1477 	const struct x86_emulate_ops *ops = ctxt->ops;  in get_descriptor_table_ptr()
1485 		if (!ops->get_segment(ctxt, &sel, &desc, &base3,  in get_descriptor_table_ptr()
1492 		ops->get_gdt(ctxt, dt);  in get_descriptor_table_ptr()
1495 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,  in get_descriptor_ptr()  argument
1502 	get_descriptor_table_ptr(ctxt, selector, &dt);  in get_descriptor_ptr()
1505 		return emulate_gp(ctxt, selector & 0xfffc);  in get_descriptor_ptr()
1513 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in get_descriptor_ptr()
1524 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in read_segment_descriptor()  argument
1530 	rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);  in read_segment_descriptor()
1534 	return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));  in read_segment_descriptor()
1538 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in write_segment_descriptor()  argument
1544 	rc = get_descriptor_ptr(ctxt, selector, &addr);  in write_segment_descriptor()
1548 	return linear_write_system(ctxt, addr, desc, sizeof(*desc));  in write_segment_descriptor()
1551 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in __load_segment_descriptor()  argument
1568 	if (ctxt->mode == X86EMUL_MODE_REAL) {  in __load_segment_descriptor()
1571 		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);  in __load_segment_descriptor()
1574 	} else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {  in __load_segment_descriptor()
1597 			if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)  in __load_segment_descriptor()
1616 	ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);  in __load_segment_descriptor()
1687 			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in __load_segment_descriptor()
1725 			ret = write_segment_descriptor(ctxt, selector,  in __load_segment_descriptor()
1730 	} else if (ctxt->mode == X86EMUL_MODE_PROT64) {  in __load_segment_descriptor()
1731 		ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));  in __load_segment_descriptor()
1735 						 ((u64)base3 << 32), ctxt))  in __load_segment_descriptor()
1736 			return emulate_gp(ctxt, err_code);  in __load_segment_descriptor()
1742 		ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,  in __load_segment_descriptor()
1743 						  sizeof(seg_desc), &ctxt->exception);  in __load_segment_descriptor()
1748 	ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);  in __load_segment_descriptor()
1753 	return emulate_exception(ctxt, err_vec, err_code, true);  in __load_segment_descriptor()
1756 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,  in load_segment_descriptor()  argument
1759 	u8 cpl = ctxt->ops->cpl(ctxt);  in load_segment_descriptor()
1772 	    ctxt->mode == X86EMUL_MODE_PROT64)  in load_segment_descriptor()
1773 		return emulate_exception(ctxt, GP_VECTOR, 0, true);  in load_segment_descriptor()
1775 	return __load_segment_descriptor(ctxt, selector, seg, cpl,  in load_segment_descriptor()
1784 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)  in writeback()  argument
1791 		if (ctxt->lock_prefix)  in writeback()
1792 			return segmented_cmpxchg(ctxt,  in writeback()
1798 			return segmented_write(ctxt,  in writeback()
1803 		return segmented_write(ctxt,  in writeback()
1822 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)  in push()  argument
1826 	rsp_increment(ctxt, -bytes);  in push()
1827 	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);  in push()
1830 	return segmented_write(ctxt, addr, data, bytes);  in push()
1833 static int em_push(struct x86_emulate_ctxt *ctxt)  in em_push()  argument
1836 	ctxt->dst.type = OP_NONE;  in em_push()
1837 	return push(ctxt, &ctxt->src.val, ctxt->op_bytes);  in em_push()
1840 static int emulate_pop(struct x86_emulate_ctxt *ctxt,  in emulate_pop()  argument
1846 	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);  in emulate_pop()
1848 	rc = segmented_read(ctxt, addr, dest, len);  in emulate_pop()
1852 	rsp_increment(ctxt, len);  in emulate_pop()
1856 static int em_pop(struct x86_emulate_ctxt *ctxt)  in em_pop()  argument
1858 	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);  in em_pop()
1861 static int emulate_popf(struct x86_emulate_ctxt *ctxt,  in emulate_popf()  argument
1866 	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;  in emulate_popf()
1867 	int cpl = ctxt->ops->cpl(ctxt);  in emulate_popf()
1869 	rc = emulate_pop(ctxt, &val, len);  in emulate_popf()
1878 	switch(ctxt->mode) {  in emulate_popf()
1889 			return emulate_gp(ctxt, 0);  in emulate_popf()
1898 		(ctxt->eflags & ~change_mask) | (val & change_mask);  in emulate_popf()
1903 static int em_popf(struct x86_emulate_ctxt *ctxt)  in em_popf()  argument
1905 	ctxt->dst.type = OP_REG;  in em_popf()
1906 	ctxt->dst.addr.reg = &ctxt->eflags;  in em_popf()
1907 	ctxt->dst.bytes = ctxt->op_bytes;  in em_popf()
1908 	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);  in em_popf()
1911 static int em_enter(struct x86_emulate_ctxt *ctxt)  in em_enter()  argument
1914 	unsigned frame_size = ctxt->src.val;  in em_enter()
1915 	unsigned nesting_level = ctxt->src2.val & 31;  in em_enter()
1921 	rbp = reg_read(ctxt, VCPU_REGS_RBP);  in em_enter()
1922 	rc = push(ctxt, &rbp, stack_size(ctxt));  in em_enter()
1925 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),  in em_enter()
1926 		      stack_mask(ctxt));  in em_enter()
1927 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),  in em_enter()
1928 		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,  in em_enter()
1929 		      stack_mask(ctxt));  in em_enter()
1933 static int em_leave(struct x86_emulate_ctxt *ctxt)  in em_leave()  argument
1935 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),  in em_leave()
1936 		      stack_mask(ctxt));  in em_leave()
1937 	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);  in em_leave()
1940 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)  in em_push_sreg()  argument
1942 	int seg = ctxt->src2.val;  in em_push_sreg()
1944 	ctxt->src.val = get_segment_selector(ctxt, seg);  in em_push_sreg()
1945 	if (ctxt->op_bytes == 4) {  in em_push_sreg()
1946 		rsp_increment(ctxt, -2);  in em_push_sreg()
1947 		ctxt->op_bytes = 2;  in em_push_sreg()
1950 	return em_push(ctxt);  in em_push_sreg()
1953 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)  in em_pop_sreg()  argument
1955 	int seg = ctxt->src2.val;  in em_pop_sreg()
1959 	rc = emulate_pop(ctxt, &selector, 2);  in em_pop_sreg()
1964 		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;  in em_pop_sreg()
1965 	if (ctxt->op_bytes > 2)  in em_pop_sreg()
1966 		rsp_increment(ctxt, ctxt->op_bytes - 2);  in em_pop_sreg()
1968 	rc = load_segment_descriptor(ctxt, (u16)selector, seg);  in em_pop_sreg()
1972 static int em_pusha(struct x86_emulate_ctxt *ctxt)  in em_pusha()  argument
1974 	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);  in em_pusha()
1980 		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));  in em_pusha()
1982 		rc = em_push(ctxt);  in em_pusha()
1992 static int em_pushf(struct x86_emulate_ctxt *ctxt)  in em_pushf()  argument
1994 	ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;  in em_pushf()
1995 	return em_push(ctxt);  in em_pushf()
1998 static int em_popa(struct x86_emulate_ctxt *ctxt)  in em_popa()  argument
2006 			rsp_increment(ctxt, ctxt->op_bytes);  in em_popa()
2010 		rc = emulate_pop(ctxt, &val, ctxt->op_bytes);  in em_popa()
2013 		assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);  in em_popa()
2019 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)  in __emulate_int_real()  argument
2021 	const struct x86_emulate_ops *ops = ctxt->ops;  in __emulate_int_real()
2029 	ctxt->src.val = ctxt->eflags;  in __emulate_int_real()
2030 	rc = em_push(ctxt);  in __emulate_int_real()
2034 	ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);  in __emulate_int_real()
2036 	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);  in __emulate_int_real()
2037 	rc = em_push(ctxt);  in __emulate_int_real()
2041 	ctxt->src.val = ctxt->_eip;  in __emulate_int_real()
2042 	rc = em_push(ctxt);  in __emulate_int_real()
2046 	ops->get_idt(ctxt, &dt);  in __emulate_int_real()
2051 	rc = linear_read_system(ctxt, cs_addr, &cs, 2);  in __emulate_int_real()
2055 	rc = linear_read_system(ctxt, eip_addr, &eip, 2);  in __emulate_int_real()
2059 	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);  in __emulate_int_real()
2063 	ctxt->_eip = eip;  in __emulate_int_real()
2068 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)  in emulate_int_real()  argument
2072 	invalidate_registers(ctxt);  in emulate_int_real()
2073 	rc = __emulate_int_real(ctxt, irq);  in emulate_int_real()
2075 		writeback_registers(ctxt);  in emulate_int_real()
2079 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)  in emulate_int()  argument
2081 	switch(ctxt->mode) {  in emulate_int()
2083 		return __emulate_int_real(ctxt, irq);  in emulate_int()
2094 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)  in emulate_iret_real()  argument
2111 	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);  in emulate_iret_real()
2117 		return emulate_gp(ctxt, 0);  in emulate_iret_real()
2119 	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);  in emulate_iret_real()
2124 	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);  in emulate_iret_real()
2129 	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);  in emulate_iret_real()
2134 	ctxt->_eip = temp_eip;  in emulate_iret_real()
2136 	if (ctxt->op_bytes == 4)  in emulate_iret_real()
2137 		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));  in emulate_iret_real()
2138 	else if (ctxt->op_bytes == 2) {  in emulate_iret_real()
2139 		ctxt->eflags &= ~0xffff;  in emulate_iret_real()
2140 		ctxt->eflags |= temp_eflags;  in emulate_iret_real()
2143 	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */  in emulate_iret_real()
2144 	ctxt->eflags |= X86_EFLAGS_FIXED;  in emulate_iret_real()
2145 	ctxt->ops->set_nmi_mask(ctxt, false);  in emulate_iret_real()
2150 static int em_iret(struct x86_emulate_ctxt *ctxt)  in em_iret()  argument
2152 	switch(ctxt->mode) {  in em_iret()
2154 		return emulate_iret_real(ctxt);  in em_iret()
2165 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)  in em_jmp_far()  argument
2170 	u8 cpl = ctxt->ops->cpl(ctxt);  in em_jmp_far()
2172 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);  in em_jmp_far()
2174 	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,  in em_jmp_far()
2180 	rc = assign_eip_far(ctxt, ctxt->src.val);  in em_jmp_far()
2188 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)  in em_jmp_abs()  argument
2190 	return assign_eip_near(ctxt, ctxt->src.val);  in em_jmp_abs()
2193 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)  in em_call_near_abs()  argument
2198 	old_eip = ctxt->_eip;  in em_call_near_abs()
2199 	rc = assign_eip_near(ctxt, ctxt->src.val);  in em_call_near_abs()
2202 	ctxt->src.val = old_eip;  in em_call_near_abs()
2203 	rc = em_push(ctxt);  in em_call_near_abs()
2207 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)  in em_cmpxchg8b()  argument
2209 	u64 old = ctxt->dst.orig_val64;  in em_cmpxchg8b()
2211 	if (ctxt->dst.bytes == 16)  in em_cmpxchg8b()
2214 	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||  in em_cmpxchg8b()
2215 	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {  in em_cmpxchg8b()
2216 		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);  in em_cmpxchg8b()
2217 		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);  in em_cmpxchg8b()
2218 		ctxt->eflags &= ~X86_EFLAGS_ZF;  in em_cmpxchg8b()
2220 		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |  in em_cmpxchg8b()
2221 			(u32) reg_read(ctxt, VCPU_REGS_RBX);  in em_cmpxchg8b()
2223 		ctxt->eflags |= X86_EFLAGS_ZF;  in em_cmpxchg8b()
2228 static int em_ret(struct x86_emulate_ctxt *ctxt)  in em_ret()  argument
2233 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);  in em_ret()
2237 	return assign_eip_near(ctxt, eip);  in em_ret()
2240 static int em_ret_far(struct x86_emulate_ctxt *ctxt)  in em_ret_far()  argument
2244 	int cpl = ctxt->ops->cpl(ctxt);  in em_ret_far()
2247 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);  in em_ret_far()
2250 	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);  in em_ret_far()
2253 	rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,  in em_ret_far()
2258 	rc = assign_eip_far(ctxt, eip);  in em_ret_far()
2266 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)  in em_ret_far_imm()  argument
2270         rc = em_ret_far(ctxt);  in em_ret_far_imm()
2273         rsp_increment(ctxt, ctxt->src.val);  in em_ret_far_imm()
2277 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)  in em_cmpxchg()  argument
2280 	ctxt->dst.orig_val = ctxt->dst.val;  in em_cmpxchg()
2281 	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);  in em_cmpxchg()
2282 	ctxt->src.orig_val = ctxt->src.val;  in em_cmpxchg()
2283 	ctxt->src.val = ctxt->dst.orig_val;  in em_cmpxchg()
2284 	fastop(ctxt, em_cmp);  in em_cmpxchg()
2286 	if (ctxt->eflags & X86_EFLAGS_ZF) {  in em_cmpxchg()
2288 		ctxt->src.type = OP_NONE;  in em_cmpxchg()
2289 		ctxt->dst.val = ctxt->src.orig_val;  in em_cmpxchg()
2292 		ctxt->src.type = OP_REG;  in em_cmpxchg()
2293 		ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);  in em_cmpxchg()
2294 		ctxt->src.val = ctxt->dst.orig_val;  in em_cmpxchg()
2296 		ctxt->dst.val = ctxt->dst.orig_val;  in em_cmpxchg()
2301 static int em_lseg(struct x86_emulate_ctxt *ctxt)  in em_lseg()  argument
2303 	int seg = ctxt->src2.val;  in em_lseg()
2307 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);  in em_lseg()
2309 	rc = load_segment_descriptor(ctxt, sel, seg);  in em_lseg()
2313 	ctxt->dst.val = ctxt->src.val;  in em_lseg()
2317 static int em_rsm(struct x86_emulate_ctxt *ctxt)  in em_rsm()  argument
2319 	if (!ctxt->ops->is_smm(ctxt))  in em_rsm()
2320 		return emulate_ud(ctxt);  in em_rsm()
2322 	if (ctxt->ops->leave_smm(ctxt))  in em_rsm()
2323 		ctxt->ops->triple_fault(ctxt);  in em_rsm()
2325 	return emulator_recalc_and_set_mode(ctxt);  in em_rsm()
2354 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)  in vendor_intel()  argument
2359 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);  in vendor_intel()
2363 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)  in em_syscall_is_enabled()  argument
2365 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_syscall_is_enabled()
2372 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_syscall_is_enabled()
2377 	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);  in em_syscall_is_enabled()
2398 static int em_syscall(struct x86_emulate_ctxt *ctxt)  in em_syscall()  argument
2400 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_syscall()
2407 	if (ctxt->mode == X86EMUL_MODE_REAL ||  in em_syscall()
2408 	    ctxt->mode == X86EMUL_MODE_VM86)  in em_syscall()
2409 		return emulate_ud(ctxt);  in em_syscall()
2411 	if (!(em_syscall_is_enabled(ctxt)))  in em_syscall()
2412 		return emulate_ud(ctxt);  in em_syscall()
2414 	ops->get_msr(ctxt, MSR_EFER, &efer);  in em_syscall()
2416 		return emulate_ud(ctxt);  in em_syscall()
2419 	ops->get_msr(ctxt, MSR_STAR, &msr_data);  in em_syscall()
2428 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);  in em_syscall()
2429 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);  in em_syscall()
2431 	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;  in em_syscall()
2434 		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;  in em_syscall()
2436 		ops->get_msr(ctxt,  in em_syscall()
2437 			     ctxt->mode == X86EMUL_MODE_PROT64 ?  in em_syscall()
2439 		ctxt->_eip = msr_data;  in em_syscall()
2441 		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);  in em_syscall()
2442 		ctxt->eflags &= ~msr_data;  in em_syscall()
2443 		ctxt->eflags |= X86_EFLAGS_FIXED;  in em_syscall()
2447 		ops->get_msr(ctxt, MSR_STAR, &msr_data);  in em_syscall()
2448 		ctxt->_eip = (u32)msr_data;  in em_syscall()
2450 		ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);  in em_syscall()
2453 	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;  in em_syscall()
2457 static int em_sysenter(struct x86_emulate_ctxt *ctxt)  in em_sysenter()  argument
2459 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_sysenter()
2465 	ops->get_msr(ctxt, MSR_EFER, &efer);  in em_sysenter()
2467 	if (ctxt->mode == X86EMUL_MODE_REAL)  in em_sysenter()
2468 		return emulate_gp(ctxt, 0);  in em_sysenter()
2474 	if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)  in em_sysenter()
2475 	    && !vendor_intel(ctxt))  in em_sysenter()
2476 		return emulate_ud(ctxt);  in em_sysenter()
2479 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_sysenter()
2482 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);  in em_sysenter()
2484 		return emulate_gp(ctxt, 0);  in em_sysenter()
2487 	ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);  in em_sysenter()
2495 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);  in em_sysenter()
2496 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);  in em_sysenter()
2498 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);  in em_sysenter()
2499 	ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;  in em_sysenter()
2501 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);  in em_sysenter()
2502 	*reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :  in em_sysenter()
2505 		ctxt->mode = X86EMUL_MODE_PROT64;  in em_sysenter()
2510 static int em_sysexit(struct x86_emulate_ctxt *ctxt)  in em_sysexit()  argument
2512 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_sysexit()
2519 	if (ctxt->mode == X86EMUL_MODE_REAL ||  in em_sysexit()
2520 	    ctxt->mode == X86EMUL_MODE_VM86)  in em_sysexit()
2521 		return emulate_gp(ctxt, 0);  in em_sysexit()
2525 	if ((ctxt->rex_prefix & 0x8) != 0x0)  in em_sysexit()
2530 	rcx = reg_read(ctxt, VCPU_REGS_RCX);  in em_sysexit()
2531 	rdx = reg_read(ctxt, VCPU_REGS_RDX);  in em_sysexit()
2535 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);  in em_sysexit()
2540 			return emulate_gp(ctxt, 0);  in em_sysexit()
2548 			return emulate_gp(ctxt, 0);  in em_sysexit()
2552 		if (emul_is_noncanonical_address(rcx, ctxt) ||  in em_sysexit()
2553 		    emul_is_noncanonical_address(rdx, ctxt))  in em_sysexit()
2554 			return emulate_gp(ctxt, 0);  in em_sysexit()
2560 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);  in em_sysexit()
2561 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);  in em_sysexit()
2563 	ctxt->_eip = rdx;  in em_sysexit()
2564 	ctxt->mode = usermode;  in em_sysexit()
2565 	*reg_write(ctxt, VCPU_REGS_RSP) = rcx;  in em_sysexit()
2570 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)  in emulator_bad_iopl()  argument
2573 	if (ctxt->mode == X86EMUL_MODE_REAL)  in emulator_bad_iopl()
2575 	if (ctxt->mode == X86EMUL_MODE_VM86)  in emulator_bad_iopl()
2577 	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;  in emulator_bad_iopl()
2578 	return ctxt->ops->cpl(ctxt) > iopl;  in emulator_bad_iopl()
2584 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,  in emulator_io_port_access_allowed()  argument
2587 	const struct x86_emulate_ops *ops = ctxt->ops;  in emulator_io_port_access_allowed()
2603 	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);  in emulator_io_port_access_allowed()
2612 	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);  in emulator_io_port_access_allowed()
2617 	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);  in emulator_io_port_access_allowed()
2625 static bool emulator_io_permitted(struct x86_emulate_ctxt *ctxt,  in emulator_io_permitted()  argument
2628 	if (ctxt->perm_ok)  in emulator_io_permitted()
2631 	if (emulator_bad_iopl(ctxt))  in emulator_io_permitted()
2632 		if (!emulator_io_port_access_allowed(ctxt, port, len))  in emulator_io_permitted()
2635 	ctxt->perm_ok = true;  in emulator_io_permitted()
2640 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)  in string_registers_quirk()  argument
2647 	if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))  in string_registers_quirk()
2650 	*reg_write(ctxt, VCPU_REGS_RCX) = 0;  in string_registers_quirk()
2652 	switch (ctxt->b) {  in string_registers_quirk()
2655 		*reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;  in string_registers_quirk()
2659 		*reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;  in string_registers_quirk()
2664 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,  in save_state_to_tss16()  argument
2667 	tss->ip = ctxt->_eip;  in save_state_to_tss16()
2668 	tss->flag = ctxt->eflags;  in save_state_to_tss16()
2669 	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);  in save_state_to_tss16()
2670 	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);  in save_state_to_tss16()
2671 	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);  in save_state_to_tss16()
2672 	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);  in save_state_to_tss16()
2673 	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);  in save_state_to_tss16()
2674 	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);  in save_state_to_tss16()
2675 	tss->si = reg_read(ctxt, VCPU_REGS_RSI);  in save_state_to_tss16()
2676 	tss->di = reg_read(ctxt, VCPU_REGS_RDI);  in save_state_to_tss16()
2678 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);  in save_state_to_tss16()
2679 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);  in save_state_to_tss16()
2680 	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);  in save_state_to_tss16()
2681 	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);  in save_state_to_tss16()
2682 	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);  in save_state_to_tss16()
2685 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,  in load_state_from_tss16()  argument
2691 	ctxt->_eip = tss->ip;  in load_state_from_tss16()
2692 	ctxt->eflags = tss->flag | 2;  in load_state_from_tss16()
2693 	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;  in load_state_from_tss16()
2694 	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;  in load_state_from_tss16()
2695 	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;  in load_state_from_tss16()
2696 	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;  in load_state_from_tss16()
2697 	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;  in load_state_from_tss16()
2698 	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;  in load_state_from_tss16()
2699 	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;  in load_state_from_tss16()
2700 	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;  in load_state_from_tss16()
2706 	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);  in load_state_from_tss16()
2707 	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);  in load_state_from_tss16()
2708 	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);  in load_state_from_tss16()
2709 	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);  in load_state_from_tss16()
2710 	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);  in load_state_from_tss16()
2718 	ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,  in load_state_from_tss16()
2722 	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,  in load_state_from_tss16()
2726 	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,  in load_state_from_tss16()
2730 	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,  in load_state_from_tss16()
2734 	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,  in load_state_from_tss16()
2742 static int task_switch_16(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,  in task_switch_16()  argument
2749 	ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_16()
2753 	save_state_to_tss16(ctxt, &tss_seg);  in task_switch_16()
2755 	ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_16()
2759 	ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_16()
2766 		ret = linear_write_system(ctxt, new_tss_base,  in task_switch_16()
2773 	return load_state_from_tss16(ctxt, &tss_seg);  in task_switch_16()
2776 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,  in save_state_to_tss32()  argument
2780 	tss->eip = ctxt->_eip;  in save_state_to_tss32()
2781 	tss->eflags = ctxt->eflags;  in save_state_to_tss32()
2782 	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);  in save_state_to_tss32()
2783 	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);  in save_state_to_tss32()
2784 	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);  in save_state_to_tss32()
2785 	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);  in save_state_to_tss32()
2786 	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);  in save_state_to_tss32()
2787 	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);  in save_state_to_tss32()
2788 	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);  in save_state_to_tss32()
2789 	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);  in save_state_to_tss32()
2791 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);  in save_state_to_tss32()
2792 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);  in save_state_to_tss32()
2793 	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);  in save_state_to_tss32()
2794 	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);  in save_state_to_tss32()
2795 	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);  in save_state_to_tss32()
2796 	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);  in save_state_to_tss32()
2799 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,  in load_state_from_tss32()  argument
2805 	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))  in load_state_from_tss32()
2806 		return emulate_gp(ctxt, 0);  in load_state_from_tss32()
2807 	ctxt->_eip = tss->eip;  in load_state_from_tss32()
2808 	ctxt->eflags = tss->eflags | 2;  in load_state_from_tss32()
2811 	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;  in load_state_from_tss32()
2812 	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;  in load_state_from_tss32()
2813 	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;  in load_state_from_tss32()
2814 	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;  in load_state_from_tss32()
2815 	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;  in load_state_from_tss32()
2816 	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;  in load_state_from_tss32()
2817 	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;  in load_state_from_tss32()
2818 	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;  in load_state_from_tss32()
2825 	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);  in load_state_from_tss32()
2826 	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);  in load_state_from_tss32()
2827 	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);  in load_state_from_tss32()
2828 	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);  in load_state_from_tss32()
2829 	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);  in load_state_from_tss32()
2830 	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);  in load_state_from_tss32()
2831 	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);  in load_state_from_tss32()
2838 	if (ctxt->eflags & X86_EFLAGS_VM) {  in load_state_from_tss32()
2839 		ctxt->mode = X86EMUL_MODE_VM86;  in load_state_from_tss32()
2842 		ctxt->mode = X86EMUL_MODE_PROT32;  in load_state_from_tss32()
2850 	ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,  in load_state_from_tss32()
2854 	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,  in load_state_from_tss32()
2858 	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,  in load_state_from_tss32()
2862 	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,  in load_state_from_tss32()
2866 	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,  in load_state_from_tss32()
2870 	ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,  in load_state_from_tss32()
2874 	ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,  in load_state_from_tss32()
2880 static int task_switch_32(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,  in task_switch_32()  argument
2889 	ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_32()
2893 	save_state_to_tss32(ctxt, &tss_seg);  in task_switch_32()
2896 	ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,  in task_switch_32()
2901 	ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));  in task_switch_32()
2908 		ret = linear_write_system(ctxt, new_tss_base,  in task_switch_32()
2915 	return load_state_from_tss32(ctxt, &tss_seg);  in task_switch_32()
2918 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,  in emulator_do_task_switch()  argument
2922 	const struct x86_emulate_ops *ops = ctxt->ops;  in emulator_do_task_switch()
2925 	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);  in emulator_do_task_switch()
2927 		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);  in emulator_do_task_switch()
2933 	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);  in emulator_do_task_switch()
2936 	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);  in emulator_do_task_switch()
2956 			ret = read_interrupt_descriptor(ctxt, idt_index,  in emulator_do_task_switch()
2962 			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)  in emulator_do_task_switch()
2963 				return emulate_gp(ctxt, (idt_index << 3) | 0x2);  in emulator_do_task_switch()
2971 		return emulate_ts(ctxt, tss_selector & 0xfffc);  in emulator_do_task_switch()
2976 		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);  in emulator_do_task_switch()
2980 		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;  in emulator_do_task_switch()
2988 		ret = task_switch_32(ctxt, old_tss_sel, old_tss_base, &next_tss_desc);  in emulator_do_task_switch()
2990 		ret = task_switch_16(ctxt, old_tss_sel,  in emulator_do_task_switch()
2996 		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;  in emulator_do_task_switch()
3000 		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);  in emulator_do_task_switch()
3003 	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);  in emulator_do_task_switch()
3004 	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);  in emulator_do_task_switch()
3007 		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;  in emulator_do_task_switch()
3008 		ctxt->lock_prefix = 0;  in emulator_do_task_switch()
3009 		ctxt->src.val = (unsigned long) error_code;  in emulator_do_task_switch()
3010 		ret = em_push(ctxt);  in emulator_do_task_switch()
3013 	ops->get_dr(ctxt, 7, &dr7);  in emulator_do_task_switch()
3014 	ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));  in emulator_do_task_switch()
3019 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,  in emulator_task_switch()  argument
3025 	invalidate_registers(ctxt);  in emulator_task_switch()
3026 	ctxt->_eip = ctxt->eip;  in emulator_task_switch()
3027 	ctxt->dst.type = OP_NONE;  in emulator_task_switch()
3029 	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,  in emulator_task_switch()
3033 		ctxt->eip = ctxt->_eip;  in emulator_task_switch()
3034 		writeback_registers(ctxt);  in emulator_task_switch()
3040 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,  in string_addr_inc()  argument
3043 	int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;  in string_addr_inc()
3045 	register_address_increment(ctxt, reg, df * op->bytes);  in string_addr_inc()
3046 	op->addr.mem.ea = register_address(ctxt, reg);  in string_addr_inc()
3049 static int em_das(struct x86_emulate_ctxt *ctxt)  in em_das()  argument
3054 	cf = ctxt->eflags & X86_EFLAGS_CF;  in em_das()
3055 	al = ctxt->dst.val;  in em_das()
3060 	af = ctxt->eflags & X86_EFLAGS_AF;  in em_das()
3073 	ctxt->dst.val = al;  in em_das()
3075 	ctxt->src.type = OP_IMM;  in em_das()
3076 	ctxt->src.val = 0;  in em_das()
3077 	ctxt->src.bytes = 1;  in em_das()
3078 	fastop(ctxt, em_or);  in em_das()
3079 	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);  in em_das()
3081 		ctxt->eflags |= X86_EFLAGS_CF;  in em_das()
3083 		ctxt->eflags |= X86_EFLAGS_AF;  in em_das()
3087 static int em_aam(struct x86_emulate_ctxt *ctxt)  in em_aam()  argument
3091 	if (ctxt->src.val == 0)  in em_aam()
3092 		return emulate_de(ctxt);  in em_aam()
3094 	al = ctxt->dst.val & 0xff;  in em_aam()
3095 	ah = al / ctxt->src.val;  in em_aam()
3096 	al %= ctxt->src.val;  in em_aam()
3098 	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);  in em_aam()
3101 	ctxt->src.type = OP_IMM;  in em_aam()
3102 	ctxt->src.val = 0;  in em_aam()
3103 	ctxt->src.bytes = 1;  in em_aam()
3104 	fastop(ctxt, em_or);  in em_aam()
3109 static int em_aad(struct x86_emulate_ctxt *ctxt)  in em_aad()  argument
3111 	u8 al = ctxt->dst.val & 0xff;  in em_aad()
3112 	u8 ah = (ctxt->dst.val >> 8) & 0xff;  in em_aad()
3114 	al = (al + (ah * ctxt->src.val)) & 0xff;  in em_aad()
3116 	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;  in em_aad()
3119 	ctxt->src.type = OP_IMM;  in em_aad()
3120 	ctxt->src.val = 0;  in em_aad()
3121 	ctxt->src.bytes = 1;  in em_aad()
3122 	fastop(ctxt, em_or);  in em_aad()
3127 static int em_call(struct x86_emulate_ctxt *ctxt)  in em_call()  argument
3130 	long rel = ctxt->src.val;  in em_call()
3132 	ctxt->src.val = (unsigned long)ctxt->_eip;  in em_call()
3133 	rc = jmp_rel(ctxt, rel);  in em_call()
3136 	return em_push(ctxt);  in em_call()
3139 static int em_call_far(struct x86_emulate_ctxt *ctxt)  in em_call_far()  argument
3145 	const struct x86_emulate_ops *ops = ctxt->ops;  in em_call_far()
3146 	int cpl = ctxt->ops->cpl(ctxt);  in em_call_far()
3147 	enum x86emul_mode prev_mode = ctxt->mode;  in em_call_far()
3149 	old_eip = ctxt->_eip;  in em_call_far()
3150 	ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);  in em_call_far()
3152 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);  in em_call_far()
3153 	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,  in em_call_far()
3158 	rc = assign_eip_far(ctxt, ctxt->src.val);  in em_call_far()
3162 	ctxt->src.val = old_cs;  in em_call_far()
3163 	rc = em_push(ctxt);  in em_call_far()
3167 	ctxt->src.val = old_eip;  in em_call_far()
3168 	rc = em_push(ctxt);  in em_call_far()
3177 	ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);  in em_call_far()
3178 	ctxt->mode = prev_mode;  in em_call_far()
3183 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)  in em_ret_near_imm()  argument
3188 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);  in em_ret_near_imm()
3191 	rc = assign_eip_near(ctxt, eip);  in em_ret_near_imm()
3194 	rsp_increment(ctxt, ctxt->src.val);  in em_ret_near_imm()
3198 static int em_xchg(struct x86_emulate_ctxt *ctxt)  in em_xchg()  argument
3201 	ctxt->src.val = ctxt->dst.val;  in em_xchg()
3202 	write_register_operand(&ctxt->src);  in em_xchg()
3205 	ctxt->dst.val = ctxt->src.orig_val;  in em_xchg()
3206 	ctxt->lock_prefix = 1;  in em_xchg()
3210 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)  in em_imul_3op()  argument
3212 	ctxt->dst.val = ctxt->src2.val;  in em_imul_3op()
3213 	return fastop(ctxt, em_imul);  in em_imul_3op()
3216 static int em_cwd(struct x86_emulate_ctxt *ctxt)  in em_cwd()  argument
3218 	ctxt->dst.type = OP_REG;  in em_cwd()
3219 	ctxt->dst.bytes = ctxt->src.bytes;  in em_cwd()
3220 	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);  in em_cwd()
3221 	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);  in em_cwd()
3226 static int em_rdpid(struct x86_emulate_ctxt *ctxt)  in em_rdpid()  argument
3230 	if (!ctxt->ops->guest_has_rdpid(ctxt))  in em_rdpid()
3231 		return emulate_ud(ctxt);  in em_rdpid()
3233 	ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux);  in em_rdpid()
3234 	ctxt->dst.val = tsc_aux;  in em_rdpid()
3238 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)  in em_rdtsc()  argument
3242 	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);  in em_rdtsc()
3243 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;  in em_rdtsc()
3244 	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;  in em_rdtsc()
3248 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)  in em_rdpmc()  argument
3252 	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))  in em_rdpmc()
3253 		return emulate_gp(ctxt, 0);  in em_rdpmc()
3254 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;  in em_rdpmc()
3255 	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;  in em_rdpmc()
3259 static int em_mov(struct x86_emulate_ctxt *ctxt)  in em_mov()  argument
3261 	memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));  in em_mov()
3265 static int em_movbe(struct x86_emulate_ctxt *ctxt)  in em_movbe()  argument
3269 	if (!ctxt->ops->guest_has_movbe(ctxt))  in em_movbe()
3270 		return emulate_ud(ctxt);  in em_movbe()
3272 	switch (ctxt->op_bytes) {  in em_movbe()
3282 		tmp = (u16)ctxt->src.val;  in em_movbe()
3283 		ctxt->dst.val &= ~0xffffUL;  in em_movbe()
3284 		ctxt->dst.val |= (unsigned long)swab16(tmp);  in em_movbe()
3287 		ctxt->dst.val = swab32((u32)ctxt->src.val);  in em_movbe()
3290 		ctxt->dst.val = swab64(ctxt->src.val);  in em_movbe()
3298 static int em_cr_write(struct x86_emulate_ctxt *ctxt)  in em_cr_write()  argument
3300 	int cr_num = ctxt->modrm_reg;  in em_cr_write()
3303 	if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val))  in em_cr_write()
3304 		return emulate_gp(ctxt, 0);  in em_cr_write()
3307 	ctxt->dst.type = OP_NONE;  in em_cr_write()
3314 		r = emulator_recalc_and_set_mode(ctxt);  in em_cr_write()
3322 static int em_dr_write(struct x86_emulate_ctxt *ctxt)  in em_dr_write()  argument
3326 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_dr_write()
3327 		val = ctxt->src.val & ~0ULL;  in em_dr_write()
3329 		val = ctxt->src.val & ~0U;  in em_dr_write()
3332 	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)  in em_dr_write()
3333 		return emulate_gp(ctxt, 0);  in em_dr_write()
3336 	ctxt->dst.type = OP_NONE;  in em_dr_write()
3340 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)  in em_wrmsr()  argument
3342 	u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);  in em_wrmsr()
3346 	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)  in em_wrmsr()
3347 		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);  in em_wrmsr()
3348 	r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data);  in em_wrmsr()
3351 		return emulate_gp(ctxt, 0);  in em_wrmsr()
3356 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)  in em_rdmsr()  argument
3358 	u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);  in em_rdmsr()
3362 	r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data);  in em_rdmsr()
3365 		return emulate_gp(ctxt, 0);  in em_rdmsr()
3368 		*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;  in em_rdmsr()
3369 		*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;  in em_rdmsr()
3374 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)  in em_store_sreg()  argument
3377 	    (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&  in em_store_sreg()
3378 	    ctxt->ops->cpl(ctxt) > 0)  in em_store_sreg()
3379 		return emulate_gp(ctxt, 0);  in em_store_sreg()
3381 	ctxt->dst.val = get_segment_selector(ctxt, segment);  in em_store_sreg()
3382 	if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)  in em_store_sreg()
3383 		ctxt->dst.bytes = 2;  in em_store_sreg()
3387 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)  in em_mov_rm_sreg()  argument
3389 	if (ctxt->modrm_reg > VCPU_SREG_GS)  in em_mov_rm_sreg()
3390 		return emulate_ud(ctxt);  in em_mov_rm_sreg()
3392 	return em_store_sreg(ctxt, ctxt->modrm_reg);  in em_mov_rm_sreg()
3395 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)  in em_mov_sreg_rm()  argument
3397 	u16 sel = ctxt->src.val;  in em_mov_sreg_rm()
3399 	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)  in em_mov_sreg_rm()
3400 		return emulate_ud(ctxt);  in em_mov_sreg_rm()
3402 	if (ctxt->modrm_reg == VCPU_SREG_SS)  in em_mov_sreg_rm()
3403 		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;  in em_mov_sreg_rm()
3406 	ctxt->dst.type = OP_NONE;  in em_mov_sreg_rm()
3407 	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);  in em_mov_sreg_rm()
3410 static int em_sldt(struct x86_emulate_ctxt *ctxt)  in em_sldt()  argument
3412 	return em_store_sreg(ctxt, VCPU_SREG_LDTR);  in em_sldt()
3415 static int em_lldt(struct x86_emulate_ctxt *ctxt)  in em_lldt()  argument
3417 	u16 sel = ctxt->src.val;  in em_lldt()
3420 	ctxt->dst.type = OP_NONE;  in em_lldt()
3421 	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);  in em_lldt()
3424 static int em_str(struct x86_emulate_ctxt *ctxt)  in em_str()  argument
3426 	return em_store_sreg(ctxt, VCPU_SREG_TR);  in em_str()
3429 static int em_ltr(struct x86_emulate_ctxt *ctxt)  in em_ltr()  argument
3431 	u16 sel = ctxt->src.val;  in em_ltr()
3434 	ctxt->dst.type = OP_NONE;  in em_ltr()
3435 	return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);  in em_ltr()
3438 static int em_invlpg(struct x86_emulate_ctxt *ctxt)  in em_invlpg()  argument
3443 	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);  in em_invlpg()
3445 		ctxt->ops->invlpg(ctxt, linear);  in em_invlpg()
3447 	ctxt->dst.type = OP_NONE;  in em_invlpg()
3451 static int em_clts(struct x86_emulate_ctxt *ctxt)  in em_clts()  argument
3455 	cr0 = ctxt->ops->get_cr(ctxt, 0);  in em_clts()
3457 	ctxt->ops->set_cr(ctxt, 0, cr0);  in em_clts()
3461 static int em_hypercall(struct x86_emulate_ctxt *ctxt)  in em_hypercall()  argument
3463 	int rc = ctxt->ops->fix_hypercall(ctxt);  in em_hypercall()
3469 	ctxt->_eip = ctxt->eip;  in em_hypercall()
3471 	ctxt->dst.type = OP_NONE;  in em_hypercall()
3475 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,  in emulate_store_desc_ptr()  argument
3476 				  void (*get)(struct x86_emulate_ctxt *ctxt,  in emulate_store_desc_ptr()  argument
3481 	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&  in emulate_store_desc_ptr()
3482 	    ctxt->ops->cpl(ctxt) > 0)  in emulate_store_desc_ptr()
3483 		return emulate_gp(ctxt, 0);  in emulate_store_desc_ptr()
3485 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in emulate_store_desc_ptr()
3486 		ctxt->op_bytes = 8;  in emulate_store_desc_ptr()
3487 	get(ctxt, &desc_ptr);  in emulate_store_desc_ptr()
3488 	if (ctxt->op_bytes == 2) {  in emulate_store_desc_ptr()
3489 		ctxt->op_bytes = 4;  in emulate_store_desc_ptr()
3493 	ctxt->dst.type = OP_NONE;  in emulate_store_desc_ptr()
3494 	return segmented_write_std(ctxt, ctxt->dst.addr.mem,  in emulate_store_desc_ptr()
3495 				   &desc_ptr, 2 + ctxt->op_bytes);  in emulate_store_desc_ptr()
3498 static int em_sgdt(struct x86_emulate_ctxt *ctxt)  in em_sgdt()  argument
3500 	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);  in em_sgdt()
3503 static int em_sidt(struct x86_emulate_ctxt *ctxt)  in em_sidt()  argument
3505 	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);  in em_sidt()
3508 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)  in em_lgdt_lidt()  argument
3513 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in em_lgdt_lidt()
3514 		ctxt->op_bytes = 8;  in em_lgdt_lidt()
3515 	rc = read_descriptor(ctxt, ctxt->src.addr.mem,  in em_lgdt_lidt()
3517 			     ctxt->op_bytes);  in em_lgdt_lidt()
3520 	if (ctxt->mode == X86EMUL_MODE_PROT64 &&  in em_lgdt_lidt()
3521 	    emul_is_noncanonical_address(desc_ptr.address, ctxt))  in em_lgdt_lidt()
3522 		return emulate_gp(ctxt, 0);  in em_lgdt_lidt()
3524 		ctxt->ops->set_gdt(ctxt, &desc_ptr);  in em_lgdt_lidt()
3526 		ctxt->ops->set_idt(ctxt, &desc_ptr);  in em_lgdt_lidt()
3528 	ctxt->dst.type = OP_NONE;  in em_lgdt_lidt()
3532 static int em_lgdt(struct x86_emulate_ctxt *ctxt)  in em_lgdt()  argument
3534 	return em_lgdt_lidt(ctxt, true);  in em_lgdt()
3537 static int em_lidt(struct x86_emulate_ctxt *ctxt)  in em_lidt()  argument
3539 	return em_lgdt_lidt(ctxt, false);  in em_lidt()
3542 static int em_smsw(struct x86_emulate_ctxt *ctxt)  in em_smsw()  argument
3544 	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&  in em_smsw()
3545 	    ctxt->ops->cpl(ctxt) > 0)  in em_smsw()
3546 		return emulate_gp(ctxt, 0);  in em_smsw()
3548 	if (ctxt->dst.type == OP_MEM)  in em_smsw()
3549 		ctxt->dst.bytes = 2;  in em_smsw()
3550 	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);  in em_smsw()
3554 static int em_lmsw(struct x86_emulate_ctxt *ctxt)  in em_lmsw()  argument
3556 	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)  in em_lmsw()
3557 			  | (ctxt->src.val & 0x0f));  in em_lmsw()
3558 	ctxt->dst.type = OP_NONE;  in em_lmsw()
3562 static int em_loop(struct x86_emulate_ctxt *ctxt)  in em_loop()  argument
3566 	register_address_increment(ctxt, VCPU_REGS_RCX, -1);  in em_loop()
3567 	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&  in em_loop()
3568 	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))  in em_loop()
3569 		rc = jmp_rel(ctxt, ctxt->src.val);  in em_loop()
3574 static int em_jcxz(struct x86_emulate_ctxt *ctxt)  in em_jcxz()  argument
3578 	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)  in em_jcxz()
3579 		rc = jmp_rel(ctxt, ctxt->src.val);  in em_jcxz()
3584 static int em_in(struct x86_emulate_ctxt *ctxt)  in em_in()  argument
3586 	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,  in em_in()
3587 			     &ctxt->dst.val))  in em_in()
3593 static int em_out(struct x86_emulate_ctxt *ctxt)  in em_out()  argument
3595 	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,  in em_out()
3596 				    &ctxt->src.val, 1);  in em_out()
3598 	ctxt->dst.type = OP_NONE;  in em_out()
3602 static int em_cli(struct x86_emulate_ctxt *ctxt)  in em_cli()  argument
3604 	if (emulator_bad_iopl(ctxt))  in em_cli()
3605 		return emulate_gp(ctxt, 0);  in em_cli()
3607 	ctxt->eflags &= ~X86_EFLAGS_IF;  in em_cli()
3611 static int em_sti(struct x86_emulate_ctxt *ctxt)  in em_sti()  argument
3613 	if (emulator_bad_iopl(ctxt))  in em_sti()
3614 		return emulate_gp(ctxt, 0);  in em_sti()
3616 	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;  in em_sti()
3617 	ctxt->eflags |= X86_EFLAGS_IF;  in em_sti()
3621 static int em_cpuid(struct x86_emulate_ctxt *ctxt)  in em_cpuid()  argument
3626 	ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);  in em_cpuid()
3628 	    ctxt->ops->cpl(ctxt)) {  in em_cpuid()
3629 		return emulate_gp(ctxt, 0);  in em_cpuid()
3632 	eax = reg_read(ctxt, VCPU_REGS_RAX);  in em_cpuid()
3633 	ecx = reg_read(ctxt, VCPU_REGS_RCX);  in em_cpuid()
3634 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);  in em_cpuid()
3635 	*reg_write(ctxt, VCPU_REGS_RAX) = eax;  in em_cpuid()
3636 	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;  in em_cpuid()
3637 	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;  in em_cpuid()
3638 	*reg_write(ctxt, VCPU_REGS_RDX) = edx;  in em_cpuid()
3642 static int em_sahf(struct x86_emulate_ctxt *ctxt)  in em_sahf()  argument
3648 	flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;  in em_sahf()
3650 	ctxt->eflags &= ~0xffUL;  in em_sahf()
3651 	ctxt->eflags |= flags | X86_EFLAGS_FIXED;  in em_sahf()
3655 static int em_lahf(struct x86_emulate_ctxt *ctxt)  in em_lahf()  argument
3657 	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;  in em_lahf()
3658 	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;  in em_lahf()
3662 static int em_bswap(struct x86_emulate_ctxt *ctxt)  in em_bswap()  argument
3664 	switch (ctxt->op_bytes) {  in em_bswap()
3667 		asm("bswap %0" : "+r"(ctxt->dst.val));  in em_bswap()
3671 		asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));  in em_bswap()
3677 static int em_clflush(struct x86_emulate_ctxt *ctxt)  in em_clflush()  argument
3683 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)  in em_clflushopt()  argument
3689 static int em_movsxd(struct x86_emulate_ctxt *ctxt)  in em_movsxd()  argument
3691 	ctxt->dst.val = (s32) ctxt->src.val;  in em_movsxd()
3695 static int check_fxsr(struct x86_emulate_ctxt *ctxt)  in check_fxsr()  argument
3697 	if (!ctxt->ops->guest_has_fxsr(ctxt))  in check_fxsr()
3698 		return emulate_ud(ctxt);  in check_fxsr()
3700 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))  in check_fxsr()
3701 		return emulate_nm(ctxt);  in check_fxsr()
3707 	if (ctxt->mode >= X86EMUL_MODE_PROT64)  in check_fxsr()
3722 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)  in fxstate_size()  argument
3725 	if (ctxt->mode == X86EMUL_MODE_PROT64)  in fxstate_size()
3728 	cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;  in fxstate_size()
3750 static int em_fxsave(struct x86_emulate_ctxt *ctxt)  in em_fxsave()  argument
3755 	rc = check_fxsr(ctxt);  in em_fxsave()
3768 	return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,  in em_fxsave()
3769 		                   fxstate_size(ctxt));  in em_fxsave()
3792 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)  in em_fxrstor()  argument
3798 	rc = check_fxsr(ctxt);  in em_fxrstor()
3802 	size = fxstate_size(ctxt);  in em_fxrstor()
3803 	rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);  in em_fxrstor()
3816 		rc = emulate_gp(ctxt, 0);  in em_fxrstor()
3829 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)  in em_xsetbv()  argument
3833 	if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSXSAVE))  in em_xsetbv()
3834 		return emulate_ud(ctxt);  in em_xsetbv()
3836 	eax = reg_read(ctxt, VCPU_REGS_RAX);  in em_xsetbv()
3837 	edx = reg_read(ctxt, VCPU_REGS_RDX);  in em_xsetbv()
3838 	ecx = reg_read(ctxt, VCPU_REGS_RCX);  in em_xsetbv()
3840 	if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))  in em_xsetbv()
3841 		return emulate_gp(ctxt, 0);  in em_xsetbv()
3858 static int check_cr_access(struct x86_emulate_ctxt *ctxt)  in check_cr_access()  argument
3860 	if (!valid_cr(ctxt->modrm_reg))  in check_cr_access()
3861 		return emulate_ud(ctxt);  in check_cr_access()
3866 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)  in check_dr7_gd()  argument
3870 	ctxt->ops->get_dr(ctxt, 7, &dr7);  in check_dr7_gd()
3875 static int check_dr_read(struct x86_emulate_ctxt *ctxt)  in check_dr_read()  argument
3877 	int dr = ctxt->modrm_reg;  in check_dr_read()
3881 		return emulate_ud(ctxt);  in check_dr_read()
3883 	cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_dr_read()
3885 		return emulate_ud(ctxt);  in check_dr_read()
3887 	if (check_dr7_gd(ctxt)) {  in check_dr_read()
3890 		ctxt->ops->get_dr(ctxt, 6, &dr6);  in check_dr_read()
3893 		ctxt->ops->set_dr(ctxt, 6, dr6);  in check_dr_read()
3894 		return emulate_db(ctxt);  in check_dr_read()
3900 static int check_dr_write(struct x86_emulate_ctxt *ctxt)  in check_dr_write()  argument
3902 	u64 new_val = ctxt->src.val64;  in check_dr_write()
3903 	int dr = ctxt->modrm_reg;  in check_dr_write()
3906 		return emulate_gp(ctxt, 0);  in check_dr_write()
3908 	return check_dr_read(ctxt);  in check_dr_write()
3911 static int check_svme(struct x86_emulate_ctxt *ctxt)  in check_svme()  argument
3915 	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);  in check_svme()
3918 		return emulate_ud(ctxt);  in check_svme()
3923 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)  in check_svme_pa()  argument
3925 	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);  in check_svme_pa()
3929 		return emulate_gp(ctxt, 0);  in check_svme_pa()
3931 	return check_svme(ctxt);  in check_svme_pa()
3934 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)  in check_rdtsc()  argument
3936 	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_rdtsc()
3938 	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))  in check_rdtsc()
3939 		return emulate_gp(ctxt, 0);  in check_rdtsc()
3944 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)  in check_rdpmc()  argument
3946 	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);  in check_rdpmc()
3947 	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);  in check_rdpmc()
3961 	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||  in check_rdpmc()
3962 	    ctxt->ops->check_pmc(ctxt, rcx))  in check_rdpmc()
3963 		return emulate_gp(ctxt, 0);  in check_rdpmc()
3968 static int check_perm_in(struct x86_emulate_ctxt *ctxt)  in check_perm_in()  argument
3970 	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);  in check_perm_in()
3971 	if (!emulator_io_permitted(ctxt, ctxt->src.val, ctxt->dst.bytes))  in check_perm_in()
3972 		return emulate_gp(ctxt, 0);  in check_perm_in()
3977 static int check_perm_out(struct x86_emulate_ctxt *ctxt)  in check_perm_out()  argument
3979 	ctxt->src.bytes = min(ctxt->src.bytes, 4u);  in check_perm_out()
3980 	if (!emulator_io_permitted(ctxt, ctxt->dst.val, ctxt->src.bytes))  in check_perm_out()
3981 		return emulate_gp(ctxt, 0);  in check_perm_out()
4544 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)  in imm_size()  argument
4548 	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in imm_size()
4554 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,  in decode_imm()  argument
4561 	op->addr.mem.ea = ctxt->_eip;  in decode_imm()
4565 		op->val = insn_fetch(s8, ctxt);  in decode_imm()
4568 		op->val = insn_fetch(s16, ctxt);  in decode_imm()
4571 		op->val = insn_fetch(s32, ctxt);  in decode_imm()
4574 		op->val = insn_fetch(s64, ctxt);  in decode_imm()
4594 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,  in decode_operand()  argument
4601 		decode_register_operand(ctxt, op);  in decode_operand()
4604 		rc = decode_imm(ctxt, op, 1, false);  in decode_operand()
4607 		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
4609 		*op = ctxt->memop;  in decode_operand()
4610 		ctxt->memopp = op;  in decode_operand()
4611 		if (ctxt->d & BitOp)  in decode_operand()
4612 			fetch_bit_operand(ctxt);  in decode_operand()
4616 		ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;  in decode_operand()
4620 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
4621 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);  in decode_operand()
4627 		op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;  in decode_operand()
4628 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);  in decode_operand()
4633 		if (ctxt->d & ByteOp) {  in decode_operand()
4638 		op->bytes = ctxt->op_bytes;  in decode_operand()
4639 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);  in decode_operand()
4645 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
4647 			register_address(ctxt, VCPU_REGS_RDI);  in decode_operand()
4655 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);  in decode_operand()
4661 		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;  in decode_operand()
4664 		rc = decode_imm(ctxt, op, 1, true);  in decode_operand()
4672 		rc = decode_imm(ctxt, op, imm_size(ctxt), true);  in decode_operand()
4675 		rc = decode_imm(ctxt, op, ctxt->op_bytes, true);  in decode_operand()
4678 		ctxt->memop.bytes = 1;  in decode_operand()
4679 		if (ctxt->memop.type == OP_REG) {  in decode_operand()
4680 			ctxt->memop.addr.reg = decode_register(ctxt,  in decode_operand()
4681 					ctxt->modrm_rm, true);  in decode_operand()
4682 			fetch_register_operand(&ctxt->memop);  in decode_operand()
4686 		ctxt->memop.bytes = 2;  in decode_operand()
4689 		ctxt->memop.bytes = 4;  in decode_operand()
4692 		rc = decode_imm(ctxt, op, 2, false);  in decode_operand()
4695 		rc = decode_imm(ctxt, op, imm_size(ctxt), false);  in decode_operand()
4699 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
4701 			register_address(ctxt, VCPU_REGS_RSI);  in decode_operand()
4702 		op->addr.mem.seg = ctxt->seg_override;  in decode_operand()
4708 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;  in decode_operand()
4710 			address_mask(ctxt,  in decode_operand()
4711 				reg_read(ctxt, VCPU_REGS_RBX) +  in decode_operand()
4712 				(reg_read(ctxt, VCPU_REGS_RAX) & 0xff));  in decode_operand()
4713 		op->addr.mem.seg = ctxt->seg_override;  in decode_operand()
4718 		op->addr.mem.ea = ctxt->_eip;  in decode_operand()
4719 		op->bytes = ctxt->op_bytes + 2;  in decode_operand()
4720 		insn_fetch_arr(op->valptr, op->bytes, ctxt);  in decode_operand()
4723 		ctxt->memop.bytes = ctxt->op_bytes + 2;  in decode_operand()
4760 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int emulation_type)  in x86_decode_insn()  argument
4763 	int mode = ctxt->mode;  in x86_decode_insn()
4771 	ctxt->memop.type = OP_NONE;  in x86_decode_insn()
4772 	ctxt->memopp = NULL;  in x86_decode_insn()
4773 	ctxt->_eip = ctxt->eip;  in x86_decode_insn()
4774 	ctxt->fetch.ptr = ctxt->fetch.data;  in x86_decode_insn()
4775 	ctxt->fetch.end = ctxt->fetch.data + insn_len;  in x86_decode_insn()
4776 	ctxt->opcode_len = 1;  in x86_decode_insn()
4777 	ctxt->intercept = x86_intercept_none;  in x86_decode_insn()
4779 		memcpy(ctxt->fetch.data, insn, insn_len);  in x86_decode_insn()
4781 		rc = __do_insn_fetch_bytes(ctxt, 1);  in x86_decode_insn()
4790 		ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);  in x86_decode_insn()
4810 	ctxt->op_bytes = def_op_bytes;  in x86_decode_insn()
4811 	ctxt->ad_bytes = def_ad_bytes;  in x86_decode_insn()
4815 		switch (ctxt->b = insn_fetch(u8, ctxt)) {  in x86_decode_insn()
4819 			ctxt->op_bytes = def_op_bytes ^ 6;  in x86_decode_insn()
4824 				ctxt->ad_bytes = def_ad_bytes ^ 12;  in x86_decode_insn()
4827 				ctxt->ad_bytes = def_ad_bytes ^ 6;  in x86_decode_insn()
4831 			ctxt->seg_override = VCPU_SREG_ES;  in x86_decode_insn()
4835 			ctxt->seg_override = VCPU_SREG_CS;  in x86_decode_insn()
4839 			ctxt->seg_override = VCPU_SREG_SS;  in x86_decode_insn()
4843 			ctxt->seg_override = VCPU_SREG_DS;  in x86_decode_insn()
4847 			ctxt->seg_override = VCPU_SREG_FS;  in x86_decode_insn()
4851 			ctxt->seg_override = VCPU_SREG_GS;  in x86_decode_insn()
4856 			ctxt->rex_prefix = ctxt->b;  in x86_decode_insn()
4859 			ctxt->lock_prefix = 1;  in x86_decode_insn()
4863 			ctxt->rep_prefix = ctxt->b;  in x86_decode_insn()
4871 		ctxt->rex_prefix = 0;  in x86_decode_insn()
4877 	if (ctxt->rex_prefix & 8)  in x86_decode_insn()
4878 		ctxt->op_bytes = 8;	/* REX.W */  in x86_decode_insn()
4881 	opcode = opcode_table[ctxt->b];  in x86_decode_insn()
4883 	if (ctxt->b == 0x0f) {  in x86_decode_insn()
4884 		ctxt->opcode_len = 2;  in x86_decode_insn()
4885 		ctxt->b = insn_fetch(u8, ctxt);  in x86_decode_insn()
4886 		opcode = twobyte_table[ctxt->b];  in x86_decode_insn()
4889 		if (ctxt->b == 0x38) {  in x86_decode_insn()
4890 			ctxt->opcode_len = 3;  in x86_decode_insn()
4891 			ctxt->b = insn_fetch(u8, ctxt);  in x86_decode_insn()
4892 			opcode = opcode_map_0f_38[ctxt->b];  in x86_decode_insn()
4895 	ctxt->d = opcode.flags;  in x86_decode_insn()
4897 	if (ctxt->d & ModRM)  in x86_decode_insn()
4898 		ctxt->modrm = insn_fetch(u8, ctxt);  in x86_decode_insn()
4901 	if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&  in x86_decode_insn()
4902 	    (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {  in x86_decode_insn()
4903 		ctxt->d = NotImpl;  in x86_decode_insn()
4906 	while (ctxt->d & GroupMask) {  in x86_decode_insn()
4907 		switch (ctxt->d & GroupMask) {  in x86_decode_insn()
4909 			goffset = (ctxt->modrm >> 3) & 7;  in x86_decode_insn()
4913 			goffset = (ctxt->modrm >> 3) & 7;  in x86_decode_insn()
4914 			if ((ctxt->modrm >> 6) == 3)  in x86_decode_insn()
4920 			goffset = ctxt->modrm & 7;  in x86_decode_insn()
4924 			if (ctxt->rep_prefix && op_prefix)  in x86_decode_insn()
4926 			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;  in x86_decode_insn()
4935 			if (ctxt->modrm > 0xbf) {  in x86_decode_insn()
4938 					ctxt->modrm - 0xc0, size);  in x86_decode_insn()
4942 				opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];  in x86_decode_insn()
4946 			if ((ctxt->modrm >> 6) == 3)  in x86_decode_insn()
4952 			if (ctxt->mode == X86EMUL_MODE_PROT64)  in x86_decode_insn()
4961 		ctxt->d &= ~(u64)GroupMask;  in x86_decode_insn()
4962 		ctxt->d |= opcode.flags;  in x86_decode_insn()
4965 	ctxt->is_branch = opcode.flags & IsBranch;  in x86_decode_insn()
4968 	if (ctxt->d == 0)  in x86_decode_insn()
4971 	ctxt->execute = opcode.u.execute;  in x86_decode_insn()
4974 	    likely(!(ctxt->d & EmulateOnUD)))  in x86_decode_insn()
4977 	if (unlikely(ctxt->d &  in x86_decode_insn()
4984 		ctxt->check_perm = opcode.check_perm;  in x86_decode_insn()
4985 		ctxt->intercept = opcode.intercept;  in x86_decode_insn()
4987 		if (ctxt->d & NotImpl)  in x86_decode_insn()
4991 			if (ctxt->op_bytes == 4 && (ctxt->d & Stack))  in x86_decode_insn()
4992 				ctxt->op_bytes = 8;  in x86_decode_insn()
4993 			else if (ctxt->d & NearBranch)  in x86_decode_insn()
4994 				ctxt->op_bytes = 8;  in x86_decode_insn()
4997 		if (ctxt->d & Op3264) {  in x86_decode_insn()
4999 				ctxt->op_bytes = 8;  in x86_decode_insn()
5001 				ctxt->op_bytes = 4;  in x86_decode_insn()
5004 		if ((ctxt->d & No16) && ctxt->op_bytes == 2)  in x86_decode_insn()
5005 			ctxt->op_bytes = 4;  in x86_decode_insn()
5007 		if (ctxt->d & Sse)  in x86_decode_insn()
5008 			ctxt->op_bytes = 16;  in x86_decode_insn()
5009 		else if (ctxt->d & Mmx)  in x86_decode_insn()
5010 			ctxt->op_bytes = 8;  in x86_decode_insn()
5014 	if (ctxt->d & ModRM) {  in x86_decode_insn()
5015 		rc = decode_modrm(ctxt, &ctxt->memop);  in x86_decode_insn()
5018 			ctxt->seg_override = ctxt->modrm_seg;  in x86_decode_insn()
5020 	} else if (ctxt->d & MemAbs)  in x86_decode_insn()
5021 		rc = decode_abs(ctxt, &ctxt->memop);  in x86_decode_insn()
5026 		ctxt->seg_override = VCPU_SREG_DS;  in x86_decode_insn()
5028 	ctxt->memop.addr.mem.seg = ctxt->seg_override;  in x86_decode_insn()
5034 	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);  in x86_decode_insn()
5042 	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);  in x86_decode_insn()
5047 	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);  in x86_decode_insn()
5049 	if (ctxt->rip_relative && likely(ctxt->memopp))  in x86_decode_insn()
5050 		ctxt->memopp->addr.mem.ea = address_mask(ctxt,  in x86_decode_insn()
5051 					ctxt->memopp->addr.mem.ea + ctxt->_eip);  in x86_decode_insn()
5055 		ctxt->have_exception = true;  in x86_decode_insn()
5059 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)  in x86_page_table_writing_insn()  argument
5061 	return ctxt->d & PageTable;  in x86_page_table_writing_insn()
5064 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)  in string_insn_completed()  argument
5073 	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||  in string_insn_completed()
5074 	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))  in string_insn_completed()
5075 	    && (((ctxt->rep_prefix == REPE_PREFIX) &&  in string_insn_completed()
5076 		 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))  in string_insn_completed()
5077 		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&  in string_insn_completed()
5078 		    ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))  in string_insn_completed()
5084 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)  in flush_pending_x87_faults()  argument
5093 		return emulate_exception(ctxt, MF_VECTOR, 0, false);  in flush_pending_x87_faults()
5104 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)  in fastop()  argument
5106 	ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;  in fastop()
5108 	if (!(ctxt->d & ByteOp))  in fastop()
5109 		fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;  in fastop()
5112 	    : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),  in fastop()
5114 	    : "c"(ctxt->src2.val));  in fastop()
5116 	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);  in fastop()
5118 		return emulate_de(ctxt);  in fastop()
5122 void init_decode_cache(struct x86_emulate_ctxt *ctxt)  in init_decode_cache()  argument
5125 	ctxt->rip_relative = false;  in init_decode_cache()
5126 	ctxt->rex_prefix = 0;  in init_decode_cache()
5127 	ctxt->lock_prefix = 0;  in init_decode_cache()
5128 	ctxt->rep_prefix = 0;  in init_decode_cache()
5129 	ctxt->regs_valid = 0;  in init_decode_cache()
5130 	ctxt->regs_dirty = 0;  in init_decode_cache()
5132 	ctxt->io_read.pos = 0;  in init_decode_cache()
5133 	ctxt->io_read.end = 0;  in init_decode_cache()
5134 	ctxt->mem_read.end = 0;  in init_decode_cache()
5137 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)  in x86_emulate_insn()  argument
5139 	const struct x86_emulate_ops *ops = ctxt->ops;  in x86_emulate_insn()
5141 	int saved_dst_type = ctxt->dst.type;  in x86_emulate_insn()
5142 	bool is_guest_mode = ctxt->ops->is_guest_mode(ctxt);  in x86_emulate_insn()
5144 	ctxt->mem_read.pos = 0;  in x86_emulate_insn()
5147 	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {  in x86_emulate_insn()
5148 		rc = emulate_ud(ctxt);  in x86_emulate_insn()
5152 	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {  in x86_emulate_insn()
5153 		rc = emulate_ud(ctxt);  in x86_emulate_insn()
5157 	if (unlikely(ctxt->d &  in x86_emulate_insn()
5159 		if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||  in x86_emulate_insn()
5160 				(ctxt->d & Undefined)) {  in x86_emulate_insn()
5161 			rc = emulate_ud(ctxt);  in x86_emulate_insn()
5165 		if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))  in x86_emulate_insn()
5166 		    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {  in x86_emulate_insn()
5167 			rc = emulate_ud(ctxt);  in x86_emulate_insn()
5171 		if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {  in x86_emulate_insn()
5172 			rc = emulate_nm(ctxt);  in x86_emulate_insn()
5176 		if (ctxt->d & Mmx) {  in x86_emulate_insn()
5177 			rc = flush_pending_x87_faults(ctxt);  in x86_emulate_insn()
5184 			fetch_possible_mmx_operand(&ctxt->src);  in x86_emulate_insn()
5185 			fetch_possible_mmx_operand(&ctxt->src2);  in x86_emulate_insn()
5186 			if (!(ctxt->d & Mov))  in x86_emulate_insn()
5187 				fetch_possible_mmx_operand(&ctxt->dst);  in x86_emulate_insn()
5190 		if (unlikely(is_guest_mode) && ctxt->intercept) {  in x86_emulate_insn()
5191 			rc = emulator_check_intercept(ctxt, ctxt->intercept,  in x86_emulate_insn()
5198 		if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {  in x86_emulate_insn()
5199 			rc = emulate_ud(ctxt);  in x86_emulate_insn()
5204 		if ((ctxt->d & Priv) && ops->cpl(ctxt)) {  in x86_emulate_insn()
5205 			if (ctxt->d & PrivUD)  in x86_emulate_insn()
5206 				rc = emulate_ud(ctxt);  in x86_emulate_insn()
5208 				rc = emulate_gp(ctxt, 0);  in x86_emulate_insn()
5213 		if (ctxt->d & CheckPerm) {  in x86_emulate_insn()
5214 			rc = ctxt->check_perm(ctxt);  in x86_emulate_insn()
5219 		if (unlikely(is_guest_mode) && (ctxt->d & Intercept)) {  in x86_emulate_insn()
5220 			rc = emulator_check_intercept(ctxt, ctxt->intercept,  in x86_emulate_insn()
5226 		if (ctxt->rep_prefix && (ctxt->d & String)) {  in x86_emulate_insn()
5228 			if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {  in x86_emulate_insn()
5229 				string_registers_quirk(ctxt);  in x86_emulate_insn()
5230 				ctxt->eip = ctxt->_eip;  in x86_emulate_insn()
5231 				ctxt->eflags &= ~X86_EFLAGS_RF;  in x86_emulate_insn()
5237 	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {  in x86_emulate_insn()
5238 		rc = segmented_read(ctxt, ctxt->src.addr.mem,  in x86_emulate_insn()
5239 				    ctxt->src.valptr, ctxt->src.bytes);  in x86_emulate_insn()
5242 		ctxt->src.orig_val64 = ctxt->src.val64;  in x86_emulate_insn()
5245 	if (ctxt->src2.type == OP_MEM) {  in x86_emulate_insn()
5246 		rc = segmented_read(ctxt, ctxt->src2.addr.mem,  in x86_emulate_insn()
5247 				    &ctxt->src2.val, ctxt->src2.bytes);  in x86_emulate_insn()
5252 	if ((ctxt->d & DstMask) == ImplicitOps)  in x86_emulate_insn()
5256 	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {  in x86_emulate_insn()
5258 		rc = segmented_read(ctxt, ctxt->dst.addr.mem,  in x86_emulate_insn()
5259 				   &ctxt->dst.val, ctxt->dst.bytes);  in x86_emulate_insn()
5261 			if (!(ctxt->d & NoWrite) &&  in x86_emulate_insn()
5263 			    ctxt->exception.vector == PF_VECTOR)  in x86_emulate_insn()
5264 				ctxt->exception.error_code |= PFERR_WRITE_MASK;  in x86_emulate_insn()
5269 	ctxt->dst.orig_val64 = ctxt->dst.val64;  in x86_emulate_insn()
5273 	if (unlikely(is_guest_mode) && (ctxt->d & Intercept)) {  in x86_emulate_insn()
5274 		rc = emulator_check_intercept(ctxt, ctxt->intercept,  in x86_emulate_insn()
5280 	if (ctxt->rep_prefix && (ctxt->d & String))  in x86_emulate_insn()
5281 		ctxt->eflags |= X86_EFLAGS_RF;  in x86_emulate_insn()
5283 		ctxt->eflags &= ~X86_EFLAGS_RF;  in x86_emulate_insn()
5285 	if (ctxt->execute) {  in x86_emulate_insn()
5286 		if (ctxt->d & Fastop)  in x86_emulate_insn()
5287 			rc = fastop(ctxt, ctxt->fop);  in x86_emulate_insn()
5289 			rc = ctxt->execute(ctxt);  in x86_emulate_insn()
5295 	if (ctxt->opcode_len == 2)  in x86_emulate_insn()
5297 	else if (ctxt->opcode_len == 3)  in x86_emulate_insn()
5300 	switch (ctxt->b) {  in x86_emulate_insn()
5302 		if (test_cc(ctxt->b, ctxt->eflags))  in x86_emulate_insn()
5303 			rc = jmp_rel(ctxt, ctxt->src.val);  in x86_emulate_insn()
5306 		ctxt->dst.val = ctxt->src.addr.mem.ea;  in x86_emulate_insn()
5309 		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))  in x86_emulate_insn()
5310 			ctxt->dst.type = OP_NONE;  in x86_emulate_insn()
5312 			rc = em_xchg(ctxt);  in x86_emulate_insn()
5315 		switch (ctxt->op_bytes) {  in x86_emulate_insn()
5316 		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;  in x86_emulate_insn()
5317 		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;  in x86_emulate_insn()
5318 		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;  in x86_emulate_insn()
5322 		rc = emulate_int(ctxt, 3);  in x86_emulate_insn()
5325 		rc = emulate_int(ctxt, ctxt->src.val);  in x86_emulate_insn()
5328 		if (ctxt->eflags & X86_EFLAGS_OF)  in x86_emulate_insn()
5329 			rc = emulate_int(ctxt, 4);  in x86_emulate_insn()
5333 		rc = jmp_rel(ctxt, ctxt->src.val);  in x86_emulate_insn()
5334 		ctxt->dst.type = OP_NONE; /* Disable writeback. */  in x86_emulate_insn()
5337 		ctxt->ops->halt(ctxt);  in x86_emulate_insn()
5341 		ctxt->eflags ^= X86_EFLAGS_CF;  in x86_emulate_insn()
5344 		ctxt->eflags &= ~X86_EFLAGS_CF;  in x86_emulate_insn()
5347 		ctxt->eflags |= X86_EFLAGS_CF;  in x86_emulate_insn()
5350 		ctxt->eflags &= ~X86_EFLAGS_DF;  in x86_emulate_insn()
5353 		ctxt->eflags |= X86_EFLAGS_DF;  in x86_emulate_insn()
5363 	if (ctxt->d & SrcWrite) {  in x86_emulate_insn()
5364 		BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);  in x86_emulate_insn()
5365 		rc = writeback(ctxt, &ctxt->src);  in x86_emulate_insn()
5369 	if (!(ctxt->d & NoWrite)) {  in x86_emulate_insn()
5370 		rc = writeback(ctxt, &ctxt->dst);  in x86_emulate_insn()
5379 	ctxt->dst.type = saved_dst_type;  in x86_emulate_insn()
5381 	if ((ctxt->d & SrcMask) == SrcSI)  in x86_emulate_insn()
5382 		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);  in x86_emulate_insn()
5384 	if ((ctxt->d & DstMask) == DstDI)  in x86_emulate_insn()
5385 		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);  in x86_emulate_insn()
5387 	if (ctxt->rep_prefix && (ctxt->d & String)) {  in x86_emulate_insn()
5389 		struct read_cache *r = &ctxt->io_read;  in x86_emulate_insn()
5390 		if ((ctxt->d & SrcMask) == SrcSI)  in x86_emulate_insn()
5391 			count = ctxt->src.count;  in x86_emulate_insn()
5393 			count = ctxt->dst.count;  in x86_emulate_insn()
5394 		register_address_increment(ctxt, VCPU_REGS_RCX, -count);  in x86_emulate_insn()
5396 		if (!string_insn_completed(ctxt)) {  in x86_emulate_insn()
5401 			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&  in x86_emulate_insn()
5408 				ctxt->mem_read.end = 0;  in x86_emulate_insn()
5409 				writeback_registers(ctxt);  in x86_emulate_insn()
5414 		ctxt->eflags &= ~X86_EFLAGS_RF;  in x86_emulate_insn()
5417 	ctxt->eip = ctxt->_eip;  in x86_emulate_insn()
5418 	if (ctxt->mode != X86EMUL_MODE_PROT64)  in x86_emulate_insn()
5419 		ctxt->eip = (u32)ctxt->_eip;  in x86_emulate_insn()
5423 		if (KVM_EMULATOR_BUG_ON(ctxt->exception.vector > 0x1f, ctxt))  in x86_emulate_insn()
5425 		ctxt->have_exception = true;  in x86_emulate_insn()
5431 		writeback_registers(ctxt);  in x86_emulate_insn()
5436 	switch (ctxt->b) {  in x86_emulate_insn()
5438 		(ctxt->ops->wbinvd)(ctxt);  in x86_emulate_insn()
5446 		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);  in x86_emulate_insn()
5449 		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);  in x86_emulate_insn()
5452 		if (test_cc(ctxt->b, ctxt->eflags))  in x86_emulate_insn()
5453 			ctxt->dst.val = ctxt->src.val;  in x86_emulate_insn()
5454 		else if (ctxt->op_bytes != 4)  in x86_emulate_insn()
5455 			ctxt->dst.type = OP_NONE; /* no writeback */  in x86_emulate_insn()
5458 		if (test_cc(ctxt->b, ctxt->eflags))  in x86_emulate_insn()
5459 			rc = jmp_rel(ctxt, ctxt->src.val);  in x86_emulate_insn()
5462 		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);  in x86_emulate_insn()
5465 		ctxt->dst.bytes = ctxt->op_bytes;  in x86_emulate_insn()
5466 		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val  in x86_emulate_insn()
5467 						       : (u16) ctxt->src.val;  in x86_emulate_insn()
5470 		ctxt->dst.bytes = ctxt->op_bytes;  in x86_emulate_insn()
5471 		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :  in x86_emulate_insn()
5472 							(s16) ctxt->src.val;  in x86_emulate_insn()
5489 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)  in emulator_invalidate_register_cache()  argument
5491 	invalidate_registers(ctxt);  in emulator_invalidate_register_cache()
5494 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)  in emulator_writeback_register_cache()  argument
5496 	writeback_registers(ctxt);  in emulator_writeback_register_cache()
5499 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)  in emulator_can_use_gpa()  argument
5501 	if (ctxt->rep_prefix && (ctxt->d & String))  in emulator_can_use_gpa()
5504 	if (ctxt->d & TwoMemOp)  in emulator_can_use_gpa()