Merge branch 'qemu-plugin' into 'master'

Qemu plugin

See merge request projectn/nemu!80

Makefile

@@ -26,18 +26,22 @@ LD_LIBS = -lSDL2 -lreadline -ldl
 endif
 
 ifndef SHARE
-DIFF ?= kvm
+DIFF ?= qemu-dl
 ifneq ($(ISA),x86)
 ifeq ($(DIFF),kvm)
-DIFF = qemu
+DIFF = qemu-dl
 $(info KVM is only supported with ISA=x86, use QEMU instead)
 endif
 endif
 
-ifeq ($(DIFF),qemu)
-DIFF_REF_PATH = $(NEMU_HOME)/tools/qemu-diff
+ifeq ($(DIFF),qemu-socket)
+DIFF_REF_PATH = $(NEMU_HOME)/tools/qemu-socket-diff
 DIFF_REF_SO = $(DIFF_REF_PATH)/build/$(ISA)-qemu-so
-CFLAGS += -D__DIFF_REF_QEMU__
+CFLAGS += -D__DIFF_REF_QEMU_SOCKET__ -D__DIFF_REF_QEMU__
+else ifeq ($(DIFF),qemu-dl)
+DIFF_REF_PATH = $(NEMU_HOME)/tools/qemu-dl-diff
+DIFF_REF_SO = $(DIFF_REF_PATH)/build/qemu-so
+CFLAGS += -D__DIFF_REF_QEMU_DL__ -D__DIFF_REF_QEMU__
 else ifeq ($(DIFF),kvm)
 DIFF_REF_PATH = $(NEMU_HOME)/tools/kvm-diff
 DIFF_REF_SO = $(DIFF_REF_PATH)/build/$(ISA)-kvm-so
@@ -48,7 +52,7 @@ DIFF_REF_SO = $(DIFF_REF_PATH)/build/$(ISA)-nemu-interpreter-so
 CFLAGS += -D__DIFF_REF_NEMU__
 MKFLAGS = ISA=$(ISA) SHARE=1 ENGINE=interpreter
 else
-$(error invalid DIFF. Supported: qemu kvm nemu)
+$(error invalid DIFF. Supported: qemu-dl kvm qemu-socket nemu)
 endif
 endif
 

include/isa.h

@@ -39,8 +39,7 @@ bool isa_difftest_checkregs(CPU_state *ref_r, vaddr_t pc);
 void isa_difftest_attach();
 
   // for ref
-void isa_difftest_getregs(void *r);
-void isa_difftest_setregs(const void *r);
+void isa_difftest_regcpy(void *dut, bool to_ref);
 void isa_difftest_raise_intr(word_t NO);
 
 #endif

include/isa/x86.h

@@ -122,7 +122,7 @@ typedef struct {
   };
 
   int mem_exception;
-  int hack_kvm_pf_write;
+  int lock;
   word_t error_code;
 
   bool INTR;

include/monitor/difftest.h

@@ -6,23 +6,24 @@
 #ifdef DIFF_TEST
 void difftest_skip_ref();
 void difftest_skip_dut(int nr_ref, int nr_dut);
+void difftest_set_patch(void (*fn)(void *arg), void *arg);
 void difftest_step(vaddr_t this_pc, vaddr_t next_pc);
 #else
 #define difftest_skip_ref()
 #define difftest_skip_dut(nr_ref, nr_dut)
+static inline void difftest_set_patch(void (*fn)(void *arg), void *arg) {}
 static inline void difftest_step(vaddr_t this_pc, vaddr_t next_pc) {}
 #endif
 
-extern void (*ref_difftest_memcpy_from_dut)(paddr_t dest, void *src, size_t n);
-extern void (*ref_difftest_getregs)(void *c);
-extern void (*ref_difftest_setregs)(const void *c);
+extern void (*ref_difftest_memcpy)(paddr_t dest, void *src, size_t n, bool to_ref);
+extern void (*ref_difftest_regcpy)(void *c, bool to_ref);
 extern void (*ref_difftest_exec)(uint64_t n);
-extern void (*ref_difftest_raise_intr)(word_t NO);
+extern void (*ref_difftest_raise_intr)(uint64_t NO);
 
 static inline bool difftest_check_reg(const char *name, vaddr_t pc, rtlreg_t ref, rtlreg_t dut) {
   if (ref != dut) {
-    Log("%s is different after executing instruction at pc = " FMT_WORD ", right = " FMT_WORD ", wrong = " FMT_WORD,
-        name, pc, ref, dut);
+    Log("%s is different after executing instruction at pc = " FMT_WORD
+        ", right = " FMT_WORD ", wrong = " FMT_WORD, name, pc, ref, dut);
     return false;
   }
   return true;

src/isa/mips32/difftest/ref.c

@@ -2,12 +2,9 @@
 #include <cpu/exec.h>
 #include "difftest.h"
 
-void isa_difftest_getregs(void *r) {
-  memcpy(r, &cpu, DIFFTEST_REG_SIZE);
-}
-
-void isa_difftest_setregs(const void *r) {
-  memcpy(&cpu, r, DIFFTEST_REG_SIZE);
+void isa_difftest_regcpy(void *dut, bool to_ref) {
+  if (to_ref) memcpy(&cpu, dut, DIFFTEST_REG_SIZE);
+  else memcpy(dut, &cpu, DIFFTEST_REG_SIZE);
 }
 
 void isa_difftest_raise_intr(word_t NO) {

src/isa/riscv32/difftest/ref.c

@@ -2,12 +2,9 @@
 #include <cpu/exec.h>
 #include "difftest.h"
 
-void isa_difftest_getregs(void *r) {
-  memcpy(r, &cpu, DIFFTEST_REG_SIZE);
-}
-
-void isa_difftest_setregs(const void *r) {
-  memcpy(&cpu, r, DIFFTEST_REG_SIZE);
+void isa_difftest_regcpy(void *dut, bool to_ref) {
+  if (to_ref) memcpy(&cpu, dut, DIFFTEST_REG_SIZE);
+  else memcpy(dut, &cpu, DIFFTEST_REG_SIZE);
 }
 
 void isa_difftest_raise_intr(word_t NO) {

src/isa/x86/decode.c

@@ -110,10 +110,10 @@ void read_ModR_M(DecodeExecState *s, Operand *rm, bool load_rm_val, Operand *reg
   if (reg != NULL) operand_reg(s, reg, load_reg_val, m.reg, reg->width);
   if (m.mod == 3) operand_reg(s, rm, load_rm_val, m.R_M, rm->width);
   else {
-#ifndef __PA__
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
     if (((s->opcode == 0x80 || s->opcode == 0x81 || s->opcode == 0x83) && s->isa.ext_opcode == 7) ||
         (s->opcode == 0x1ba && s->isa.ext_opcode == 4)) {
-      cpu.hack_kvm_pf_write = 0; // fix with cmp and bt, since they do not write memory
+      cpu.lock = 0; // fix with cmp and bt, since they do not write memory
     }
 #endif
     load_addr(s, &m, rm);

src/isa/x86/difftest/dut.c

@@ -7,7 +7,7 @@
 #include <memory/paddr.h>
 
 #ifdef ENABLE_DIFFTEST_INSTR_QUEUE
-#define INSTR_QUEUE_SIZE (1 << 15)
+#define INSTR_QUEUE_SIZE (1 << 7)
 static uint32_t q_idx = 0;
 struct {
   vaddr_t pc;
@@ -56,7 +56,7 @@ bool isa_difftest_checkregs(CPU_state *ref_r, vaddr_t pc) {
 
 void isa_difftest_attach() {
   // first copy the image
-  ref_difftest_memcpy_from_dut(0, guest_to_host(0), PMEM_SIZE);
+  ref_difftest_memcpy(0, guest_to_host(0), PMEM_SIZE, true);
 
   // then set some special registers
   uint8_t code[] = {
@@ -77,15 +77,15 @@ void isa_difftest_attach() {
   *(uint32_t *)(idtdesc + 2) = cpu.idtr.base;
 
   assert(sizeof(code) < 0x40);
-  ref_difftest_memcpy_from_dut(0x7e00, code, sizeof(code));
-  ref_difftest_memcpy_from_dut(0x7e40, idtdesc, sizeof(idtdesc));
+  ref_difftest_memcpy(0x7e00, code, sizeof(code), true);
+  ref_difftest_memcpy(0x7e40, idtdesc, sizeof(idtdesc), true);
 
   CPU_state r = cpu;
   r.pc = 0x7e00;
-  ref_difftest_setregs(&r);
+  ref_difftest_regcpy(&r, true);
   ref_difftest_exec(5);
 
-  ref_difftest_setregs(&cpu);
+  ref_difftest_regcpy(&cpu, true);
 }
 #else
 bool isa_difftest_checkregs(CPU_state *ref_r, vaddr_t pc) {

src/isa/x86/difftest/ref.c

@@ -2,12 +2,9 @@
 #include <cpu/exec.h>
 #include "difftest.h"
 
-void isa_difftest_getregs(void *r) {
-  memcpy(r, &cpu, DIFFTEST_REG_SIZE);
-}
-
-void isa_difftest_setregs(const void *r) {
-  memcpy(&cpu, r, DIFFTEST_REG_SIZE);
+void isa_difftest_regcpy(void *dut, bool to_ref) {
+  if (to_ref) memcpy(&cpu, dut, DIFFTEST_REG_SIZE);
+  else memcpy(dut, &cpu, DIFFTEST_REG_SIZE);
 }
 
 void isa_difftest_raise_intr(word_t NO) {

src/isa/x86/exec/eflags.c

 #include "../local-include/rtl.h"
 #include "../local-include/reg.h"
+#include <monitor/difftest.h>
 
 #define EFLAGS_BIT_CF 0
 #define EFLAGS_BIT_PF 2
@@ -36,3 +37,15 @@ void rtl_compute_eflags(DecodeExecState *s, rtlreg_t *dest) {
 void rtl_set_eflags(DecodeExecState *s, const rtlreg_t *src) {
   MAP(_EFLAGS, DECODE)
 }
+
+void difftest_fix_eflags(void *arg) {
+#define EFLAGS_MASK_ID (1 << 21)
+#define EFLAGS_MASK_AC (1 << 18)
+#define EFLAGS_MASK_AF (1 << 4)
+#define EFLAGS_FIX_MASK (EFLAGS_MASK_ID | EFLAGS_MASK_AC | EFLAGS_MASK_AF)
+  uint32_t esp = (uintptr_t)arg;
+  uint32_t flags;
+  ref_difftest_memcpy(esp, &flags, 4, false);
+  flags &= ~EFLAGS_FIX_MASK;
+  ref_difftest_memcpy(esp, &flags, 4, true);
+}

src/isa/x86/exec/eflags.h

@@ -35,7 +35,10 @@ static inline def_EHelper(pushf) {
   void rtl_compute_eflags(DecodeExecState *s, rtlreg_t *dest);
   rtl_compute_eflags(s, s0);
   rtl_push(s, s0);
-  print_asm("popf");
+  print_asm("pushf");
+
+  extern void difftest_fix_eflags(void *arg);
+  difftest_set_patch(difftest_fix_eflags, (void *)(uintptr_t)cpu.esp);
 }
 
 static inline def_EHelper(popf) {

src/isa/x86/exec/exec.c

@@ -280,7 +280,7 @@ EX   (0xf8, clc)            EX   (0xf9, stc)            EX   (0xfa, cli)
 EX   (0xfc, cld)            EX   (0xfd, std)            IDEXW(0xfe, E, gp4, 1)      IDEX (0xff, E, gp5)
   case 0x64: s->isa.sreg_base = &cpu.sreg[SR_FS].base; goto again;
   case 0x65: s->isa.sreg_base = &cpu.sreg[SR_GS].base; goto again;
-  case 0xf0: goto again; // LOCK prefix
+  case 0xf0: cpu.lock = 1; goto again; // LOCK prefix
   case 0xf2: s->isa.rep_flags = PREFIX_REPNZ; goto again;
   case 0xf3: s->isa.rep_flags = PREFIX_REP; goto again;
 #endif
@@ -332,7 +332,7 @@ vaddr_t isa_exec_once() {
     raise_intr(&s, cpu.mem_exception, cpu.pc);
     cpu.mem_exception = 0;
   }
-  cpu.hack_kvm_pf_write = 0;
+  cpu.lock = 0;
 #endif
 
   update_pc(&s);

src/isa/x86/exec/logic.h

@@ -90,19 +90,28 @@ static inline def_EHelper(sar) {
 static inline def_EHelper(shl) {
 #ifndef __PA__
   int count = *dsrc1 & 0x1f;
-  if (count == 1) {
-    rtl_msb(s, s0, ddest, id_dest->width);
-    rtl_shli(s, s1, ddest, 1);
-    rtl_msb(s, s1, s1, id_dest->width);
-    rtl_xor(s, s0, s0, s1);
-    rtl_set_OF(s, s0);
-  }
   if (count != 0) {
     rtl_msb(s, s1, ddest, id_dest->width);
     rtl_set_CF(s, s1);
   }
+#ifdef __DIFF_REF_KVM__
+  int update_of = (count == 1);
+#else
+  int update_of = 1;
+#endif
+  if (update_of) {
+    rtl_mv(s, s0, ddest);
+  }
+
   rtl_shl(s, ddest, ddest, dsrc1);
   operand_write(s, id_dest, ddest);
+
+  if (update_of) {
+    rtl_xor(s, s0, s0, ddest);
+    rtl_msb(s, s0, s0, id_dest->width);
+    rtl_set_OF(s, s0);
+  }
+
   if (count != 0) {
     rtl_update_ZFSF(s, ddest, id_dest->width);
   }

src/isa/x86/exec/misc.h

@@ -27,6 +27,7 @@ static inline def_EHelper(cpuid) {
   rtl_mv(s, &cpu.ecx, rz);
   rtl_mv(s, &cpu.edx, rz);
 
+  difftest_skip_ref();
   print_asm("cpuid");
 }
 
@@ -40,5 +41,8 @@ static inline def_EHelper(fpu) {
 }
 
 static inline def_EHelper(hlt) {
+  extern void raise_intr(DecodeExecState *s, uint32_t NO, vaddr_t ret_addr);
+  raise_intr(s, 48, s->seq_pc);
+  if (ref_difftest_raise_intr) ref_difftest_raise_intr(48);
   print_asm("hlt");
 }

src/isa/x86/exec/system.h

@@ -101,10 +101,6 @@ static inline def_EHelper(int) {
   void raise_intr(DecodeExecState *s, uint32_t, vaddr_t);
   raise_intr(s, *ddest, s->seq_pc);
   print_asm("int %s", id_dest->str);
-
-#ifndef __DIFF_REF_NEMU__
-  difftest_skip_dut(1, 2);
-#endif
 }
 
 static inline def_EHelper(iret) {

src/isa/x86/intr.c

@@ -2,17 +2,6 @@
 #include "monitor/difftest.h"
 #include "local-include/rtl.h"
 
-typedef union GateDescriptor {
-  struct {
-    uint32_t offset_15_0      : 16;
-    uint32_t dont_care0       : 16;
-    uint32_t dont_care1       : 15;
-    uint32_t present          : 1;
-    uint32_t offset_31_16     : 16;
-  };
-  uint32_t val;
-} GateDesc;
-
 #ifndef __ICS_EXPORT
 void raise_intr(DecodeExecState *s, uint32_t NO, vaddr_t ret_addr) {
   assert(NO < 256);
@@ -57,6 +46,7 @@ void raise_intr(DecodeExecState *s, uint32_t NO, vaddr_t ret_addr) {
   void rtl_compute_eflags(DecodeExecState *s, rtlreg_t *dest);
   rtl_compute_eflags(s, s0);
   rtl_push(s, s0);
+  word_t eflags_esp = cpu.esp;
   rtl_li(s, s0, old_cs);
   rtl_push(s, s0);   // cs
   rtl_li(s, s0, ret_addr);
@@ -75,8 +65,10 @@ void raise_intr(DecodeExecState *s, uint32_t NO, vaddr_t ret_addr) {
 
   rtl_jr(s, s1);
 
-#ifdef DIFF_TEST
-  if (ref_difftest_raise_intr) ref_difftest_raise_intr(NO);
+#ifndef __DIFF_REF_NEMU__
+  difftest_skip_dut(1, 2);
+  void difftest_fix_eflags(void *arg);
+  difftest_set_patch(difftest_fix_eflags, (void *)(uintptr_t)eflags_esp);
 #endif
 }
 

src/isa/x86/local-include/decode.h

@@ -110,10 +110,10 @@ static inline def_DopHelper(O) {
  * Ev <- Gv
  */
 static inline def_DHelper(G2E) {
-#ifndef __PA__
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
   if (s->opcode != 0x38 && s->opcode != 0x39 && // cmp
       s->opcode != 0x84 && s->opcode != 0x85) { // test
-    cpu.hack_kvm_pf_write = 1;
+    cpu.lock = 1;
   }
 #endif
   operand_rm(s, id_dest, true, id_src1, true);
@@ -131,9 +131,11 @@ static inline def_DHelper(bit_G2E) {
     rtl_shli(s, s0, s0, 2);
     rtl_add(s, &s->isa.mbr, s->isa.mbase, s0);
     s->isa.mbase = &s->isa.mbr;
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
     if (s->opcode != 0x1a3) { // bt
-      cpu.hack_kvm_pf_write = 1;
+      cpu.lock = 1;
     }
+#endif
     rtl_lm(s, &id_dest->val, s->isa.mbase, s->isa.moff, id_dest->width);
   }
   rtl_andi(s, &id_src1->val, dsrc1, 0x1f);
@@ -175,8 +177,8 @@ static inline def_DHelper(I_E2G) {
  * Ev <- Iv
  */
 static inline def_DHelper(I2E) {
-#ifndef __PA__
-  cpu.hack_kvm_pf_write = 1;
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
+  cpu.lock = 1;
 #endif
   operand_rm(s, id_dest, true, NULL, false);
   decode_op_I(s, id_src1, true);
@@ -232,8 +234,8 @@ static inline def_DHelper(test_I) {
 
 static inline def_DHelper(SI2E) {
   assert(id_dest->width == 2 || id_dest->width == 4);
-#ifndef __PA__
-  cpu.hack_kvm_pf_write = 1;
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
+  cpu.lock = 1;
 #endif
   operand_rm(s, id_dest, true, NULL, false);
   id_src1->width = 1;
@@ -259,8 +261,8 @@ static inline def_DHelper(gp2_1_E) {
 }
 
 static inline def_DHelper(gp2_cl2E) {
-#ifndef __PA__
-  cpu.hack_kvm_pf_write = 1;
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
+  cpu.lock = 1;
 #endif
   operand_rm(s, id_dest, true, NULL, false);
   // shift instructions will eventually use the lower
@@ -269,8 +271,8 @@ static inline def_DHelper(gp2_cl2E) {
 }
 
 static inline def_DHelper(gp2_Ib2E) {
-#ifndef __PA__
-  cpu.hack_kvm_pf_write = 1;
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
+  cpu.lock = 1;
 #endif
   operand_rm(s, id_dest, true, NULL, false);
   id_src1->width = 1;
@@ -280,8 +282,8 @@ static inline def_DHelper(gp2_Ib2E) {
 /* Ev <- GvIb
  * use for shld/shrd */
 static inline def_DHelper(Ib_G2E) {
-#ifndef __PA__
-  cpu.hack_kvm_pf_write = 1;
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
+  cpu.lock = 1;
 #endif
   operand_rm(s, id_dest, true, id_src2, true);
   id_src1->width = 1;
@@ -291,8 +293,8 @@ static inline def_DHelper(Ib_G2E) {
 /* Ev <- GvCL
  * use for shld/shrd */
 static inline def_DHelper(cl_G2E) {
-#ifndef __PA__
-  cpu.hack_kvm_pf_write = 1;
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
+  cpu.lock = 1;
 #endif
   operand_rm(s, id_dest, true, id_src2, true);
   // shift instructions will eventually use the lower
@@ -302,7 +304,9 @@ static inline def_DHelper(cl_G2E) {
 
 // for cmpxchg
 static inline def_DHelper(a_G2E) {
-  cpu.hack_kvm_pf_write = 1;
+#if !defined(__PA__) && defined(DIFF_TEST) && defined(__DIFF_REF_KVM__)
+  cpu.lock = 1;
+#endif
   operand_rm(s, id_dest, true, id_src2, true);
   operand_reg(s, id_src1, true, R_EAX, 4);
 }

src/isa/x86/mmu.c

@@ -46,14 +46,14 @@ static inline bool check_permission(PTE *pte, bool ok, vaddr_t vaddr, int type)
 #else
 static inline bool check_permission(PTE *pte, bool ok, vaddr_t vaddr, int type) {
   int is_user = cpu.sreg[SR_CS].rpl == MODE_R3;
-  int is_write = (type == MEM_TYPE_WRITE) || (type == MEM_TYPE_READ && cpu.hack_kvm_pf_write);
+  int is_write = (type == MEM_TYPE_WRITE) || (type == MEM_TYPE_READ && cpu.lock);
   ok = ok && pte->p;
   ok = ok && !(is_user && !pte->u);
   ok = ok && !(is_write && !pte->w); // assume that CR0.WP is always enabled
   if (!ok && cpu.mem_exception == 0) {
     cpu.cr2 = vaddr;
     cpu.mem_exception = 14;
-    cpu.hack_kvm_pf_write = 0;
+    cpu.lock = 0;
     cpu.error_code = pte->p | (is_write << 1) | (is_user << 2);
   }
   return ok;
@@ -82,7 +82,11 @@ static inline paddr_t ptw(vaddr_t vaddr, int type) {
     pte[1].a = 1;
     paddr_write(p_pte[1], pte[1].val, PTE_SIZE);
 #ifdef DIFF_TEST
-    ref_difftest_memcpy_from_dut(p_pte[1], &pte[1].val, PTE_SIZE);
+#ifdef __DIFF_REF_KVM__
+    ref_difftest_memcpy(p_pte[1], &pte[1].val, PTE_SIZE, true);
+#elif __DIFF_REF_QEMU_DL__
+    ref_difftest_memcpy(p_pte[1] + 0xc0000000, &pte[1].val, PTE_SIZE, true);
+#endif
 #endif
   }
   bool is_write = (type == MEM_TYPE_WRITE);
@@ -91,7 +95,11 @@ static inline paddr_t ptw(vaddr_t vaddr, int type) {
     pte[0].d |= is_write;
     paddr_write(p_pte[0], pte[0].val, PTE_SIZE);
 #ifdef DIFF_TEST
-    ref_difftest_memcpy_from_dut(p_pte[0], &pte[0].val, PTE_SIZE);
+#ifdef __DIFF_REF_KVM__
+    ref_difftest_memcpy(p_pte[0], &pte[0].val, PTE_SIZE, true);
+#elif __DIFF_REF_QEMU_DL__
+    ref_difftest_memcpy(p_pte[0] + 0xc0000000, &pte[0].val, PTE_SIZE, true);
+#endif
 #endif
   }
 #endif

src/monitor/difftest/dut.c

@@ -4,14 +4,19 @@
 #include <memory/paddr.h>
 #include <monitor/monitor.h>
 
-void (*ref_difftest_memcpy_from_dut)(paddr_t dest, void *src, size_t n) = NULL;
-void (*ref_difftest_getregs)(void *c) = NULL;
-void (*ref_difftest_setregs)(const void *c) = NULL;
+#ifdef __DIFF_REF_QEMU_DL__
+__thread uint8_t resereve_for_qemu_tls[4096];
+#endif
+
+void (*ref_difftest_memcpy)(paddr_t addr, void *buf, size_t n, bool to_ref) = NULL;
+void (*ref_difftest_regcpy)(void *dut, bool to_ref) = NULL;
 void (*ref_difftest_exec)(uint64_t n) = NULL;
-void (*ref_difftest_raise_intr)(word_t NO) = NULL;
+void (*ref_difftest_raise_intr)(uint64_t NO) = NULL;
 
 static bool is_skip_ref = false;
 static int skip_dut_nr_instr = 0;
+void (*patch_fn)(void *arg) = NULL;
+static void* patch_arg = NULL;
 #ifndef __ICS_EXPORT
 static bool is_detach = false;
 #endif
@@ -50,6 +55,11 @@ void difftest_skip_dut(int nr_ref, int nr_dut) {
   }
 }
 
+void difftest_set_patch(void (*fn)(void *arg), void *arg) {
+  patch_fn = fn;
+  patch_arg = arg;
+}
+
 void init_difftest(char *ref_so_file, long img_size, int port) {
 #ifndef DIFF_TEST
   return;
@@ -61,14 +71,11 @@ void init_difftest(char *ref_so_file, long img_size, int port) {
   handle = dlopen(ref_so_file, RTLD_LAZY | RTLD_DEEPBIND);
   assert(handle);
 
-  ref_difftest_memcpy_from_dut = dlsym(handle, "difftest_memcpy_from_dut");
-  assert(ref_difftest_memcpy_from_dut);
+  ref_difftest_memcpy = dlsym(handle, "difftest_memcpy");
+  assert(ref_difftest_memcpy);
 
-  ref_difftest_getregs = dlsym(handle, "difftest_getregs");
-  assert(ref_difftest_getregs);
-
-  ref_difftest_setregs = dlsym(handle, "difftest_setregs");
-  assert(ref_difftest_setregs);
+  ref_difftest_regcpy = dlsym(handle, "difftest_regcpy");
+  assert(ref_difftest_regcpy);
 
   ref_difftest_exec = dlsym(handle, "difftest_exec");
   assert(ref_difftest_exec);
@@ -85,8 +92,8 @@ void init_difftest(char *ref_so_file, long img_size, int port) {
       "If it is not necessary, you can turn it off in include/common.h.", ref_so_file);
 
   ref_difftest_init(port);
-  ref_difftest_memcpy_from_dut(IMAGE_START + PMEM_BASE, guest_to_host(IMAGE_START), img_size);
-  ref_difftest_setregs(&cpu);
+  ref_difftest_memcpy(IMAGE_START + PMEM_BASE, guest_to_host(IMAGE_START), img_size, true);
+  ref_difftest_regcpy(&cpu, true);
 }
 
 static void checkregs(CPU_state *ref, vaddr_t pc) {
@@ -105,7 +112,7 @@ void difftest_step(vaddr_t this_pc, vaddr_t next_pc) {
 
 #endif
   if (skip_dut_nr_instr > 0) {
-    ref_difftest_getregs(&ref_r);
+    ref_difftest_regcpy(&ref_r, false);
     if (ref_r.pc == next_pc) {
       checkregs(&ref_r, next_pc);
       skip_dut_nr_instr = 0;
@@ -119,13 +126,19 @@ void difftest_step(vaddr_t this_pc, vaddr_t next_pc) {
 
   if (is_skip_ref) {
     // to skip the checking of an instruction, just copy the reg state to reference design
-    ref_difftest_setregs(&cpu);
+    ref_difftest_regcpy(&cpu, true);
     is_skip_ref = false;
     return;
   }
 
   ref_difftest_exec(1);
-  ref_difftest_getregs(&ref_r);
+
+  if (patch_fn) {
+    patch_fn(patch_arg);
+    patch_fn = NULL;
+  }
+
+  ref_difftest_regcpy(&ref_r, false);
 
   checkregs(&ref_r, this_pc);
 }

src/monitor/difftest/ref.c

@@ -5,16 +5,13 @@
 
 void cpu_exec(uint64_t);
 
-void difftest_memcpy_from_dut(paddr_t dest, void *src, size_t n) {
-  memcpy(guest_to_host(dest - PMEM_BASE), src, n);
+void difftest_memcpy(paddr_t addr, void *buf, size_t n, bool to_ref) {
+  if (to_ref) memcpy(guest_to_host(addr - PMEM_BASE), buf, n);
+  else memcpy(buf, guest_to_host(addr - PMEM_BASE), n);
 }
 
-void difftest_getregs(void *r) {
-  isa_difftest_getregs(r);
-}
-
-void difftest_setregs(const void *r) {
-  isa_difftest_setregs(r);
+void difftest_regcpy(void *dut, bool to_ref) {
+  isa_difftest_regcpy(dut, to_ref);
 }
 
 void difftest_exec(uint64_t n) {

tools/kvm-diff/src/kvm.c

@@ -336,39 +336,37 @@ static void run_protected_mode() {
   kvm_exec(10);
 }
 
-void difftest_memcpy_from_dut(paddr_t dest, void *src, size_t n) {
-  memcpy(vm.mem + dest, src, n);
+void difftest_memcpy(paddr_t addr, void *buf, size_t n, bool to_ref) {
+  if (to_ref) memcpy(vm.mem + addr, buf, n);
+  else memcpy(buf, vm.mem + addr, n);
 }
 
-void difftest_getregs(void *r) {
+void difftest_regcpy(void *r, bool to_ref) {
   struct kvm_regs *ref = &(vcpu.kvm_run->s.regs.regs);
   x86_CPU_state *x86 = r;
-  x86->eax = ref->rax;
-  x86->ebx = ref->rbx;
-  x86->ecx = ref->rcx;
-  x86->edx = ref->rdx;
-  x86->esp = ref->rsp;
-  x86->ebp = ref->rbp;
-  x86->esi = ref->rsi;
-  x86->edi = ref->rdi;
-  x86->pc  = ref->rip;
-}
-
-void difftest_setregs(const void *r) {
-  struct kvm_regs *ref = &(vcpu.kvm_run->s.regs.regs);
-  const x86_CPU_state *x86 = r;
-  ref->rax = x86->eax;
-  ref->rbx = x86->ebx;
-  ref->rcx = x86->ecx;
-  ref->rdx = x86->edx;
-  ref->rsp = x86->esp;
-  ref->rbp = x86->ebp;
-  ref->rsi = x86->esi;
-  ref->rdi = x86->edi;
-  ref->rip = x86->pc;
-  ref->rflags |= RFLAGS_TF;
-
-  vcpu.kvm_run->kvm_dirty_regs = KVM_SYNC_X86_REGS;
+  if (to_ref) {
+    ref->rax = x86->eax;
+    ref->rbx = x86->ebx;
+    ref->rcx = x86->ecx;
+    ref->rdx = x86->edx;
+    ref->rsp = x86->esp;
+    ref->rbp = x86->ebp;
+    ref->rsi = x86->esi;
+    ref->rdi = x86->edi;
+    ref->rip = x86->pc;
+    ref->rflags |= RFLAGS_TF;
+    vcpu.kvm_run->kvm_dirty_regs = KVM_SYNC_X86_REGS;
+  } else {
+    x86->eax = ref->rax;
+    x86->ebx = ref->rbx;
+    x86->ecx = ref->rcx;
+    x86->edx = ref->rdx;
+    x86->esp = ref->rsp;
+    x86->ebp = ref->rbp;
+    x86->esi = ref->rsi;
+    x86->edi = ref->rdi;
+    x86->pc  = ref->rip;
+  }
 }
 
 void difftest_exec(uint64_t n) {

tools/qemu-dl-diff/Makefile

+INC_DIR += ./include
+BUILD_DIR ?= ./build
+OBJ_DIR ?= $(BUILD_DIR)/obj
+BINARY ?= $(BUILD_DIR)/qemu-so
+
+.DEFAULT_GOAL = app
+
+# Compilation flags
+CC = gcc
+LD = gcc
+INCLUDES  = $(addprefix -I, $(INC_DIR))
+CFLAGS   += -O2 -fPIC -MMD -Wall -Werror -DNEMU_HOME=$(NEMU_HOME) $(INCLUDES)
+
+# Files to be compiled
+SRCS = $(shell find src/ -name "*.c")
+OBJS = $(SRCS:src/%.c=$(OBJ_DIR)/%.o)
+
+# Compilation patterns
+$(OBJ_DIR)/%.o: src/%.c
+	@echo + CC $<
+	@mkdir -p $(dir $@)
+	@$(CC) $(CFLAGS)  -c -o $@ $<
+
+
+# Depencies
+-include $(OBJS:.o=.d)
+
+# Some convinient rules
+
+.PHONY: app clean
+app: $(BINARY)
+
+$(BINARY): $(OBJS)
+	@echo + LD $@
+	@$(LD) -O2 -rdynamic -shared -fPIC -o $@ $^
+
+clean: 
+	-rm -rf $(BUILD_DIR)

tools/qemu-dl-diff/include/common.h

+#ifndef __COMMON_H__
+#define __COMMON_H__
+
+#include <stdint.h>
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+
+typedef uint8_t bool;
+#define true 1
+#define false 0
+
+typedef uint32_t paddr_t;
+
+#endif

tools/qemu-dl-diff/src/diff-test.c

+#include <common.h>
+#include <elf.h>
+#include <setjmp.h>
+
+static int (*qemu_cpu_memory_rw_debug)(void *cpu, long addr, uint8_t *buf, int len, int is_write) = NULL;
+static int (*qemu_gdb_write_register)(void *cpu, uint8_t *buf, int reg) = NULL;
+static int (*qemu_gdb_read_register)(void *cpu, uint8_t *buf, int reg) = NULL;
+static int (*qemu_cpu_exec)(void *) = NULL;
+static void (*qemu_do_interrupt_all)(void *cpu, int intno,
+    int is_int, int error_code, uint32_t next_eip, int is_hw) = NULL;
+static void *qemu_cpu = NULL;
+
+void difftest_memcpy(paddr_t addr, void *buf, size_t n, bool to_ref) {
+  int ret = qemu_cpu_memory_rw_debug(qemu_cpu, addr, buf, n, to_ref);
+  assert(ret == 0);
+}
+
+void difftest_regcpy(void *dut, bool to_ref) {
+  uint32_t *regs = dut;
+  int (*fn)(void *cpu, uint8_t *buf, int reg) =
+    (to_ref ? qemu_gdb_write_register : qemu_gdb_read_register);
+  for (int i = 0; i < 9; i ++) { fn(qemu_cpu, (void *)&regs[i], i); }
+}
+
+void difftest_raise_intr(uint64_t NO) {
+  qemu_do_interrupt_all(qemu_cpu, NO, 0, 0, 0, 1);
+}
+
+#define EXCP_INTERRUPT 0x10000
+#define EXCP_HLT       0x10001
+#define EXCP_DEBUG     0x10002
+#define EXCP_ATOMIC    0x10005
+void difftest_exec(uint64_t n) {
+  for (; n > 0; n --) {
+    int ret = qemu_cpu_exec(qemu_cpu);
+    switch (ret) {
+      case EXCP_ATOMIC:
+      case EXCP_INTERRUPT: n ++; // fall through
+      case EXCP_HLT:
+      case EXCP_DEBUG: break;
+      default: assert(0);
+    }
+  }
+}
+
+static jmp_buf jbuf = {};
+
+void difftest_init(int port) {
+  extern void dl_load(char *argv[]);
+  if (setjmp(jbuf) == 0) {
+    // first path
+    char *argv[] = {
+      "/usr/bin/qemu-system-i386",
+//      "/home/yzh/software/qemu-v3.1.0/i386-softmmu/qemu-system-i386",
+      "-nographic", "-S", "-serial", "none", "-monitor", "none",
+      "-cpu", "Broadwell",
+      NULL
+    };
+    dl_load(argv);
+  }
+}
+
+void* get_loaded_addr(char *sym, int type);
+
+void difftest_init_late() {
+  qemu_cpu_memory_rw_debug = get_loaded_addr("cpu_memory_rw_debug", STT_FUNC);
+  qemu_gdb_write_register = get_loaded_addr("gdb_write_register", STT_FUNC);
+  qemu_gdb_read_register = get_loaded_addr("gdb_read_register", STT_FUNC);
+  qemu_cpu_exec = get_loaded_addr("cpu_exec", STT_FUNC);
+  qemu_do_interrupt_all = get_loaded_addr("do_interrupt_all", STT_FUNC);
+
+  int (*qemu_cpu_single_step)(void *cpu, int enabled) = get_loaded_addr("cpu_single_step", STT_FUNC);
+  void (*qemu_mutex_unlock_iothread)() = get_loaded_addr("qemu_mutex_unlock_iothread", STT_FUNC);
+  void* (*qemu_get_cpu)(int) = get_loaded_addr("qemu_get_cpu", STT_FUNC);
+  int qemu_sstep_flags = *(int *)get_loaded_addr("sstep_flags", STT_OBJECT);
+
+  qemu_cpu = qemu_get_cpu(0);
+  assert(qemu_cpu);
+  qemu_cpu_single_step(qemu_cpu, qemu_sstep_flags);
+  qemu_mutex_unlock_iothread();
+
+  static uint8_t mbr[] = {
+    // start16:
+    0xfa,                           // cli
+    0x31, 0xc0,                     // xorw   %ax,%ax
+    0x8e, 0xd8,                     // movw   %ax,%ds
+    0x8e, 0xc0,                     // movw   %ax,%es
+    0x8e, 0xd0,                     // movw   %ax,%ss
+    0x0f, 0x01, 0x16, 0x44, 0x7c,   // lgdt   gdtdesc
+    0x0f, 0x20, 0xc0,               // movl   %cr0,%eax
+    0x66, 0x83, 0xc8, 0x01,         // orl    $CR0_PE,%eax
+    0x0f, 0x22, 0xc0,               // movl   %eax,%cr0
+    0xea, 0x1d, 0x7c, 0x08, 0x00,   // ljmp   $GDT_ENTRY(1),$start32
+
+    // start32:
+    0x66, 0xb8, 0x10, 0x00,         // movw   $0x10,%ax
+    0x8e, 0xd8,                     // movw   %ax, %ds
+    0x8e, 0xc0,                     // movw   %ax, %es
+    0x8e, 0xd0,                     // movw   %ax, %ss
+    0xeb, 0xfe,                     // jmp    7c27
+    0x8d, 0x76, 0x00,               // lea    0x0(%esi),%esi
+
+    // GDT
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0xff, 0xff, 0x00, 0x00, 0x00, 0x9a, 0xcf, 0x00,
+    0xff, 0xff, 0x00, 0x00, 0x00, 0x92, 0xcf, 0x00,
+
+    // GDT descriptor
+    0x17, 0x00, 0x2c, 0x7c, 0x00, 0x00
+  };
+
+  // put the MBR code to QEMU to enable protected mode
+  difftest_memcpy(0x7c00, mbr, sizeof(mbr), true);
+
+  // set cs:eip to 0000:7c00
+  uint32_t val = 0;
+  qemu_gdb_write_register(qemu_cpu, (void *)&val, 10); // cs
+  val = 0x7c00;
+  qemu_gdb_write_register(qemu_cpu, (void *)&val, 8); // pc
+
+  // execute enough instructions to enter protected mode
+  difftest_exec(20);
+
+  longjmp(jbuf, 1);
+}

tools/qemu-dl-diff/src/hack.c

+#define _GNU_SOURCE
+#include <common.h>
+#include <sys/mman.h>
+#include <elf.h>
+#include <link.h>
+
+#define ALIGN_UP(a, sz) ((((uintptr_t)a) + (sz) - 1) & ~((sz) - 1))
+
+static char *strtab = NULL;
+static int strtab_size = 0;
+static Elf64_Sym *symtab = NULL;
+static int symtab_nr_entry = 0;
+static uintptr_t elf_base = 0;
+static uintptr_t qemu_tls_size = 0;
+
+static void mprotect_page(uintptr_t addr, int prot) {
+  addr &= ~0xfffl;
+  int ret = mprotect((void *)addr, 4096, prot);
+  assert(ret == 0);
+}
+
+typedef struct {
+  char *name;
+  uintptr_t base;
+  uintptr_t tls_offset_diff;
+  int next_tls_modid;
+  char *debug_elf_path;
+} Info;
+
+typedef int (*ELF_sh_handler)(void *buf, int size, void **);
+typedef struct {
+  char *name;
+  ELF_sh_handler h;
+  void *userdata;
+} ELF_sh_callback;
+
+static int build_symtab(void *buf, int size, void **userdata) {
+  assert(size > 0);
+  symtab = buf;
+  symtab_nr_entry = size / sizeof(Elf64_Sym);
+  return 0;
+}
+
+static int build_strtab(void *buf, int size, void **userdata) {
+  assert(size > 0);
+  strtab = buf;
+  strtab_size = size;
+  return 0;
+}
+
+static void ELF_sh_foreach(char *filename, ELF_sh_callback *cb_list) {
+  FILE *fp = fopen(filename, "r");
+  assert(fp != NULL);
+
+  Elf64_Ehdr elf;
+  fread(&elf, sizeof(elf), 1, fp);
+  assert(elf.e_ident[EI_MAG0] == ELFMAG0);
+  assert(elf.e_ident[EI_MAG1] == ELFMAG1);
+  assert(elf.e_ident[EI_MAG2] == ELFMAG2);
+  assert(elf.e_ident[EI_MAG3] == ELFMAG3);
+
+  Elf64_Shdr *sh = NULL;
+  int shdr_size = sizeof(*sh) * elf.e_shnum;
+  sh = malloc(shdr_size);
+  assert(sh);
+  fseek(fp, elf.e_shoff, SEEK_SET);
+  int ret = fread(sh, shdr_size, 1, fp);
+  assert(ret == 1);
+
+  int shstrtab_size = sh[elf.e_shstrndx].sh_size;
+  assert(shstrtab_size > 0);
+  char *shstrtab = malloc(shstrtab_size);
+  assert(shstrtab);
+  fseek(fp, sh[elf.e_shstrndx].sh_offset, SEEK_SET);
+  ret = fread(shstrtab, sh[elf.e_shstrndx].sh_size, 1, fp);
+  assert(ret == 1);
+
+  int i;
+  for (i = 0; i < elf.e_shnum; i ++) {
+    ELF_sh_callback *cb;
+    for (cb = cb_list; cb->name != NULL; cb ++) {
+      if (strcmp(shstrtab + sh[i].sh_name, cb->name) == 0) {
+        void *buf = malloc(sh[i].sh_size);
+        assert(buf);
+        fseek(fp, sh[i].sh_offset, SEEK_SET);
+        int ret = fread(buf, sh[i].sh_size, 1, fp);
+        assert(ret == 1);
+        int needfree = cb->h(buf, sh[i].sh_size, &cb->userdata);
+        if (needfree) free(buf);
+      }
+    }
+  }
+  free(shstrtab);
+  free(sh);
+  fclose(fp);
+}
+
+static void parse_debug_elf(char *filename) {
+  ELF_sh_callback cb_list[3] = {
+    { .name = ".symtab", .h = build_symtab },
+    { .name = ".strtab", .h = build_strtab },
+    { .name = NULL},
+  };
+  ELF_sh_foreach(filename, cb_list);
+  assert(symtab != NULL && strtab != NULL);
+}
+
+static int fix_tls_offset(void *buf, int size, void **userdata) {
+  Info *info = *userdata;
+  Elf64_Rela *rela_dyn = NULL;
+  int nr_rela_dyn = 0;
+
+  Elf64_Dyn *dyn = buf;
+  int nr_dyn = size / sizeof(dyn[0]);
+  int i;
+  for (i = 0; i < nr_dyn; i ++ ) {
+    switch (dyn[i].d_tag) {
+      case DT_RELA:    rela_dyn = (void *)info->base + dyn[i].d_un.d_ptr; break;
+      case DT_RELASZ:  nr_rela_dyn = dyn[i].d_un.d_val / sizeof(rela_dyn[0]); break;
+      case DT_RELAENT: assert(dyn[i].d_un.d_val == sizeof(rela_dyn[0])); break;
+    }
+  }
+
+  for (i = 0; i < nr_rela_dyn; i ++) {
+    if (ELF64_R_TYPE(rela_dyn[i].r_info) == R_X86_64_TPOFF64) {
+      uintptr_t ptr = info->base + rela_dyn[i].r_offset;
+      mprotect_page(ptr, PROT_READ|PROT_WRITE);
+      *(uintptr_t *)ptr += info->tls_offset_diff;
+      mprotect_page(ptr, PROT_READ);
+    }
+  }
+  return 1;
+}
+
+static int get_build_id(void *buf, int size, void **userdata) {
+  assert(size > 0);
+  void *name, *desc;
+  Elf64_Nhdr *note = buf;
+  name = note->n_namesz == 0 ? NULL : buf + sizeof(*note);
+  assert(strcmp(name, ELF_NOTE_GNU) == 0);
+  assert(note->n_descsz == 160 / 8); // SHA1 is of 160-bit length
+  desc = note->n_descsz == 0 ? NULL :
+    buf + sizeof(*note) + ALIGN_UP(note->n_namesz, 4);
+  char *id = malloc(note->n_descsz + 1); // +1 for '\0'
+  memcpy(id, desc, note->n_descsz);
+  id[note->n_descsz] = '\0';
+  *userdata = id;
+  return 1;
+}
+
+static void parse_origin_elf(Info *info) {
+  ELF_sh_callback cb_list[3] = {
+    { .name = ".dynamic", fix_tls_offset },
+    { .name = ".note.gnu.build-id", .h = get_build_id },
+    { .name = NULL }
+  };
+
+  const char *prefix = "/usr/bin/";
+  bool need_dbgsym = (strncmp(info->name, prefix, strlen(prefix)) == 0);
+  if (!need_dbgsym) {
+    cb_list[1].name = NULL;
+  }
+
+  cb_list[0].userdata = info;
+
+  ELF_sh_foreach(info->name, cb_list);
+
+  if (!need_dbgsym) {
+    info->debug_elf_path = strdup(info->name);
+    return;
+  }
+
+  uint8_t *id = cb_list[1].userdata;
+  char *path = malloc(512);
+  int len = sprintf(path, "/usr/lib/debug/.build-id/%02x/", id[0]);
+  int i;
+  for (i = 1; i < 20; i ++) {
+    len += sprintf(path + len, "%02x", id[i]);
+  }
+  strcat(path, ".debug");
+  free(id);
+  info->debug_elf_path = path;
+}
+
+static uintptr_t get_sym_addr(const char *sym, int type) {
+  int i;
+  for (i = 0; i < symtab_nr_entry; i ++) {
+    if ((strcmp(strtab + symtab[i].st_name, sym) == 0) &&
+        ELF64_ST_TYPE(symtab[i].st_info) == type) {
+      return symtab[i].st_value;
+    }
+  }
+  assert(0);
+}
+
+void* get_loaded_addr(const char *sym, int type) {
+  return (void *)get_sym_addr(sym, type) + elf_base;
+}
+
+static int callback(struct dl_phdr_info *info, size_t size, void *data) {
+  Info *arg = data;
+  int found = 0;
+  if (strcmp(info->dlpi_name, arg->name) == 0) {
+    arg->base = info->dlpi_addr;
+    found = 1;
+  }
+
+  int i;
+  for (i = 0; i < info->dlpi_phnum; i ++) {
+    if (info->dlpi_phdr[i].p_type == PT_TLS) {
+      assert(info->dlpi_tls_modid == arg->next_tls_modid);
+      uintptr_t size = ALIGN_UP(info->dlpi_phdr[i].p_memsz, info->dlpi_phdr[i].p_align);
+      if (!found) {
+        arg->tls_offset_diff += size;
+        arg->next_tls_modid ++;
+      } else {
+        qemu_tls_size = size;
+      }
+      break;
+    }
+  }
+  return found;
+}
+
+static inline void hack_fun_entry(const char *funname, const void *code, int len, bool protect) {
+  void *fun = get_loaded_addr(funname, STT_FUNC);
+  if (len) {
+    mprotect_page((uintptr_t)fun, PROT_READ | PROT_WRITE | PROT_EXEC);
+    memcpy(fun, code, len);
+  }
+  if (protect) mprotect_page((uintptr_t)fun, PROT_READ | PROT_EXEC);
+}
+
+static void hack_entry() {
+  hack_fun_entry("main_loop_wait", NULL, 0, true);
+
+  void *volatile **tcg_ctxs = (void *)get_loaded_addr("tcg_ctxs", STT_OBJECT);
+  uintptr_t tcg_ctx = get_sym_addr("tcg_ctx", STT_TLS);
+  void *tp;
+  asm volatile ("mov %%fs:0, %0" : "=r"(tp));
+  void **this_tcg_ctx = tp - qemu_tls_size + tcg_ctx;
+
+  while ((*tcg_ctxs)[0] == NULL) usleep(1);
+  *this_tcg_ctx = (*tcg_ctxs)[0];
+
+  extern void difftest_init_late();
+  difftest_init_late();
+}
+
+static inline void hack_main_loop_wait() {
+  uint8_t code[] = {
+    0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // movabs $imm, %rax
+    0xff, 0xe0, // jmp *%rax
+  };
+  *(uintptr_t *)(code + 2) = (uintptr_t)hack_entry;
+  assert(sizeof(code) == 12);
+  hack_fun_entry("main_loop_wait", code, sizeof(code), false);
+}
+
+static inline void hack_fun_return_void(char *funname) {
+  const uint8_t code[] = { 0xc3 }; // ret
+  assert(sizeof(code) == 1);
+  hack_fun_entry(funname, code, sizeof(code), true);
+}
+
+static inline void hack_fun_return_1(char *funname) {
+  const uint8_t code[] = {
+    0xb8, 0x01, 0x00, 0x00, 0x00, // mov $0x1, %eax
+    0xc3, // ret
+  };
+  assert(sizeof(code) == 6);
+  hack_fun_entry(funname, code, sizeof(code), true);
+}
+
+static void hack_prepare(Info *info) {
+  parse_origin_elf(info);
+  if (access(info->debug_elf_path, R_OK) != 0) {
+    printf("File '%s' does not exist!\n", info->debug_elf_path);
+    printf("Make sure you are using QEMU installed by apt-get, "
+           "and you have already installed the debug symbol package for qemu\n");
+    assert(0);
+  }
+
+  parse_debug_elf(info->debug_elf_path);
+  free(info->debug_elf_path);
+
+  hack_main_loop_wait();
+  hack_fun_return_1("qemu_cpu_is_self");
+  hack_fun_return_void("os_setup_signal_handling");
+  hack_fun_return_1("x86_cpu_has_work");
+}
+
+void dl_load(char *argv[]) {
+  void *qemu = dlopen(argv[0], RTLD_LAZY);
+  assert(qemu);
+  int (*qemu_main)(int, char **, char **) = dlsym(qemu, "main");
+  assert(qemu_main);
+
+  Info info = { .name = argv[0], .next_tls_modid = 1, .tls_offset_diff = 0 };
+  dl_iterate_phdr(callback, &info);
+  elf_base = info.base;
+
+  hack_prepare(&info);
+
+  char **p = argv;
+  while (*p != NULL) p ++;
+  int argc = p - argv;
+  extern char **environ;
+  qemu_main(argc, argv, environ);
+  assert(0);
+}

tools/qemu-diff/src/diff-test.c → tools/qemu-socket-diff/src/diff-test.c

@@ -12,22 +12,23 @@ void gdb_exit();
 
 void init_isa();
 
-void difftest_memcpy_from_dut(paddr_t dest, void *src, size_t n) {
-  bool ok = gdb_memcpy_to_qemu(dest, src, n);
-  assert(ok == 1);
-}
-
-void difftest_getregs(void *r) {
-  union isa_gdb_regs qemu_r;
-  gdb_getregs(&qemu_r);
-  memcpy(r, &qemu_r, DIFFTEST_REG_SIZE);
+void difftest_memcpy(paddr_t addr, void *buf, size_t n, bool to_ref) {
+  assert(to_ref);
+  if (to_ref) {
+    bool ok = gdb_memcpy_to_qemu(addr, buf, n);
+    assert(ok == 1);
+  }
 }
 
-void difftest_setregs(const void *r) {
+void difftest_regcpy(void *dut, bool to_ref) {
   union isa_gdb_regs qemu_r;
   gdb_getregs(&qemu_r);
-  memcpy(&qemu_r, r, DIFFTEST_REG_SIZE);
-  gdb_setregs(&qemu_r);
+  if (to_ref) {
+    memcpy(&qemu_r, dut, DIFFTEST_REG_SIZE);
+    gdb_setregs(&qemu_r);
+  } else {
+    memcpy(dut, &qemu_r, DIFFTEST_REG_SIZE);
+  }
 }
 
 void difftest_exec(uint64_t n) {