Merge remote-tracking branch 'origin/master' into replay-br-update

src/main/scala/xiangshan/backend/issue/ReservationStationNew.scala

@@ -170,7 +170,7 @@ class ReservationStationCtrl
   // redirect and feedback
   for (i <- 0 until iqSize) {
     val cnt = cntQueue(idxQueue(i))
-
+    when (!(deqIdx === i.U && deqValid)) {
     if (i != 0) { // TODO: combine the two case
       val nextIdx = i.U - moveMask(i-1)
       when (stateQueue(i)===s_replay) {
@@ -194,6 +194,7 @@ class ReservationStationCtrl
       }
       when (redHitVec(i)) { stateQueue(nextIdx) := s_idle }
     }}
+    }
   }
 
   // output
@@ -454,4 +455,4 @@ class ReservationStationData
       p"${Binary(io.ctrl.srcUpdate(i).asUInt)}|${uop(i).pdest}:${uop(i).ctrl.rfWen}:" +
       p"${uop(i).ctrl.fpWen}|${uop(i).roqIdx} |${Hexadecimal(uop(i).cf.pc)}\n")
   }
-}
\ No newline at end of file
+}

src/main/scala/xiangshan/cache/dtlb.scala

@@ -117,7 +117,7 @@ class TlbEntry extends TlbBundle {
 class TlbEntires(num: Int, tagLen: Int) extends TlbBundle {
   require(log2Up(num)==log2Down(num))
   /* vpn can be divide into three part */
-  // vpn: tagPart + addrPart
+  // vpn: tagPart(17bit) + addrPart(8bit) + cutLenPart(2bit)
   val cutLen  = log2Up(num)
 
   val tag     = UInt(tagLen.W) // NOTE: high part of vpn
@@ -127,9 +127,10 @@ class TlbEntires(num: Int, tagLen: Int) extends TlbBundle {
   val vs      = Vec(num, Bool())
 
   def tagClip(vpn: UInt, level: UInt) = { // full vpn => tagLen
-    Mux(level===0.U, Cat(vpn(vpnLen-1, vpnnLen*2+cutLen), 0.U(vpnnLen*2+cutLen)),
-    Mux(level===1.U, Cat(vpn(vpnLen-1, vpnnLen*1+cutLen), 0.U(vpnnLen*1+cutLen)),
-                     Cat(vpn(vpnLen-1, vpnnLen*0+cutLen), 0.U(vpnnLen*0+cutLen))))(tagLen-1, 0)
+    val tmp = Mux(level===0.U, Cat(vpn(vpnLen-1, vpnnLen*2+cutLen), 0.U(vpnnLen*2)),
+              Mux(level===1.U, Cat(vpn(vpnLen-1, vpnnLen*1+cutLen), 0.U(vpnnLen*1)),
+                               Cat(vpn(vpnLen-1, vpnnLen*0+cutLen), 0.U(vpnnLen*0))))
+    tmp(tmp.getWidth-1, tmp.getWidth-tagLen)
   }
 
   // NOTE: get insize idx
@@ -527,4 +528,4 @@ object TLB {
 
     tlb.io.ptw
   }
-}
\ No newline at end of file
+}

src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala

@@ -468,7 +468,8 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
   io.rollback := DontCare
   // Note that we use roqIdx - 1.U to flush the load instruction itself.
   // Thus, here if last cycle's roqIdx equals to this cycle's roqIdx, it still triggers the redirect.
-  io.rollback.valid := rollbackSelected.valid && (!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx))
+  io.rollback.valid := rollbackSelected.valid && (!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx)) &&
+    !(lastCycleRedirect.valid && (lastCycleRedirect.bits.isFlushPipe || lastCycleRedirect.bits.isException))
 
   io.rollback.bits.roqIdx := rollbackSelected.bits.roqIdx - 1.U
   io.rollback.bits.isReplay := true.B

src/test/csrc/compress.cpp

@@ -40,7 +40,7 @@ long snapshot_compressToFile(uint8_t *ptr, const char *filename, long buf_size)
     // assert(bytes_write % sizeof(long) == 0);
 
   }
-  printf("Write %lu bytes from gz stream in total\n", curr_size);
+  // printf("Write %lu bytes from gz stream in total\n", curr_size);
 
   delete [] temp_page;
 
@@ -65,7 +65,7 @@ long readFromGz(void* ptr, const char *file_name, long buf_size, uint8_t load_ty
 
   // Only load from RAM need check
   if (load_type == LOAD_RAM && (buf_size % chunk_size) != 0) {
-    printf("RAMSIZE must be divisible by chunk_size\n");
+    printf("buf_size must be divisible by chunk_size\n");
     assert(0);
   }
   
@@ -88,10 +88,10 @@ long readFromGz(void* ptr, const char *file_name, long buf_size, uint8_t load_ty
   }
 
   if(gzread(compressed_mem, temp_page, chunk_size) > 0) {
-    printf("File size is larger than RAMSIZE!\n");
+    printf("File size is larger than buf_size!\n");
     assert(0);
   }
-  printf("Read %lu bytes from gz stream in total\n", curr_size);
+  // printf("Read %lu bytes from gz stream in total\n", curr_size);
 
   delete [] temp_page;
 
@@ -101,3 +101,26 @@ long readFromGz(void* ptr, const char *file_name, long buf_size, uint8_t load_ty
   }
   return curr_size;
 }
+
+void nonzero_large_memcpy(const void* __restrict dest, const void* __restrict src, size_t n) {
+  uint64_t *_dest = (uint64_t *)dest;
+  uint64_t *_src  = (uint64_t *)src;
+  while (n >= sizeof(uint64_t)) {
+    if (*_src != 0) {
+      *_dest = *_src;
+    }
+    _dest++;
+    _src++;
+    n -= sizeof(uint64_t);
+  }
+  if (n > 0) {
+    uint8_t *dest8 = (uint8_t *)_dest;
+    uint8_t *src8  = (uint8_t *)_src;
+    while (n > 0) {
+      *dest8 = *src8;
+      dest8++;
+      src8++;
+      n--;
+    }
+  }
+}

src/test/csrc/compress.h

@@ -15,4 +15,6 @@ int isGzFile(const char *filename);
 long snapshot_compressToFile(uint8_t *ptr, const char *filename, long buf_size);
 long readFromGz(void* ptr, const char *file_name, long buf_size, uint8_t load_type);
 
+void nonzero_large_memcpy(const void* __restrict dest, const void* __restrict src, size_t n);
+
 #endif

src/test/csrc/emu.cpp

@@ -4,6 +4,7 @@
 #include <getopt.h>
 #include "ram.h"
 #include "zlib.h"
+#include "compress.h"
 
 void* get_ram_start();
 long get_ram_size();
@@ -14,15 +15,16 @@ void set_nemu_this_pc(uint64_t pc);
 static inline void print_help(const char *file) {
   printf("Usage: %s [OPTION...]\n", file);
   printf("\n");
-  printf("  -s, --seed=NUM        use this seed\n");
-  printf("  -C, --max-cycles=NUM  execute at most NUM cycles\n");
-  printf("  -I, --max-instr=NUM   execute at most NUM instructions\n");
-  printf("  -i, --image=FILE      run with this image file\n");
-  printf("  -b, --log-begin=NUM   display log from NUM th cycle\n");
-  printf("  -e, --log-end=NUM     stop display log at NUM th cycle\n");
+  printf("  -s, --seed=NUM             use this seed\n");
+  printf("  -C, --max-cycles=NUM       execute at most NUM cycles\n");
+  printf("  -I, --max-instr=NUM        execute at most NUM instructions\n");
+  printf("  -i, --image=FILE           run with this image file\n");
+  printf("  -b, --log-begin=NUM        display log from NUM th cycle\n");
+  printf("  -e, --log-end=NUM          stop display log at NUM th cycle\n");
   printf("      --load-snapshot=PATH   load snapshot from PATH\n");
-  printf("      --dump-wave       dump waveform when log is enabled\n");
-  printf("  -h, --help            print program help info\n");
+  printf("      --no-snapshot          disable saving snapshots\n");
+  printf("      --dump-wave            dump waveform when log is enabled\n");
+  printf("  -h, --help                 print program help info\n");
   printf("\n");
 }
 
@@ -32,6 +34,7 @@ inline EmuArgs parse_args(int argc, const char *argv[]) {
   const struct option long_options[] = {
     { "load-snapshot",  1, NULL,  0  },
     { "dump-wave",      0, NULL,  0  },
+    { "no-snapshot",    0, NULL,  0  },
     { "seed",           1, NULL, 's' },
     { "max-cycles",     1, NULL, 'C' },
     { "max-instr",      1, NULL, 'I' },
@@ -50,6 +53,7 @@ inline EmuArgs parse_args(int argc, const char *argv[]) {
         switch (long_index) {
           case 0: args.snapshot_path = optarg; continue;
           case 1: args.enable_waveform = true; continue;
+          case 2: args.enable_snapshot = false; continue;
         }
         // fall through
       default:
@@ -128,9 +132,12 @@ Emulator::~Emulator() {
   ram_finish();
 
 #ifdef VM_SAVABLE
-  snapshot_slot[0].save();
-  snapshot_slot[1].save();
-  printf("Please remove unused snapshots manually\n");
+  if (args.enable_snapshot && trapCode != STATE_GOODTRAP && trapCode != STATE_LIMIT_EXCEEDED) {
+    printf("Saving snapshots to file system. Please wait.\n");
+    snapshot_slot[0].save();
+    snapshot_slot[1].save();
+    printf("Please remove unused snapshots manually\n");
+  }
 #endif
 }
 
@@ -317,8 +324,8 @@ uint64_t Emulator::execute(uint64_t max_cycle, uint64_t max_instr) {
     }
 #ifdef VM_SAVABLE
     static int snapshot_count = 0;
-    if (trapCode != STATE_GOODTRAP && t - lasttime_snapshot > 1000 * SNAPSHOT_INTERVAL) {
-      // save snapshot every 10s
+    if (args.enable_snapshot && trapCode != STATE_GOODTRAP && t - lasttime_snapshot > 6000 * SNAPSHOT_INTERVAL) {
+      // save snapshot every 60s
       time_t now = time(NULL);
       snapshot_save(snapshot_filename(now));
       lasttime_snapshot = t;
@@ -443,7 +450,7 @@ void Emulator::snapshot_save(const char *filename) {
   char *buf = new char[size];
   ref_difftest_memcpy_from_ref(buf, 0x80000000, size);
   stream.unbuf_write(buf, size);
-  delete buf;
+  delete [] buf;
 
   struct SyncState sync_mastate;
   ref_difftest_get_mastatus(&sync_mastate);
@@ -484,7 +491,7 @@ void Emulator::snapshot_load(const char *filename) {
   char *buf = new char[size];
   stream.read(buf, size);
   ref_difftest_memcpy_from_dut(0x80000000, buf, size);
-  delete buf;
+  delete [] buf;
 
   struct SyncState sync_mastate;
   stream.read(&sync_mastate, sizeof(struct SyncState));

src/test/csrc/emu.h

@@ -14,6 +14,7 @@ struct EmuArgs {
   const char *image;
   const char *snapshot_path;
   bool enable_waveform;
+  bool enable_snapshot;
 
   EmuArgs() {
     seed = 0;
@@ -24,6 +25,7 @@ struct EmuArgs {
     snapshot_path = NULL;
     image = NULL;
     enable_waveform = false;
+    enable_snapshot = true;
   }
 };
 

src/test/csrc/ram.cpp

@@ -4,7 +4,6 @@
 #include "ram.h"
 #include "compress.h"
 
-#define RAMSIZE (256 * 1024 * 1024UL)
 
 #ifdef WITH_DRAMSIM3
 #include "cosimulation.h"
@@ -16,7 +15,7 @@ static long img_size = 0;
 void* get_img_start() { return &ram[0]; }
 long get_img_size() { return img_size; }
 void* get_ram_start() { return &ram[0]; }
-long get_ram_size() { return RAMSIZE; }
+long get_ram_size() { return EMU_RAM_SIZE; }
 
 #ifdef TLB_UNITTEST
 void addpageSv39() {
@@ -109,17 +108,17 @@ void init_ram(const char *img) {
   printf("The image is %s\n", img);
 
   // initialize memory using Linux mmap
-  printf("Using simulated %luMB RAM\n", RAMSIZE / (1024 * 1024));
-  ram = (uint64_t *)mmap(NULL, RAMSIZE, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
+  printf("Using simulated %luMB RAM\n", EMU_RAM_SIZE / (1024 * 1024));
+  ram = (uint64_t *)mmap(NULL, EMU_RAM_SIZE, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
   if (ram == (uint64_t *)MAP_FAILED) {
-    printf("Cound not mmap 0x%lx bytes\n", RAMSIZE);
+    printf("Cound not mmap 0x%lx bytes\n", EMU_RAM_SIZE);
     assert(0);
   }
 
   int ret;
   if (isGzFile(img)) {
     printf("Gzip file detected and loading image from extracted gz file\n");
-    img_size = readFromGz(ram, img, RAMSIZE, LOAD_RAM);
+    img_size = readFromGz(ram, img, EMU_RAM_SIZE, LOAD_RAM);
     assert(img_size >= 0);
   }
   else {
@@ -131,8 +130,8 @@ void init_ram(const char *img) {
 
     fseek(fp, 0, SEEK_END);
     img_size = ftell(fp);
-    if (img_size > RAMSIZE) {
-      img_size = RAMSIZE;
+    if (img_size > EMU_RAM_SIZE) {
+      img_size = EMU_RAM_SIZE;
     }
 
     fseek(fp, 0, SEEK_SET);
@@ -159,24 +158,24 @@ void init_ram(const char *img) {
 }
 
 void ram_finish() {
-  munmap(ram, RAMSIZE);
+  munmap(ram, EMU_RAM_SIZE);
 #ifdef WITH_DRAMSIM3
   dramsim3_finish();
 #endif
 }
 
 extern "C" uint64_t ram_read_helper(uint8_t en, uint64_t rIdx) {
-  if (en && rIdx >= RAMSIZE / sizeof(uint64_t)) {
-    rIdx %= RAMSIZE / sizeof(uint64_t);
+  if (en && rIdx >= EMU_RAM_SIZE / sizeof(uint64_t)) {
+    rIdx %= EMU_RAM_SIZE / sizeof(uint64_t);
   }
   return (en) ? ram[rIdx] : 0;
 }
 
 extern "C" void ram_write_helper(uint64_t wIdx, uint64_t wdata, uint64_t wmask, uint8_t wen) {
   if (wen) {
-    if (wIdx >= RAMSIZE / sizeof(uint64_t)) {
+    if (wIdx >= EMU_RAM_SIZE / sizeof(uint64_t)) {
       printf("ERROR: ram wIdx = 0x%lx out of bound!\n", wIdx);
-      assert(wIdx < RAMSIZE / sizeof(uint64_t));
+      assert(wIdx < EMU_RAM_SIZE / sizeof(uint64_t));
     }
     ram[wIdx] = (ram[wIdx] & ~wmask) | (wdata & wmask);
   }
@@ -201,7 +200,7 @@ struct dramsim3_meta {
 };
 
 void axi_read_data(const axi_ar_channel &ar, dramsim3_meta *meta) {
-  uint64_t address = ar.addr % RAMSIZE;
+  uint64_t address = ar.addr % EMU_RAM_SIZE;
   uint64_t beatsize = 1 << ar.size;
   uint8_t  beatlen  = ar.len + 1;
   uint64_t transaction_size = beatsize * beatlen;
@@ -350,7 +349,7 @@ void dramsim3_helper(axi_channel &axi) {
     axi.b.id = meta->id;
     // assert(axi.b.ready == 1);
     for (int i = 0; i < meta->len; i++) {
-      uint64_t address = wait_resp_b->req->address % RAMSIZE;
+      uint64_t address = wait_resp_b->req->address % EMU_RAM_SIZE;
       ram[address / sizeof(uint64_t) + i] = meta->data[i];
     }
     // printf("axi b channel fired\n");

src/test/csrc/ram.h

@@ -3,6 +3,8 @@
 
 #include "common.h"
 
+#define EMU_RAM_SIZE (256 * 1024 * 1024UL)
+
 void init_ram(const char *img);
 void ram_finish();
 

src/test/csrc/snapshot.cpp

@@ -15,20 +15,23 @@ void VerilatedSaveMem::save() {
   if (size == 0) return;
   trailer();
   flush();
-  if(size <= (512 * 1024 * 1024UL)){
-    FILE *fp = fopen(m_filename.c_str(), "w");
+  auto saved_filename = m_filename;
+  if (size <= (512 * 1024 * 1024UL)) {
+    FILE *fp = fopen(saved_filename.c_str(), "w");
     assert(fp != NULL);
     fwrite(buf, size, 1, fp);
     fclose(fp);
-  } else {
-    timeval s, e;
-    gettimeofday(&s, NULL);
-    snapshot_compressToFile(buf, (m_filename + ".gz").c_str(), size);
-    gettimeofday(&e, NULL);
-    printf("Compress cost time (msec.usec): %lf\n", calcTime(s, e));
+  }
+  else {
+    saved_filename = saved_filename + ".gz";
+    // timeval s, e;
+    // gettimeofday(&s, NULL);
+    snapshot_compressToFile(buf, saved_filename.c_str(), size);
+    // gettimeofday(&e, NULL);
+    // printf("Compress cost time (msec.usec): %lf\n", calcTime(s, e));
   }
   size = 0;
-  printf("save snapshot to %s...\n", m_filename.c_str());
+  printf("save snapshot to %s...\n", saved_filename.c_str());
 }
 
 void VerilatedRestoreMem::fill() {
@@ -79,7 +82,7 @@ void VerilatedRestoreMem::open(const char* filename) {
       gettimeofday(&s, NULL);
       size = readFromGz(buf, filename, buf_size, LOAD_SNAPSHOT);
       gettimeofday(&e, NULL);
-      printf("Uncompress cost time (msec.usec): %lf\n", calcTime(s, e));
+      // printf("Uncompress cost time (msec.usec): %lf\n", calcTime(s, e));
       assert(size > 0);
     } else {
       FILE *fp = fopen(filename, "r");

src/test/csrc/snapshot.h

@@ -5,33 +5,40 @@
 #include "VXSSimSoC.h"
 #include <verilated_save.h>
 #include <sys/mman.h>
+#include "compress.h"
+#include "ram.h"
 
-#define SNAPSHOT_SIZE (3 * 16 * 1024 * 1024 * 1024UL)
+#define SNAPSHOT_SIZE (3UL * EMU_RAM_SIZE)
 
 class VerilatedSaveMem : public VerilatedSerialize {
   const static long buf_size = SNAPSHOT_SIZE;
-  uint8_t *buf;
+  uint8_t *buf = NULL;
   long size;
 
 public:
   VerilatedSaveMem() {
-    buf = (uint8_t*)mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
-    if (buf == (uint8_t *)MAP_FAILED) {
-      printf("Cound not mmap 0x%lx bytes\n", SNAPSHOT_SIZE);
-      assert(0);
-    }
+    buf = NULL;
     size = 0;
   }
   ~VerilatedSaveMem() { }
 
   void init(const char *filename) {
+    if (buf != NULL) {
+      munmap(buf, SNAPSHOT_SIZE);
+      buf = NULL;
+    }
+    buf = (uint8_t*)mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
+    if (buf == (uint8_t *)MAP_FAILED) {
+      printf("Cound not mmap 0x%lx bytes\n", SNAPSHOT_SIZE);
+      assert(0);
+    }
     size = 0;
     m_filename = filename;
     header();
   }
 
   void unbuf_write(const void* __restrict datap, size_t size) VL_MT_UNSAFE_ONE {
-    memcpy(buf + this->size, datap, size);
+    nonzero_large_memcpy(buf + this->size, datap, size);
     this->size += size;
   }