Merge remote-tracking branch 'origin/master' into dev-prefetch-switch

src/main/scala/xiangshan/backend/CtrlBlock.scala

@@ -226,8 +226,10 @@ class CtrlBlock extends XSModule with HasCircularQueuePtrHelper {
   val flushReg = RegNext(flush)
 
   redirectGen.io.exuMispredict.zip(io.fromIntBlock.exuRedirect).map({case (x, y) =>
-    x.valid := y.valid && y.bits.redirect.cfiUpdate.isMisPred
-    x.bits := y.bits
+    val misPred = y.valid && y.bits.redirect.cfiUpdate.isMisPred
+    val killedByOlder = y.bits.uop.roqIdx.needFlush(backendRedirect, flush)
+    x.valid := RegNext(misPred && !killedByOlder, init = false.B)
+    x.bits := RegEnable(y.bits, y.valid)
   })
   redirectGen.io.loadRelay := io.fromLsBlock.replay
   redirectGen.io.flush := flushReg

src/main/scala/xiangshan/backend/roq/Roq.scala

@@ -473,9 +473,9 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   val misPredWb = Cat(VecInit((0 until numWbPorts).map(i =>
     io.exeWbResults(i).bits.redirect.cfiUpdate.isMisPred && io.exeWbResults(i).bits.redirectValid
   ))).orR()
-  val misPredBlockCounter = Reg(UInt(2.W))
+  val misPredBlockCounter = Reg(UInt(3.W))
   misPredBlockCounter := Mux(misPredWb,
-    "b11".U,
+    "b111".U,
     misPredBlockCounter >> 1.U
   )
   val misPredBlock = misPredBlockCounter(0)

src/main/scala/xiangshan/cache/DCacheWrapper.scala

@@ -252,10 +252,12 @@ class DCacheImp(outer: DCache) extends LazyModuleImp(outer) with HasDCacheParame
   // tilelink stuff
   bus.a <> missQueue.io.mem_acquire
   bus.e <> missQueue.io.mem_finish
+  missQueue.io.probe_req := bus.b.bits.address
 
   //----------------------------------------
   // probe
-  probeQueue.io.mem_probe <> bus.b
+  // probeQueue.io.mem_probe <> bus.b
+  block_decoupled(bus.b, probeQueue.io.mem_probe, missQueue.io.probe_block)
 
   //----------------------------------------
   // mainPipe

src/main/scala/xiangshan/cache/MissQueue.scala

@@ -58,9 +58,12 @@ class MissEntry(edge: TLEdgeOut) extends DCacheModule
 
     val pipe_req  = DecoupledIO(new MainPipeReq)
     val pipe_resp = Flipped(ValidIO(new MainPipeResp))
+
+    // block probe
+    val block_addr = ValidIO(UInt(PAddrBits.W))
   })
 
-  // MSHR:
+  // old MSHR:
   // 1. receive req
   // 2. send acquire req
   // 3. receive grant resp
@@ -72,7 +75,10 @@ class MissEntry(edge: TLEdgeOut) extends DCacheModule
   //    See Tilelink spec 1.8.1 page 69
   //    A slave should not issue a Probe if there is a pending GrantAck on the block. Once the Probe is
   //    issued, the slave should not issue further Probes on that block until it receives a ProbeAck.
-  val s_invalid :: s_refill_req :: s_refill_resp :: s_main_pipe_req :: s_main_pipe_resp :: s_mem_finish :: Nil = Enum(6)
+
+  // new MSHR:
+  // send finish to end the transaction before sending pipe_req
+  val s_invalid :: s_refill_req :: s_refill_resp :: s_mem_finish :: s_main_pipe_req :: s_main_pipe_resp :: s_release_entry :: Nil = Enum(7)
 
   val state = RegInit(s_invalid)
 
@@ -147,6 +153,9 @@ class MissEntry(edge: TLEdgeOut) extends DCacheModule
   io.pipe_req.valid := false.B
   io.pipe_req.bits  := DontCare
 
+  io.block_addr.valid := state === s_mem_finish || state === s_main_pipe_req || state === s_main_pipe_resp
+  io.block_addr.bits := req.addr
+
   when (state =/= s_invalid) {
     XSDebug("entry: %d state: %d\n", io.id, state)
     req.dump()
@@ -277,7 +286,7 @@ class MissEntry(edge: TLEdgeOut) extends DCacheModule
       grantack.bits := edge.GrantAck(io.mem_grant.bits)
       grant_param := io.mem_grant.bits.param
 
-      state := s_main_pipe_req
+      state := s_mem_finish
     }
   }
 
@@ -316,7 +325,7 @@ class MissEntry(edge: TLEdgeOut) extends DCacheModule
 
   when (state === s_main_pipe_resp) {
     when (io.pipe_resp.fire()) {
-      state := s_mem_finish
+      state := s_release_entry
     }
   }
 
@@ -326,9 +335,13 @@ class MissEntry(edge: TLEdgeOut) extends DCacheModule
 
     when (io.mem_finish.fire()) {
       grantack.valid := false.B
-      state := s_invalid
+      state := s_main_pipe_req
     }
   }
+
+  when (state === s_release_entry) {
+    state := s_invalid
+  }
 }
 
 
@@ -344,6 +357,10 @@ class MissQueue(edge: TLEdgeOut) extends DCacheModule with HasTLDump
 
     val pipe_req  = DecoupledIO(new MainPipeReq)
     val pipe_resp = Flipped(ValidIO(new MainPipeResp))
+
+    // block probe
+    val probe_req = Input(UInt(PAddrBits.W))
+    val probe_block = Output(Bool())
   })
 
   val pipe_req_arb = Module(new RRArbiter(new MainPipeReq, cfg.nMissEntries))
@@ -353,6 +370,7 @@ class MissQueue(edge: TLEdgeOut) extends DCacheModule with HasTLDump
   val primary_ready  = Wire(Vec(cfg.nMissEntries, Bool()))
   val secondary_ready  = Wire(Vec(cfg.nMissEntries, Bool()))
   val secondary_reject  = Wire(Vec(cfg.nMissEntries, Bool()))
+  val probe_block_vec = Wire(Vec(cfg.nMissEntries, Bool()))
 
   // try merging with existing reqs
   val merge = secondary_ready.asUInt.orR
@@ -390,6 +408,7 @@ class MissQueue(edge: TLEdgeOut) extends DCacheModule with HasTLDump
     primary_ready(i)    := entry.io.primary_ready
     secondary_ready(i)  := entry.io.secondary_ready
     secondary_reject(i) := entry.io.secondary_reject
+    probe_block_vec(i)  := entry.io.block_addr.valid && entry.io.block_addr.bits === io.probe_req
     entry.io.req        := io.req.bits
 
     // entry refill
@@ -437,6 +456,7 @@ class MissQueue(edge: TLEdgeOut) extends DCacheModule with HasTLDump
 
   io.pipe_req <> pipe_req_arb.io.out
 
+  io.probe_block := probe_block_vec.asUInt.orR
 
   // print all input/output requests for debug purpose
 
@@ -489,5 +509,9 @@ class MissQueue(edge: TLEdgeOut) extends DCacheModule with HasTLDump
     io.mem_finish.bits.dump
   }
 
+  when (io.probe_block) {
+    XSDebug(p"block probe req ${Hexadecimal(io.probe_req)}\n")
+  }
+
   XSPerf("dcache_miss", io.req.fire())
 }

src/test/csrc/goldenmem.cpp

@@ -2,6 +2,7 @@
 #include <stdlib.h>
 #include <sys/mman.h>
 #include <time.h>
+#include "compress.h"
 
 uint8_t *pmem;
 
@@ -14,7 +15,7 @@ void init_goldenmem() {
   }
   void* get_img_start();
   long get_img_size();
-  memcpy(pmem, get_img_start(), get_img_size());
+  nonzero_large_memcpy(pmem, get_img_start(), get_img_size());
 }
 
 void update_goldenmem(paddr_t addr, void *data, uint64_t mask, int len) {
@@ -79,4 +80,5 @@ inline word_t paddr_read(paddr_t addr, int len) {
 inline void paddr_write(paddr_t addr, word_t data, int len) {
   if (in_pmem(addr)) pmem_write(addr, data, len);
   else panic("write not in pmem!");
-}
\ No newline at end of file
+}
+