[WIP] Lsq: fix stq forward & rollback logic

debug/Makefile

@@ -24,9 +24,7 @@ cpu:
 # ------------------------------------------------------------------
 
 cputest:
-	$(MAKE) -C $(AM_HOME)/tests/cputest $(ARCH) $(EMU_ARGS) run 2>&1 | tee > cpu.log
-	cat cpu.log | grep different
-	cat cpu.log | grep IPC
+	bash cputest.sh
 
 # bputest:
 # 	$(MAKE) -C $(AM_HOME)/tests/bputest $(ARCH) run 2>&1 | tee > bpu.log

src/main/scala/xiangshan/mem/Memend.scala

@@ -59,8 +59,8 @@ class LoadForwardQueryIO extends XSBundle {
   val forwardData = Input(Vec(8, UInt(8.W)))
 
   val lsroqIdx = Output(UInt(LsroqIdxWidth.W))
-  val lqIdx = Output(UInt(LoadQueueIdxWidth.W))
-  // val sqIdx = Output(UInt(LsroqIdxWidth.W))
+  // val lqIdx = Output(UInt(LoadQueueIdxWidth.W))
+  val sqIdx = Output(UInt(StoreQueueIdxWidth.W))
 }
 
 class MemToBackendIO extends XSBundle {

src/main/scala/xiangshan/mem/lsqueue/separated/LSQWrapper.scala

@@ -59,7 +59,6 @@ class LsqWrappper extends XSModule with HasDCacheParameters with NeedImpl {
     loadQueue.io.loadIn <> io.loadIn
     loadQueue.io.storeIn <> io.storeIn
     loadQueue.io.ldout <> io.ldout
-    loadQueue.io.forward <> io.forward
     loadQueue.io.commits <> io.commits
     loadQueue.io.rollback <> io.rollback
     loadQueue.io.dcache <> io.dcache
@@ -72,11 +71,12 @@ class LsqWrappper extends XSModule with HasDCacheParameters with NeedImpl {
     storeQueue.io.storeIn <> io.storeIn
     storeQueue.io.sbuffer <> io.sbuffer
     storeQueue.io.stout <> io.stout
-    storeQueue.io.forward <> io.forward
     storeQueue.io.commits <> io.commits
-    storeQueue.io.rollback <> io.rollback
     storeQueue.io.roqDeqPtr <> io.roqDeqPtr
     
+    loadQueue.io.forward <> io.forward
+    storeQueue.io.forward <> io.forward // overlap forwardMask & forwardData, DO NOT CHANGE SEQUENCE
+
     // naive uncache arbiter
     val s_idle :: s_load :: s_store :: Nil = Enum(3)
     val uncacheState = RegInit(s_idle)

src/main/scala/xiangshan/mem/lsqueue/separated/LoadQueue.scala

@@ -298,58 +298,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with NeedImpl {
     }
   })
 
-  // load forward query
-  // check over all lq entries and forward data from the first matched store
-  // TODO: FIXME
-  (0 until LoadPipelineWidth).map(i => {
-    io.forward(i).forwardMask := 0.U(8.W).asBools
-    io.forward(i).forwardData := DontCare
-
-    // Compare ringBufferTail (deqPtr) and forward.lqIdx, we have two cases:
-    // (1) if they have the same flag, we need to check range(tail, lqIdx)
-    // (2) if they have different flags, we need to check range(tail, LoadQueueSize) and range(0, lqIdx)
-    // Forward1: Mux(same_flag, range(tail, lqIdx), range(tail, LoadQueueSize))
-    // Forward2: Mux(same_flag, 0.U,                   range(0, lqIdx)    )
-    // i.e. forward1 is the target entries with the same flag bits and forward2 otherwise
-    val forwardMask1 = WireInit(VecInit(Seq.fill(8)(false.B)))
-    val forwardData1 = WireInit(VecInit(Seq.fill(8)(0.U(8.W))))
-    val forwardMask2 = WireInit(VecInit(Seq.fill(8)(false.B)))
-    val forwardData2 = WireInit(VecInit(Seq.fill(8)(0.U(8.W))))
-
-    val differentFlag = ringBufferTailExtended(InnerLoadQueueIdxWidth) =/= io.forward(i).lqIdx(InnerLoadQueueIdxWidth)
-    val forwardMask = ((1.U((LoadQueueSize + 1).W)) << io.forward(i).lqIdx(InnerLoadQueueIdxWidth - 1, 0)).asUInt - 1.U
-    val needForward1 = Mux(differentFlag, ~tailMask, tailMask ^ forwardMask)
-    val needForward2 = Mux(differentFlag, forwardMask, 0.U(LoadQueueSize.W))
-
-    // entry with larger index should have higher priority since it's data is younger
-    for (j <- 0 until LoadQueueSize) {
-      val needCheck = valid(j) && allocated(j) && // all valid terms need to be checked
-        io.forward(i).paddr(PAddrBits - 1, 3) === data(j).paddr(PAddrBits - 1, 3)
-      (0 until XLEN / 8).foreach(k => {
-        when (needCheck && data(j).mask(k)) {
-          when (needForward1(j)) {
-            forwardMask1(k) := true.B
-            forwardData1(k) := data(j).data(8 * (k + 1) - 1, 8 * k)
-          }
-          when (needForward2(j)) {
-            forwardMask2(k) := true.B
-            forwardData2(k) := data(j).data(8 * (k + 1) - 1, 8 * k)
-          }
-          XSDebug(needForward1(j) || needForward2(j),
-            p"forwarding $k-th byte ${Hexadecimal(data(j).data(8 * (k + 1) - 1, 8 * k))} " +
-            p"from ptr $j pc ${Hexadecimal(uop(j).cf.pc)}\n")
-        }
-      })
-    }
-
-    // merge forward lookup results
-    // forward2 is younger than forward1 and should have higher priority
-    (0 until XLEN / 8).map(k => {
-      io.forward(i).forwardMask(k) := forwardMask1(k) || forwardMask2(k)
-      io.forward(i).forwardData(k) := Mux(forwardMask2(k), forwardData2(k), forwardData1(k))
-    })
-  })
-
   // rollback check
   val rollback = Wire(Vec(StorePipelineWidth, Valid(new Redirect)))
 
@@ -396,6 +344,12 @@ class LoadQueue extends XSModule with HasDCacheParameters with NeedImpl {
     Mux(start(InnerLoadQueueIdxWidth) === end(InnerLoadQueueIdxWidth), xorMask, ~xorMask)
   }
 
+  // ignore data forward
+  (0 until LoadPipelineWidth).foreach(i => {
+    io.forward(i).forwardMask := DontCare
+    io.forward(i).forwardData := DontCare
+  })
+
   // store backward query and rollback
   //  val needCheck = Seq.fill(8)(WireInit(true.B))
   (0 until StorePipelineWidth).foreach(i => {

src/main/scala/xiangshan/mem/lsqueue/separated/StoreQueue.scala

@@ -19,7 +19,6 @@ class StoreQueue extends XSModule with HasDCacheParameters with NeedImpl {
     val stout = Vec(2, DecoupledIO(new ExuOutput)) // writeback store
     val forward = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO))
     val commits = Flipped(Vec(CommitWidth, Valid(new RoqCommit)))
-    val rollback = Output(Valid(new Redirect))
     val uncache = new DCacheWordIO
     val roqDeqPtr = Input(UInt(RoqIdxWidth.W))
     // val refill = Flipped(Valid(new DCacheLineReq ))
@@ -170,6 +169,59 @@ class StoreQueue extends XSModule with HasDCacheParameters with NeedImpl {
   val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, ringBufferHeadExtended)
   ringBufferTailExtended := nextTail
 
+  // load forward query
+  // check over all lq entries and forward data from the first matched store
+  (0 until LoadPipelineWidth).map(i => {
+    io.forward(i).forwardMask := 0.U(8.W).asBools
+    io.forward(i).forwardData := DontCare
+
+    // Compare ringBufferTail (deqPtr) and forward.sqIdx, we have two cases:
+    // (1) if they have the same flag, we need to check range(tail, sqIdx)
+    // (2) if they have different flags, we need to check range(tail, LoadQueueSize) and range(0, sqIdx)
+    // Forward1: Mux(same_flag, range(tail, sqIdx), range(tail, LoadQueueSize))
+    // Forward2: Mux(same_flag, 0.U,                   range(0, sqIdx)    )
+    // i.e. forward1 is the target entries with the same flag bits and forward2 otherwise
+    val forwardMask1 = WireInit(VecInit(Seq.fill(8)(false.B)))
+    val forwardData1 = WireInit(VecInit(Seq.fill(8)(0.U(8.W))))
+    val forwardMask2 = WireInit(VecInit(Seq.fill(8)(false.B)))
+    val forwardData2 = WireInit(VecInit(Seq.fill(8)(0.U(8.W))))
+
+    val differentFlag = ringBufferTailExtended(InnerStoreQueueIdxWidth) =/= io.forward(i).sqIdx(InnerStoreQueueIdxWidth)
+    val forwardMask = ((1.U((StoreQueueSize + 1).W)) << io.forward(i).sqIdx(InnerStoreQueueIdxWidth - 1, 0)).asUInt - 1.U
+    val needForward1 = Mux(differentFlag, ~tailMask, tailMask ^ forwardMask)
+    val needForward2 = Mux(differentFlag, forwardMask, 0.U(StoreQueueSize.W))
+
+    XSDebug("" + i + " f1 %b f2 %b sqIdx %d pa %x\n", needForward1, needForward2, io.forward(i).sqIdx, io.forward(i).paddr)
+
+    // entry with larger index should have higher priority since it's data is younger
+    for (j <- 0 until StoreQueueSize) {
+      val needCheck = valid(j) && allocated(j) && // all valid terms need to be checked
+        io.forward(i).paddr(PAddrBits - 1, 3) === data(j).paddr(PAddrBits - 1, 3)
+      (0 until XLEN / 8).foreach(k => {
+        when (needCheck && data(j).mask(k)) {
+          when (needForward1(j)) {
+            forwardMask1(k) := true.B
+            forwardData1(k) := data(j).data(8 * (k + 1) - 1, 8 * k)
+          }
+          when (needForward2(j)) {
+            forwardMask2(k) := true.B
+            forwardData2(k) := data(j).data(8 * (k + 1) - 1, 8 * k)
+          }
+          XSDebug(needForward1(j) || needForward2(j),
+            p"forwarding $k-th byte ${Hexadecimal(data(j).data(8 * (k + 1) - 1, 8 * k))} " +
+            p"from ptr $j pc ${Hexadecimal(uop(j).cf.pc)}\n")
+        }
+      })
+    }
+
+    // merge forward lookup results
+    // forward2 is younger than forward1 and should have higher priority
+    (0 until XLEN / 8).map(k => {
+      io.forward(i).forwardMask(k) := forwardMask1(k) || forwardMask2(k)
+      io.forward(i).forwardData(k) := Mux(forwardMask2(k), forwardData2(k), forwardData1(k))
+    })
+  })
+
   // CommitedStoreQueue is not necessary
   // send commited store inst to sbuffer
   // select up to 2 writebacked store insts
@@ -277,7 +329,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with NeedImpl {
 
   // Read vaddr for mem exception
   val mexcLsIdx = WireInit(0.U.asTypeOf(new LSIdx()))
-  val memExceptionAddr = WireInit(data(mexcLsIdx.lqIdx(InnerStoreQueueIdxWidth - 1, 0)).vaddr)
+  val memExceptionAddr = WireInit(data(mexcLsIdx.sqIdx(InnerStoreQueueIdxWidth - 1, 0)).vaddr)
   ExcitingUtils.addSink(mexcLsIdx, "EXECPTION_LSROQIDX")
   ExcitingUtils.addSource(memExceptionAddr, "EXECPTION_STORE_VADDR")
 

src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala

@@ -189,7 +189,7 @@ class LoadUnit extends XSModule {
   io.lsroq.forward.paddr := l4_out.bits.paddr
   io.lsroq.forward.mask := io.dcache.resp.bits.meta.mask
   io.lsroq.forward.lsroqIdx := l4_out.bits.uop.lsroqIdx
-  io.lsroq.forward.lqIdx := l4_out.bits.uop.lqIdx
+  io.lsroq.forward.sqIdx := l4_out.bits.uop.sqIdx
   io.lsroq.forward.uop := l4_out.bits.uop
   io.lsroq.forward.pc := l4_out.bits.uop.cf.pc
   io.lsroq.forward.valid := io.dcache.resp.valid //TODO: opt timing
@@ -197,7 +197,7 @@ class LoadUnit extends XSModule {
   io.sbuffer.paddr := l4_out.bits.paddr
   io.sbuffer.mask := io.dcache.resp.bits.meta.mask
   io.sbuffer.lsroqIdx := l4_out.bits.uop.lsroqIdx
-  io.sbuffer.lqIdx := l4_out.bits.uop.lqIdx
+  io.sbuffer.sqIdx := l4_out.bits.uop.sqIdx
   io.sbuffer.uop := DontCare
   io.sbuffer.pc := l4_out.bits.uop.cf.pc
   io.sbuffer.valid := l4_out.valid