StoreQueueData: put paddr into paddrModule

src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala

@@ -51,6 +51,8 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
   // val data = Reg(Vec(StoreQueueSize, new LsqEntry))
   val dataModule = Module(new StoreQueueData(StoreQueueSize, numRead = StorePipelineWidth, numWrite = StorePipelineWidth, numForward = StorePipelineWidth))
   dataModule.io := DontCare
+  val paddrModule = Module(new SQPaddrModule(StoreQueueSize, numRead = StorePipelineWidth, numWrite = StorePipelineWidth, numForward = StorePipelineWidth))
+  paddrModule.io := DontCare
   val vaddrModule = Module(new AsyncDataModuleTemplate(UInt(VAddrBits.W), StoreQueueSize, numRead = 1, numWrite = StorePipelineWidth))
   vaddrModule.io := DontCare
 
@@ -86,9 +88,9 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
       deqPtrExt
     )
   ))
-  val dataModuleRead = dataModule.io.rdata
   for (i <- 0 until StorePipelineWidth) {
     dataModule.io.raddr(i) := deqPtrExtNext(i).value
+    paddrModule.io.raddr(i) := deqPtrExtNext(i).value
   }
   vaddrModule.io.raddr(0) := io.exceptionAddr.lsIdx.sqIdx.value
 
@@ -129,6 +131,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     */
   for (i <- 0 until StorePipelineWidth) {
     dataModule.io.wen(i) := false.B
+    paddrModule.io.wen(i) := false.B
     vaddrModule.io.wen(i) := false.B
     when (io.storeIn(i).fire()) {
       val stWbIndex = io.storeIn(i).bits.uop.sqIdx.value
@@ -138,13 +141,17 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
 
       val storeWbData = Wire(new SQDataEntry)
       storeWbData := DontCare
-      storeWbData.paddr := io.storeIn(i).bits.paddr
       storeWbData.mask := io.storeIn(i).bits.mask
       storeWbData.data := io.storeIn(i).bits.data
+
       dataModule.io.waddr(i) := stWbIndex
       dataModule.io.wdata(i) := storeWbData
       dataModule.io.wen(i) := true.B
 
+      paddrModule.io.waddr(i) := stWbIndex
+      paddrModule.io.wdata(i) := io.storeIn(i).bits.paddr
+      paddrModule.io.wen(i) := true.B
+
       vaddrModule.io.waddr(i) := stWbIndex
       vaddrModule.io.wdata(i) := io.storeIn(i).bits.vaddr
       vaddrModule.io.wen(i) := true.B
@@ -193,15 +200,13 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     )
 
     // do real fwd query
-    dataModule.io.forwardQuery(
-      numForward = i,
-      paddr = io.forward(i).paddr,
-      needForward1 = needForward1,
-      needForward2 = needForward2
-    )
+    dataModule.io.needForward(i)(0) := needForward1 & paddrModule.io.forwardMmask(i).asUInt
+    dataModule.io.needForward(i)(1) := needForward2 & paddrModule.io.forwardMmask(i).asUInt
+
+    paddrModule.io.forwardMdata(i) := io.forward(i).paddr
 
-    io.forward(i).forwardMask := dataModule.io.forward(i).forwardMask
-    io.forward(i).forwardData := dataModule.io.forward(i).forwardData
+    io.forward(i).forwardMask := dataModule.io.forwardMask(i)
+    io.forward(i).forwardData := dataModule.io.forwardData(i)
   }
 
   /**
@@ -221,13 +226,13 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     !io.commits.isWalk
 
   io.uncache.req.bits.cmd  := MemoryOpConstants.M_XWR
-  io.uncache.req.bits.addr := dataModule.io.rdata(0).paddr // data(deqPtr) -> rdata(0)
+  io.uncache.req.bits.addr := paddrModule.io.rdata(0) // data(deqPtr) -> rdata(0)
   io.uncache.req.bits.data := dataModule.io.rdata(0).data
   io.uncache.req.bits.mask := dataModule.io.rdata(0).mask
 
   io.uncache.req.bits.meta.id       := DontCare
   io.uncache.req.bits.meta.vaddr    := DontCare
-  io.uncache.req.bits.meta.paddr    := dataModule.io.rdata(0).paddr
+  io.uncache.req.bits.meta.paddr    := paddrModule.io.rdata(0)
   io.uncache.req.bits.meta.uop      := uop(deqPtr)
   io.uncache.req.bits.meta.mmio     := true.B
   io.uncache.req.bits.meta.tlb_miss := false.B
@@ -256,7 +261,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
   io.mmioStout.valid := allocated(deqPtr) && datavalid(deqPtr) && !writebacked(deqPtr)
   io.mmioStout.bits.uop := uop(deqPtr)
   io.mmioStout.bits.uop.sqIdx := deqPtrExt(0)
-  io.mmioStout.bits.data := dataModuleRead(0).data // dataModuleRead.read(deqPtr)
+  io.mmioStout.bits.data := dataModule.io.rdata(0).data // dataModule.io.rdata.read(deqPtr)
   io.mmioStout.bits.redirectValid := false.B
   io.mmioStout.bits.redirect := DontCare
   io.mmioStout.bits.brUpdate := DontCare
@@ -291,9 +296,9 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     // if sbuffer.fire(), read next
     io.sbuffer(i).valid := allocated(ptr) && commited(ptr) && !mmio(ptr)
     io.sbuffer(i).bits.cmd  := MemoryOpConstants.M_XWR
-    io.sbuffer(i).bits.addr := dataModuleRead(i).paddr
-    io.sbuffer(i).bits.data := dataModuleRead(i).data
-    io.sbuffer(i).bits.mask := dataModuleRead(i).mask
+    io.sbuffer(i).bits.addr := paddrModule.io.rdata(i)
+    io.sbuffer(i).bits.data := dataModule.io.rdata(i).data
+    io.sbuffer(i).bits.mask := dataModule.io.rdata(i).mask
     io.sbuffer(i).bits.meta          := DontCare
     io.sbuffer(i).bits.meta.tlb_miss := false.B
     io.sbuffer(i).bits.meta.uop      := DontCare
@@ -385,7 +390,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
 
   for (i <- 0 until StoreQueueSize) {
     if (i % 4 == 0) XSDebug("")
-    XSDebug(false, true.B, "%x [%x] ", uop(i).cf.pc, dataModule.io.debug(i).paddr)
+    XSDebug(false, true.B, "%x ", uop(i).cf.pc)
     PrintFlag(allocated(i), "a")
     PrintFlag(allocated(i) && datavalid(i), "v")
     PrintFlag(allocated(i) && writebacked(i), "w")

src/main/scala/xiangshan/mem/lsqueue/StoreQueueData.scala

@@ -11,12 +11,52 @@ import xiangshan.mem._
 import xiangshan.backend.roq.RoqPtr
 
 
+// Data module define
+// These data modules are like SyncDataModuleTemplate, but support cam-like ops
+class SQPaddrModule(numEntries: Int, numRead: Int, numWrite: Int, numForward: Int) extends XSModule with HasDCacheParameters {
+  val io = IO(new Bundle {
+    val raddr = Input(Vec(numRead, UInt(log2Up(numEntries).W)))
+    val rdata = Output(Vec(numRead, UInt((PAddrBits).W)))
+    val wen   = Input(Vec(numWrite, Bool()))
+    val waddr = Input(Vec(numWrite, UInt(log2Up(numEntries).W)))
+    val wdata = Input(Vec(numWrite, UInt((PAddrBits).W)))
+    val forwardMdata = Input(Vec(numForward, UInt((PAddrBits).W)))
+    val forwardMmask = Output(Vec(numForward, Vec(numEntries, Bool())))
+  })
+
+  val data = Reg(Vec(numEntries, UInt((PAddrBits).W)))
+
+  // read ports
+  for (i <- 0 until numRead) {
+    io.rdata(i) := data(RegNext(io.raddr(i)))
+  }
+
+  // below is the write ports (with priorities)
+  for (i <- 0 until numWrite) {
+    when (io.wen(i)) {
+      data(io.waddr(i)) := io.wdata(i)
+    }
+  }
+  
+  // content addressed match
+  for (i <- 0 until numForward) {
+    for (j <- 0 until numEntries) {
+      io.forwardMmask(i)(j) := io.forwardMdata(i)(PAddrBits-1, 3) === data(j)(PAddrBits-1, 3)
+    }
+  }
+
+  // DataModuleTemplate should not be used when there're any write conflicts
+  for (i <- 0 until numWrite) {
+    for (j <- i+1 until numWrite) {
+      assert(!(io.wen(i) && io.wen(j) && io.waddr(i) === io.waddr(j)))
+    }
+  }
+}
+
 class SQDataEntry extends XSBundle {
-//   val vaddr = UInt(VAddrBits.W) // TODO: need opt
-  val paddr = UInt(PAddrBits.W)
+  // val paddr = UInt(PAddrBits.W)
   val mask = UInt(8.W)
   val data = UInt(XLEN.W)
-//   val exception = UInt(16.W) // TODO: opt size
 }
 
 class StoreQueueData(size: Int, numRead: Int, numWrite: Int, numForward: Int) extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper {
@@ -29,13 +69,8 @@ class StoreQueueData(size: Int, numRead: Int, numWrite: Int, numForward: Int) ex
     val debug = Vec(size, Output(new SQDataEntry))
 
     val needForward = Input(Vec(numForward, Vec(2, UInt(size.W))))
-    val forward = Vec(numForward, Flipped(new LoadForwardQueryIO))
-
-    def forwardQuery(numForward: Int, paddr: UInt, needForward1: Data, needForward2: Data): Unit = {
-      this.needForward(numForward)(0) := needForward1
-      this.needForward(numForward)(1) := needForward2
-      this.forward(numForward).paddr := paddr
-    }
+    val forwardMask = Vec(numForward, Output(Vec(8, Bool())))
+    val forwardData = Vec(numForward, Output(Vec(8, UInt(8.W))))
   })
 
   io := DontCare
@@ -72,32 +107,7 @@ class StoreQueueData(size: Int, numRead: Int, numWrite: Int, numForward: Int) ex
   // entry with larger index should have higher priority since it's data is younger
 
   (0 until numForward).map(i => {
-    val forwardMask1 = WireInit(VecInit(Seq.fill(8)(false.B)))
-    val forwardData1 = WireInit(VecInit(Seq.fill(8)(0.U(8.W))))
-    val forwardMask2 = WireInit(VecInit(Seq.fill(8)(false.B)))
-    val forwardData2 = WireInit(VecInit(Seq.fill(8)(0.U(8.W))))
-
-    for (j <- 0 until size) {
-      val needCheck = io.forward(i).paddr(PAddrBits - 1, 3) === data(j).paddr(PAddrBits - 1, 3)
-      (0 until XLEN / 8).foreach(k => {
-        when (needCheck && data(j).mask(k)) {
-          when (io.needForward(i)(0)(j)) {
-            forwardMask1(k) := true.B
-            forwardData1(k) := data(j).data(8 * (k + 1) - 1, 8 * k)
-          }
-          when (io.needForward(i)(1)(j)) {
-            forwardMask2(k) := true.B
-            forwardData2(k) := data(j).data(8 * (k + 1) - 1, 8 * k)
-          }
-          XSDebug(io.needForward(i)(0)(j) || io.needForward(i)(1)(j),
-            p"forwarding $k-th byte ${Hexadecimal(data(j).data(8 * (k + 1) - 1, 8 * k))} " +
-            p"from ptr $j\n")
-        }
-      })
-    }
-
     // parallel fwd logic
-    val paddrMatch = Wire(Vec(size, Bool()))
     val matchResultVec = Wire(Vec(size * 2, new FwdEntry))
 
     def parallelFwd(xs: Seq[Data]): Data = {
@@ -113,13 +123,14 @@ class StoreQueueData(size: Int, numRead: Int, numWrite: Int, numForward: Int) ex
       })
     }
 
-    for (j <- 0 until size) {
-      paddrMatch(j) := io.forward(i).paddr(PAddrBits - 1, 3) === data(j).paddr(PAddrBits - 1, 3)
-    }
+    // paddrMatch is now included in io.needForward
+    // for (j <- 0 until size) {
+    //   paddrMatch(j) := io.forward(i).paddr(PAddrBits - 1, 3) === data(j).paddr(PAddrBits - 1, 3)
+    // }
 
     for (j <- 0 until size) {
-      val needCheck0 = RegNext(paddrMatch(j) && io.needForward(i)(0)(j))
-      val needCheck1 = RegNext(paddrMatch(j) && io.needForward(i)(1)(j))
+      val needCheck0 = RegNext(io.needForward(i)(0)(j))
+      val needCheck1 = RegNext(io.needForward(i)(1)(j))
       (0 until XLEN / 8).foreach(k => {
         matchResultVec(j).mask(k) := needCheck0 && data(j).mask(k)
         matchResultVec(j).data(k) := data(j).data(8 * (k + 1) - 1, 8 * k)
@@ -130,8 +141,8 @@ class StoreQueueData(size: Int, numRead: Int, numWrite: Int, numForward: Int) ex
 
     val parallelFwdResult = parallelFwd(matchResultVec).asTypeOf(new FwdEntry)
 
-    io.forward(i).forwardMask := parallelFwdResult.mask
-    io.forward(i).forwardData := parallelFwdResult.data
+    io.forwardMask(i) := parallelFwdResult.mask
+    io.forwardData(i) := parallelFwdResult.data
 
   })