lq: add 1 extra stage for lq data write (#1705)

Now lq data is divided into 8 banks by default. Write to lq
data takes 2 cycles to finish

Lq data will not be read in at least 2 cycles after write, so it is ok
to add this delay. For example:
T0: update lq meta, lq data write req start
T1: lq data write finish, new wbidx selected
T2: read lq data according to new wbidx selected

src/main/scala/xiangshan/mem/MaskedDataModule.scala

@@ -23,7 +23,12 @@ import utils._
 import xiangshan.cache._
 
 class MaskedSyncDataModuleTemplate[T <: Data](
-  gen: T, numEntries: Int, numRead: Int, numWrite: Int, numMRead: Int = 0, numMWrite: Int = 0
+  gen: T,
+  numEntries: Int,
+  numRead: Int,
+  numWrite: Int,
+  numMRead: Int = 0,
+  numMWrite: Int = 0
 ) extends Module {
   val io = IO(new Bundle {
     // address indexed sync read
@@ -77,3 +82,126 @@ class MaskedSyncDataModuleTemplate[T <: Data](
     }
   }
 }
+
+class MaskedBankedSyncDataModuleTemplate[T <: Data](
+  gen: T,
+  numEntries: Int,
+  numRead: Int,
+  numWrite: Int,
+  numMRead: Int = 0,
+  numMWrite: Int = 0,
+  numWBanks: Int = 2
+) extends Module {
+  val io = IO(new Bundle {
+    // address indexed sync read
+    val raddr = Input(Vec(numRead, UInt(log2Up(numEntries).W)))
+    val rdata = Output(Vec(numRead, gen))
+    // masked sync read (1H)
+    val mrmask = Input(Vec(numMRead, Vec(numEntries, Bool())))
+    val mrdata = Output(Vec(numMRead, gen))
+    // address indexed write
+    val wen   = Input(Vec(numWrite, Bool()))
+    val waddr = Input(Vec(numWrite, UInt(log2Up(numEntries).W)))
+    val wdata = Input(Vec(numWrite, gen))
+    // masked write
+    val mwmask = Input(Vec(numMWrite, Vec(numEntries, Bool())))
+    val mwdata = Input(Vec(numMWrite, gen))
+  })
+
+  require(isPow2(numWBanks))
+  require(numWBanks >= 2)
+
+  val numEntryPerBank = numEntries / numWBanks
+
+  val data = Reg(Vec(numEntries, gen))
+
+  // read ports
+  for (i <- 0 until numRead) {
+    val raddr_dec = RegNext(UIntToOH(io.raddr(i)))
+    io.rdata(i) := Mux1H(raddr_dec, data)
+  }
+
+  // masked read ports
+  for (i <- 0 until numMRead) {
+    io.mrdata(i) := Mux1H(RegNext(io.mrmask(i)), data)
+  }
+
+  val waddr_dec = io.waddr.map(a => UIntToOH(a))
+
+  def selectBankMask(in: UInt, bank: Int): UInt = {
+    in((bank + 1) * numEntryPerBank - 1, bank * numEntryPerBank)
+  }
+
+  for (bank <- 0 until numWBanks) {
+    // write ports
+    // s0: write to bank level buffer
+    val s0_bank_waddr_dec = waddr_dec.map(a => selectBankMask(a, bank))
+    val s0_bank_write_en = io.wen.zip(s0_bank_waddr_dec).map(w => w._1 && w._2.orR)
+    s0_bank_waddr_dec.zipWithIndex.map(a =>
+      a._1.suggestName("s0_bank_waddr_dec" + bank + "_" + a._2)
+    )
+    s0_bank_write_en.zipWithIndex.map(a =>
+      a._1.suggestName("s0_bank_write_en" + bank + "_" + a._2)
+    )
+    // s1: write data to entries
+    val s1_bank_waddr_dec = s0_bank_waddr_dec.zip(s0_bank_write_en).map(w => RegEnable(w._1, w._2))
+    val s1_bank_wen = RegNext(VecInit(s0_bank_write_en))
+    val s1_wdata = io.wdata.zip(s0_bank_write_en).map(w => RegEnable(w._1, w._2))
+    s1_bank_waddr_dec.zipWithIndex.map(a =>
+      a._1.suggestName("s1_bank_waddr_dec" + bank + "_" + a._2)
+    )
+    s1_bank_wen.zipWithIndex.map(a =>
+      a._1.suggestName("s1_bank_wen" + bank + "_" + a._2)
+    )
+    s1_wdata.zipWithIndex.map(a =>
+      a._1.suggestName("s1_wdata" + bank + "_" + a._2)
+    )
+    // masked write ports
+    // s0: write to bank level buffer
+    val s0_bank_mwmask = io.mwmask.map(a => selectBankMask(a.asUInt, bank))
+    val s0_bank_mwrite_en = s0_bank_mwmask.map(w => w.orR)
+    s0_bank_mwmask.zipWithIndex.map(a =>
+      a._1.suggestName("s0_bank_mwmask" + bank + "_" + a._2)
+    )
+    s0_bank_mwrite_en.zipWithIndex.map(a =>
+      a._1.suggestName("s0_bank_mwrite_en" + bank + "_" + a._2)
+    )
+    // s1: write data to entries
+    val s1_bank_mwmask = s0_bank_mwmask.map(a => RegNext(a))
+    val s1_mwdata = io.mwdata.zip(s0_bank_mwrite_en).map(w => RegEnable(w._1, w._2))
+    s1_bank_mwmask.zipWithIndex.map(a =>
+      a._1.suggestName("s1_bank_mwmask" + bank + "_" + a._2)
+    )
+    s1_mwdata.zipWithIndex.map(a =>
+      a._1.suggestName("s1_mwdata" + bank + "_" + a._2)
+    )
+
+    // entry write
+    for (entry <- 0 until numEntryPerBank) {
+      // write ports
+      val s1_entry_write_en_vec = s1_bank_wen.zip(s1_bank_waddr_dec).map(w => w._1 && w._2(entry))
+      val s1_entry_write_en = VecInit(s1_entry_write_en_vec).asUInt.orR
+      val s1_entry_write_data = Mux1H(s1_entry_write_en_vec, s1_wdata)
+      // masked write ports
+      val s1_bank_mwrite_en_vec = s1_bank_mwmask.map(_(entry))
+      val s1_bank_mwrite_en = VecInit(s1_bank_mwrite_en_vec).asUInt.orR
+      val s1_bank_mwrite_data = Mux1H(s1_bank_mwrite_en_vec, s1_mwdata)
+      when (s1_entry_write_en || s1_bank_mwrite_en) {
+        data(bank * numEntryPerBank + entry) := Mux1H(
+          Seq(s1_entry_write_en, s1_bank_mwrite_en), 
+          Seq(s1_entry_write_data, s1_bank_mwrite_data)
+        )
+      }
+      s1_entry_write_en_vec.zipWithIndex.map(a =>
+        a._1.suggestName("s1_entry_write_en_vec" + bank + "_" + entry + "_" + a._2)
+      )
+      s1_bank_mwrite_en_vec.zipWithIndex.map(a =>
+        a._1.suggestName("s1_bank_mwrite_en_vec" + bank + "_" + entry + "_" + a._2)
+      )
+      s1_entry_write_en.suggestName("s1_entry_write_en" + bank + "_" + entry)
+      s1_entry_write_data.suggestName("s1_entry_write_data" + bank + "_" + entry)
+      s1_bank_mwrite_en.suggestName("s1_bank_mwrite_en" + bank + "_" + entry)
+      s1_bank_mwrite_data.suggestName("s1_bank_mwrite_data" + bank + "_" + entry)
+    }
+  }
+}

src/main/scala/xiangshan/mem/lsqueue/LoadQueueData.scala

@@ -166,7 +166,9 @@ class LQDataModule(numEntries: Int, numRead: Int, numWrite: Int)(implicit p: Par
     val refillData = Input(UInt(l1BusDataWidth.W))
   })
 
-  val data8 = Seq.fill(8)(Module(new MaskedSyncDataModuleTemplate(UInt(8.W), numEntries, numRead, numWrite, numMWrite = refillWords)))
+  val data8 = Seq.fill(8)(Module(new MaskedBankedSyncDataModuleTemplate(
+    UInt(8.W), numEntries, numRead, numWrite, numMWrite = refillWords, numWBanks = LoadQueueNWriteBanks
+  )))
   val fwdMask = Reg(Vec(numEntries, UInt(8.W)))
   val wordIndex = Reg(Vec(numEntries, UInt((refillOffBits - wordOffBits).W)))
 

src/main/scala/xiangshan/mem/lsqueue/StoreQueueData.scala

@@ -125,7 +125,7 @@ class SQData8Module(numEntries: Int, numRead: Int, numWrite: Int, numForward: In
 
   require(isPow2(StoreQueueNWriteBanks))
   require(StoreQueueNWriteBanks > 1)
-  def get_bank_num(in: UInt): UInt = in(log2Up(StoreQueueNWriteBanks) -1, 0)
+  def get_bank(in: UInt): UInt = in(log2Up(StoreQueueNWriteBanks) -1, 0)
   def get_bank_index(in: UInt): UInt = in >> log2Up(StoreQueueNWriteBanks)
   def get_vec_index(index: Int, bank: Int): Int = {
     (index << log2Up(StoreQueueNWriteBanks)) + bank
@@ -140,7 +140,7 @@ class SQData8Module(numEntries: Int, numRead: Int, numWrite: Int, numForward: In
   // })
   (0 until numWrite).map(i => {
     (0 until StoreQueueNWriteBanks).map(bank => {
-      val s0_wen = io.data.wen(i) && get_bank_num(io.data.waddr(i)) === bank.U
+      val s0_wen = io.data.wen(i) && get_bank(io.data.waddr(i)) === bank.U
       val s1_wen = RegNext(s0_wen)
       val s1_wdata = RegEnable(io.data.wdata(i), s0_wen)
       val s1_waddr = RegEnable(get_bank_index(io.data.waddr(i)), s0_wen)
@@ -164,7 +164,7 @@ class SQData8Module(numEntries: Int, numRead: Int, numWrite: Int, numForward: In
   // })
   (0 until numWrite).map(i => {
     (0 until StoreQueueNWriteBanks).map(bank => {
-      val s0_wen = io.mask.wen(i) && get_bank_num(io.mask.waddr(i)) === bank.U
+      val s0_wen = io.mask.wen(i) && get_bank(io.mask.waddr(i)) === bank.U
       val s1_wen = RegNext(s0_wen)
       val s1_wdata = RegEnable(io.mask.wdata(i), s0_wen)
       val s1_waddr = RegEnable(get_bank_index(io.mask.waddr(i)), s0_wen)