diff --git a/src/main/scala/xiangshan/backend/MemBlock.scala b/src/main/scala/xiangshan/backend/MemBlock.scala index c2fd34192315c156f128762ca808f064ad055071..926685a4784b1b58431409f9eef282a9f50ae399 100644 --- a/src/main/scala/xiangshan/backend/MemBlock.scala +++ b/src/main/scala/xiangshan/backend/MemBlock.scala @@ -287,6 +287,9 @@ class MemBlockImp(outer: MemBlock) extends LazyModuleImp(outer) // Lsq to load unit's rs + // passdown to lsq (load s1) + lsq.io.loadPaddrIn(i) <> loadUnits(i).io.lsq.loadPaddrIn + // passdown to lsq (load s2) lsq.io.loadIn(i) <> loadUnits(i).io.lsq.loadIn lsq.io.ldout(i) <> loadUnits(i).io.lsq.ldout diff --git a/src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala b/src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala index ba5e453daf69216f59f022501f4158ee1b4e29b2..0fbb3f6384eb2ca936dc25ccebbf228a590f05ba 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala @@ -57,6 +57,7 @@ class LsqWrappper(implicit p: Parameters) extends XSModule with HasDCacheParamet val hartId = Input(UInt(8.W)) val enq = new LsqEnqIO val brqRedirect = Flipped(ValidIO(new Redirect)) + val loadPaddrIn = Vec(LoadPipelineWidth, Flipped(Valid(new LqPaddrWriteBundle))) val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LqWriteBundle))) val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) val storeInRe = Vec(StorePipelineWidth, Input(new LsPipelineBundle())) @@ -116,6 +117,7 @@ class LsqWrappper(implicit p: Parameters) extends XSModule with HasDCacheParamet // load queue wiring loadQueue.io.brqRedirect <> io.brqRedirect + loadQueue.io.loadPaddrIn <> io.loadPaddrIn loadQueue.io.loadIn <> io.loadIn loadQueue.io.storeIn <> io.storeIn loadQueue.io.s2_load_data_forwarded <> io.s2_load_data_forwarded diff --git a/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala b/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala index 64d1ec2e263a0d5e7f56469b54d8d20d5d6c4dec..a3e41e81d14985c14b2c7f7a4ab71537a0ef11a1 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala @@ -69,6 +69,11 @@ class LqEnqIO(implicit p: Parameters) extends XSBundle { val resp = Vec(exuParameters.LsExuCnt, Output(new LqPtr)) } +class LqPaddrWriteBundle(implicit p: Parameters) extends XSBundle { + val paddr = Output(UInt(PAddrBits.W)) + val lqIdx = Output(new LqPtr) +} + class LqTriggerIO(implicit p: Parameters) extends XSBundle { val hitLoadAddrTriggerHitVec = Input(Vec(3, Bool())) val lqLoadAddrTriggerHitVec = Output(Vec(3, Bool())) @@ -84,6 +89,7 @@ class LoadQueue(implicit p: Parameters) extends XSModule val io = IO(new Bundle() { val enq = new LqEnqIO val brqRedirect = Flipped(ValidIO(new Redirect)) + val loadPaddrIn = Vec(LoadPipelineWidth, Flipped(Valid(new LqPaddrWriteBundle))) val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LqWriteBundle))) val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) val s2_load_data_forwarded = Vec(LoadPipelineWidth, Input(Bool())) @@ -189,6 +195,7 @@ class LoadQueue(implicit p: Parameters) extends XSModule */ for (i <- 0 until LoadPipelineWidth) { dataModule.io.wb.wen(i) := false.B + dataModule.io.paddr.wen(i) := false.B vaddrTriggerResultModule.io.wen(i) := false.B val loadWbIndex = io.loadIn(i).bits.uop.lqIdx.value @@ -296,6 +303,12 @@ class LoadQueue(implicit p: Parameters) extends XSModule vaddrTriggerResultModule.io.wen(i) := true.B } + when(io.loadPaddrIn(i).valid) { + dataModule.io.paddr.wen(i) := true.B + dataModule.io.paddr.waddr(i) := io.loadPaddrIn(i).bits.lqIdx.value + dataModule.io.paddr.wdata(i) := io.loadPaddrIn(i).bits.paddr + } + // vaddrModule write is delayed, as vaddrModule will not be read right after write vaddrModule.io.waddr(i) := RegNext(loadWbIndex) vaddrModule.io.wdata(i) := RegNext(io.loadIn(i).bits.vaddr) diff --git a/src/main/scala/xiangshan/mem/lsqueue/LoadQueueData.scala b/src/main/scala/xiangshan/mem/lsqueue/LoadQueueData.scala index c7ef37c100639cf3febb7326d0b701ae9ec8131b..709eea7d3020e47e2ecec1f75ab03d68b0b249b5 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/LoadQueueData.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/LoadQueueData.scala @@ -26,13 +26,16 @@ import xiangshan.cache.{DCacheWordIO, DCacheLineIO, MemoryOpConstants} import xiangshan.mem._ import xiangshan.backend.rob.RobPtr -class LQDataEntry(implicit p: Parameters) extends XSBundle { - val paddr = UInt(PAddrBits.W) +class LQDataEntryWoPaddr(implicit p: Parameters) extends XSBundle { val mask = UInt(8.W) val data = UInt(XLEN.W) val fwdMask = Vec(8, Bool()) } +class LQDataEntry(implicit p: Parameters) extends LQDataEntryWoPaddr { + val paddr = UInt(PAddrBits.W) +} + // Data module define // These data modules are like SyncDataModuleTemplate, but support cam-like ops @@ -42,7 +45,7 @@ class LQDataEntry(implicit p: Parameters) extends XSBundle { // * st-ld violation addr cam // * data release addr cam // * data refill addr cam -class LQPaddrModule(numEntries: Int, numRead: Int, numWrite: Int)(implicit p: Parameters) extends XSModule with HasDCacheParameters { +class LQPaddrModule(numEntries: Int, numRead: Int, numWrite: Int, numWBanks: Int)(implicit p: Parameters) extends XSModule with HasDCacheParameters { val io = IO(new Bundle { // normal read/write ports val raddr = Input(Vec(numRead, UInt(log2Up(numEntries).W))) @@ -61,6 +64,11 @@ class LQPaddrModule(numEntries: Int, numRead: Int, numWrite: Int)(implicit p: Pa val refillMmask = Output(Vec(numEntries, Bool())) }) + require(isPow2(numWBanks)) + require(numWBanks >= 2) + + val numEntryPerBank = numEntries / numWBanks + val data = Reg(Vec(numEntries, UInt((PAddrBits).W))) // read ports @@ -68,10 +76,50 @@ class LQPaddrModule(numEntries: Int, numRead: Int, numWrite: Int)(implicit p: Pa io.rdata(i) := data(RegNext(io.raddr(i))) } - // below is the write ports (with priorities) - for (i <- 0 until numWrite) { - when (io.wen(i)) { - data(io.waddr(i)) := io.wdata(i) + // write ports + val waddr_dec = io.waddr.map(a => UIntToOH(a)) + def selectBankMask(in: UInt, bank: Int): UInt = { + in((bank + 1) * numEntryPerBank - 1, bank * numEntryPerBank) + } + for (bank <- 0 until numWBanks) { + // write ports + // s0: write to bank level buffer + val s0_bank_waddr_dec = waddr_dec.map(a => selectBankMask(a, bank)) + val s0_bank_write_en = io.wen.zip(s0_bank_waddr_dec).map(w => w._1 && w._2.orR) + s0_bank_waddr_dec.zipWithIndex.map(a => + a._1.suggestName("s0_bank_waddr_dec" + bank + "_" + a._2) + ) + s0_bank_write_en.zipWithIndex.map(a => + a._1.suggestName("s0_bank_write_en" + bank + "_" + a._2) + ) + // s1: write data to entries + val s1_bank_waddr_dec = s0_bank_waddr_dec.zip(s0_bank_write_en).map(w => RegEnable(w._1, w._2)) + val s1_bank_wen = RegNext(VecInit(s0_bank_write_en)) + val s1_wdata = io.wdata.zip(s0_bank_write_en).map(w => RegEnable(w._1, w._2)) + s1_bank_waddr_dec.zipWithIndex.map(a => + a._1.suggestName("s1_bank_waddr_dec" + bank + "_" + a._2) + ) + s1_bank_wen.zipWithIndex.map(a => + a._1.suggestName("s1_bank_wen" + bank + "_" + a._2) + ) + s1_wdata.zipWithIndex.map(a => + a._1.suggestName("s1_wdata" + bank + "_" + a._2) + ) + + // entry write + for (entry <- 0 until numEntryPerBank) { + // write ports + val s1_entry_write_en_vec = s1_bank_wen.zip(s1_bank_waddr_dec).map(w => w._1 && w._2(entry)) + val s1_entry_write_en = VecInit(s1_entry_write_en_vec).asUInt.orR + val s1_entry_write_data = Mux1H(s1_entry_write_en_vec, s1_wdata) + when (s1_entry_write_en) { + data(bank * numEntryPerBank + entry) := s1_entry_write_data + } + s1_entry_write_en_vec.zipWithIndex.map(a => + a._1.suggestName("s1_entry_write_en_vec" + bank + "_" + entry + "_" + a._2) + ) + s1_entry_write_en.suggestName("s1_entry_write_en" + bank + "_" + entry) + s1_entry_write_data.suggestName("s1_entry_write_data" + bank + "_" + entry) } } @@ -119,10 +167,12 @@ class LQMaskModule(numEntries: Int, numRead: Int, numWrite: Int)(implicit p: Par io.rdata(i) := data(RegNext(io.raddr(i))) } - // below is the write ports (with priorities) - for (i <- 0 until numWrite) { - when (io.wen(i)) { - data(io.waddr(i)) := io.wdata(i) + // write ports + val waddr_dec = io.waddr.map(a => UIntToOH(a)) + for (j <- 0 until numEntries) { + val write_wen = io.wen.zip(waddr_dec).map(w => w._1 && w._2(j)) + when (VecInit(write_wen).asUInt.orR) { + data(j) := Mux1H(write_wen, io.wdata) } } @@ -190,6 +240,7 @@ class LQDataModule(numEntries: Int, numRead: Int, numWrite: Int)(implicit p: Par } // write ctrl info + // TODO: optimize that when (io.wen(i)) { fwdMask(io.waddr(i)) := io.fwdMaskWdata(i) } @@ -235,6 +286,11 @@ class LQDataModule(numEntries: Int, numRead: Int, numWrite: Int)(implicit p: Par // and their interconnect class LoadQueueDataWrapper(size: Int, wbNumRead: Int, wbNumWrite: Int)(implicit p: Parameters) extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper { val io = IO(new Bundle() { + val paddr = new Bundle() { + val wen = Vec(wbNumWrite, Input(Bool())) + val waddr = Input(Vec(wbNumWrite, UInt(log2Up(size).W))) + val wdata = Input(Vec(wbNumWrite, UInt(PAddrBits.W))) + } val wb = new Bundle() { val wen = Vec(wbNumWrite, Input(Bool())) val waddr = Input(Vec(wbNumWrite, UInt(log2Up(size).W))) @@ -285,7 +341,7 @@ class LoadQueueDataWrapper(size: Int, wbNumRead: Int, wbNumWrite: Int)(implicit }) // data module - val paddrModule = Module(new LQPaddrModule(size, numRead = LoadPipelineWidth+1, numWrite = LoadPipelineWidth)) + val paddrModule = Module(new LQPaddrModule(size, numRead = LoadPipelineWidth+1, numWrite = LoadPipelineWidth, numWBanks = LoadQueueNWriteBanks)) val maskModule = Module(new LQMaskModule(size, numRead = LoadPipelineWidth+1, numWrite = LoadPipelineWidth)) val dataModule = Module(new LQDataModule(size, numRead = LoadPipelineWidth+1, numWrite = LoadPipelineWidth+1)) @@ -319,21 +375,22 @@ class LoadQueueDataWrapper(size: Int, wbNumRead: Int, wbNumWrite: Int)(implicit maskModule.io.wen(i) := false.B dataModule.io.wen(i) := false.B - paddrModule.io.waddr(i) := io.wb.waddr(i) maskModule.io.waddr(i) := io.wb.waddr(i) dataModule.io.waddr(i) := io.wb.waddr(i) - paddrModule.io.wdata(i) := io.wb.wdata(i).paddr maskModule.io.wdata(i) := io.wb.wdata(i).mask dataModule.io.wdata(i) := io.wb.wdata(i).data dataModule.io.fwdMaskWdata(i) := io.wb.wdata(i).fwdMask.asUInt dataModule.io.paddrWdata(i) := io.wb.wdata(i).paddr when(io.wb.wen(i)){ - paddrModule.io.wen(i) := true.B maskModule.io.wen(i) := true.B dataModule.io.wen(i) := true.B } + + paddrModule.io.wen(i) := io.paddr.wen(i) + paddrModule.io.waddr(i) := io.paddr.waddr(i) + paddrModule.io.wdata(i) := io.paddr.wdata(i) }) // write port wbNumWrite diff --git a/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala b/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala index 0fe1a241efd87986fdaa1c58c43d5e220e520817..dc3302d1e86c1b4f7c5541b33986c737029411aa 100644 --- a/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala +++ b/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala @@ -28,6 +28,7 @@ import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} class LoadToLsqIO(implicit p: Parameters) extends XSBundle { val loadIn = ValidIO(new LqWriteBundle) + val loadPaddrIn = ValidIO(new LqPaddrWriteBundle) val ldout = Flipped(DecoupledIO(new ExuOutput)) val s2_load_data_forwarded = Output(Bool()) val s3_delayed_load_error = Output(Bool()) @@ -611,6 +612,11 @@ class LoadUnit(implicit p: Parameters) extends XSModule PipelineConnect(load_s1.io.out, load_s2.io.in, true.B, load_s1.io.out.bits.uop.robIdx.needFlush(io.redirect) || cancelPointerChasing) + // provide paddr for lq + io.lsq.loadPaddrIn.valid := load_s1.io.out.valid + io.lsq.loadPaddrIn.bits.lqIdx := load_s1.io.out.bits.uop.lqIdx + io.lsq.loadPaddrIn.bits.paddr := load_s1.io.lsuPAddr + // load s2 io.dcache.s2_kill := load_s2.io.dcache_kill // to kill mmio resp which are redirected load_s2.io.dcacheResp <> io.dcache.resp