PTW: rewrite ptw for multiple requests support (#811)

* PTW: add ptw multi-processing graph

* [WIP] PTW: try to add miss queue, failed for complexity and not very useful

* [WIP] PTW: rewrite ptw for multi req support

* PTW: remove some assert, fix level init bug

* PTW: itlb has highter priority than dtlb

* PTW: fix bug that mix cache's resp logic

* PTW: fix stupid bug that mix .U and .W

* PTW: replay will not be blocked if fsm empty

* PTW: miss queue req may return miss queue

In the before design, only miss queue req can go into
fsm, and would not be blocked.
Now, to simplify design, miss queue req are just the
same with new req, may blocked, going to fsm or miss queue.

* PTW: fix ptw filter iss valid bug

* PTW.fsm: fix bug that should not mem.req when sfenceLatch

* PTW: fix ptw sfenceLatch's bug

* PTW: add some perf counters

* PTW: fix bug in filter enq ptr logic

* PTW: fix bug of sfence in ptw

* test: add current branch to ci-test, tmp

* PTW: fix bug of cache's hit logic and fsm's pf

* PTW: fix bug of filter's enq and block* signal

* PTW: fix bug of filter's pteResp filter

* PTW: add some assert of filter's counter

* PTW: fix bug of filter's enq logic

* PTW: set PTWMSHRSIZE 16

* PTW: fix naive perf counter's bug

* PTW: set PTWMSHRSIZE 8

* PTW: set PTWMSHRSIZE 32

* Revert "PTW: set PTWMSHRSIZE 32"

This reverts commit fd3981ae8bbb015c6cd398c4db60486d39fc92ef.

* Revert "test: add current branch to ci-test, tmp"

This reverts commit 8a7a8a494d5c05789e05a385a9fc7791a8ffef2f.

src/main/scala/xiangshan/Parameters.scala

@@ -92,6 +92,7 @@ case class XSCoreParameters
   PtwSPEntrySize: Int = 16,
   PtwL1EntrySize: Int = 16,
   PtwL2EntrySize: Int = 2048, //(256 * 8)
+  PtwMissQueueSize: Int = 8,
   NumPerfCounters: Int = 16,
   icacheParameters: ICacheParameters = ICacheParameters(
     tagECC = Some("parity"),
@@ -227,6 +228,7 @@ trait HasXSParameter {
   val PtwSPEntrySize = coreParams.PtwSPEntrySize
   val PtwL1EntrySize = coreParams.PtwL1EntrySize
   val PtwL2EntrySize = coreParams.PtwL2EntrySize
+  val PtwMissQueueSize = coreParams.PtwMissQueueSize
   val NumPerfCounters = coreParams.NumPerfCounters
 
   val instBytes = if (HasCExtension) 2 else 4

src/main/scala/xiangshan/XSCore.scala

@@ -7,7 +7,7 @@ import xiangshan.backend.fu.HasExceptionNO
 import xiangshan.backend.dispatch.DispatchParameters
 import xiangshan.frontend._
 import xiangshan.mem._
-import xiangshan.cache.{DCacheParameters, ICacheParameters, L1plusCacheWrapper, L1plusCacheParameters, PTWWrapper, PTWRepeater}
+import xiangshan.cache.{DCacheParameters, ICacheParameters, L1plusCacheWrapper, L1plusCacheParameters, PTWWrapper, PTWRepeater, PTWFilter}
 import xiangshan.cache.prefetch._
 import chipsalliance.rocketchip.config
 import chipsalliance.rocketchip.config.Parameters
@@ -199,13 +199,13 @@ class XSCoreImp(outer: XSCore) extends LazyModuleImp(outer)
   memBlock.io.lsqio.exceptionAddr.isStore := CommitType.lsInstIsStore(ctrlBlock.io.roqio.exception.bits.uop.ctrl.commitType)
 
   val itlbRepeater = Module(new PTWRepeater())
-  val dtlbRepeater = Module(new PTWRepeater())
+  val dtlbRepeater = Module(new PTWFilter(LoadPipelineWidth + StorePipelineWidth, PtwMissQueueSize))
   itlbRepeater.io.tlb <> frontend.io.ptw
   dtlbRepeater.io.tlb <> memBlock.io.ptw
   itlbRepeater.io.sfence <> integerBlock.io.fenceio.sfence
   dtlbRepeater.io.sfence <> integerBlock.io.fenceio.sfence
-  ptw.io.tlb(0) <> dtlbRepeater.io.ptw
-  ptw.io.tlb(1) <> itlbRepeater.io.ptw
+  ptw.io.tlb(0) <> itlbRepeater.io.ptw
+  ptw.io.tlb(1) <> dtlbRepeater.io.ptw
   ptw.io.sfence <> integerBlock.io.fenceio.sfence
   ptw.io.csr <> integerBlock.io.csrio.tlb
 

src/main/scala/xiangshan/backend/MemBlock.scala

@@ -72,7 +72,7 @@ class MemBlockImp(outer: MemBlock) extends LazyModuleImp(outer)
 
     val ldFastWakeUpInt = Flipped(new WakeUpBundle(exuParameters.LduCnt, 0))
 
-    val ptw = new TlbPtwIO
+    val ptw = new TlbPtwIO(LoadPipelineWidth + StorePipelineWidth)
     val sfence = Input(new SfenceBundle)
     val tlbCsr = Input(new TlbCsrBundle)
     val fenceToSbuffer = Flipped(new FenceToSbuffer)

src/main/scala/xiangshan/cache/TLB.scala

@@ -267,18 +267,20 @@ class BlockTlbRequestIO()(implicit p: Parameters) extends TlbBundle {
   val resp = Flipped(DecoupledIO(new TlbResp))
 }
 
-class TlbPtwIO(implicit p: Parameters) extends TlbBundle {
-  val req = DecoupledIO(new PtwReq)
+class TlbPtwIO(Width: Int = 1)(implicit p: Parameters) extends TlbBundle {
+  val req = Vec(Width, DecoupledIO(new PtwReq))
   val resp = Flipped(DecoupledIO(new PtwResp))
 
+  override def cloneType: this.type = (new TlbPtwIO(Width)).asInstanceOf[this.type]
+
   override def toPrintable: Printable = {
-    p"req:${req.valid} ${req.ready} ${req.bits} | resp:${resp.valid} ${resp.ready} ${resp.bits}"
+    p"req(0):${req(0).valid} ${req(0).ready} ${req(0).bits} | resp:${resp.valid} ${resp.ready} ${resp.bits}"
   }
 }
 
 class TlbIO(Width: Int)(implicit p: Parameters) extends TlbBundle {
   val requestor = Vec(Width, Flipped(new TlbRequestIO))
-  val ptw = new TlbPtwIO
+  val ptw = new TlbPtwIO(Width)
   val sfence = Input(new SfenceBundle)
   val csr = Input(new TlbCsrBundle)
 
@@ -358,19 +360,25 @@ class TLB(Width: Int, isDtlb: Boolean)(implicit p: Parameters) extends TlbModule
       val refillIdx = sRefillIdx
       refillIdx.suggestName(s"SuperRefillIdx")
 
-      sv(refillIdx) := true.B
-      sMeta(refillIdx).apply(
-        vpn = resp.entry.tag,
-        level = resp.entry.level.getOrElse(0.U)
-      )
-      sData(refillIdx).apply(
-        ppn   = resp.entry.ppn,
-        level = resp.entry.level.getOrElse(0.U),
-        perm  = VecInit(resp.entry.perm.getOrElse(0.U)).asUInt,
-        pf    = resp.pf
-      )
-      sReplace.access(sRefillIdx)
-      XSDebug(p"Refill superpage: idx:${refillIdx} entry:${resp.entry} pf:${resp.pf}\n")
+      val dup = Cat(sv.zip(sMeta).map{ case (v, m) =>
+        v && m.hit(resp.entry.tag)
+      }).orR // NOTE: may have long latency, RegNext it
+
+      when (!dup) {
+        sv(refillIdx) := true.B
+        sMeta(refillIdx).apply(
+          vpn = resp.entry.tag,
+          level = resp.entry.level.getOrElse(0.U)
+        )
+        sData(refillIdx).apply(
+          ppn   = resp.entry.ppn,
+          level = resp.entry.level.getOrElse(0.U),
+          perm  = VecInit(resp.entry.perm.getOrElse(0.U)).asUInt,
+          pf    = resp.pf
+        )
+        sReplace.access(sRefillIdx)
+        XSDebug(p"Refill superpage: idx:${refillIdx} entry:${resp.entry} pf:${resp.pf}\n")
+      }
     }
   }
 
@@ -476,30 +484,12 @@ class TLB(Width: Int, isDtlb: Boolean)(implicit p: Parameters) extends TlbModule
   val missVec = readResult.map(res => res._2)
   val hitVecVec = readResult.map(res => res._3)
   val validRegVec = readResult.map(res => res._4)
-  val hasMissReq = Cat(missVec).orR
 
-  // ptw
-  val waiting = RegInit(false.B)
-  when (ptw.req.fire()) {
-    waiting := true.B
-  }
-  when (sfence.valid || ptw.resp.valid) {
-    waiting := false.B
-  }
-  assert(!ptw.resp.valid || waiting)
-
-  // ptw <> DontCare // TODO: need check it
-  ptw.req.valid := hasMissReq && !waiting && !RegNext(refill)
-  ptw.resp.ready := waiting
-
-  // val ptwReqSeq = Wire(Seq.fill(Width)(new comBundle()))
-  val ptwReqSeq = Seq.fill(Width)(Wire(new comBundle()))
   for (i <- 0 until Width) {
-    ptwReqSeq(i).valid := ((if (isDtlb) RegNext(valid(i)) else valid(i)) && missVec(i))
-    ptwReqSeq(i).roqIdx := (if (isDtlb) RegNext(req(i).bits.roqIdx) else req(i).bits.roqIdx)
-    ptwReqSeq(i).bits.vpn := (if (isDtlb) RegNext(reqAddr(i).vpn) else reqAddr(i).vpn)
+    io.ptw.req(i).valid := validRegVec(i) && missVec(i) && !RegNext(refill)
+    io.ptw.req(i).bits.vpn := RegNext(reqAddr(i).vpn)
   }
-  ptw.req.bits := Compare(ptwReqSeq).bits
+  io.ptw.resp.ready := true.B
 
   // val tooManyPf = PopCount(pf) > 5.U
   // when (tooManyPf) { // when too much pf, just clear
@@ -540,18 +530,13 @@ class TLB(Width: Int, isDtlb: Boolean)(implicit p: Parameters) extends TlbModule
     // NOTE: ITLB is blocked, so every resp will be valid only when hit
     // every req will be ready only when hit
     XSPerfAccumulate("access", io.requestor(0).req.fire() && vmEnable)
-    XSPerfAccumulate("miss", ptw.req.fire())
+    XSPerfAccumulate("miss", ptw.req(0).fire())
   }
-  val reqCycleCnt = Reg(UInt(16.W))
-  when (ptw.req.fire()) {
-    reqCycleCnt := 1.U
-  }
-  when (waiting) {
-    reqCycleCnt := reqCycleCnt + 1.U
-  }
-  XSPerfAccumulate("ptw_req_count", ptw.req.fire())
-  XSPerfAccumulate("ptw_req_cycle", Mux(ptw.resp.fire(), reqCycleCnt, 0.U))
-  XSPerfAccumulate("wait_blocked_count", waiting && hasMissReq)
+  //val reqCycleCnt = Reg(UInt(16.W))
+  //reqCycleCnt := reqCycleCnt + BoolStopWatch(ptw.req(0).fire(), ptw.resp.fire || sfence.valid)
+  //XSPerfAccumulate("ptw_req_count", ptw.req.fire())
+  //XSPerfAccumulate("ptw_req_cycle", Mux(ptw.resp.fire(), reqCycleCnt, 0.U))
+  XSPerfAccumulate("ptw_resp_count", ptw.resp.fire())
   XSPerfAccumulate("ptw_resp_pf_count", ptw.resp.fire() && ptw.resp.bits.pf)
   for (i <- 0 until TlbEntrySize) {
     val indexHitVec = hitVecVec.zip(validRegVec).map{ case (h, v) => h(i) && v }
@@ -577,19 +562,13 @@ class TLB(Width: Int, isDtlb: Boolean)(implicit p: Parameters) extends TlbModule
   XSDebug(sfence.valid, p"Sfence: ${sfence}\n")
   XSDebug(ParallelOR(valid)|| ptw.resp.valid, p"CSR: ${csr}\n")
   XSDebug(ParallelOR(valid) || ptw.resp.valid, p"vmEnable:${vmEnable} hit:${Binary(VecInit(hitVec).asUInt)} miss:${Binary(VecInit(missVec).asUInt)} v:${Hexadecimal(VecInit(v).asUInt)} pf:${Hexadecimal(pf.asUInt)}\n")
-  XSDebug(ptw.req.fire(), p"PTW req:${ptw.req.bits}\n")
+  for (i <- ptw.req.indices) {
+    XSDebug(ptw.req(i).fire(), p"PTW req:${ptw.req(i).bits}\n")
+  }
   XSDebug(ptw.resp.valid, p"PTW resp:${ptw.resp.bits} (v:${ptw.resp.valid}r:${ptw.resp.ready}) \n")
 
 //   // NOTE: just for simple tlb debug, comment it after tlb's debug
-//   for (i <- 0 until Width) {
-//     if(isDtlb) {
-//       XSDebug(!(!vmEnable || RegNext(req(i).bits.vaddr)===resp(i).bits.paddr || !resp(i).valid || resp(i).bits.miss || Cat(VecInit(resp(i).bits.excp.pf).asUInt).orR), p"Dtlb: vaddr:${Hexadecimal(RegNext(req(i).bits.vaddr))} paddr:${Hexadecimal(resp(i).bits.paddr)} should be equal\n")
-//       assert(!vmEnable || RegNext(req(i).bits.vaddr)===resp(i).bits.paddr || !resp(i).valid || resp(i).bits.miss || Cat(VecInit(resp(i).bits.excp.pf).asUInt).orR)
-//     } else {
-//       XSDebug(!(!vmEnable || req(i).bits.vaddr===resp(i).bits.paddr || !resp(i).valid || resp(i).bits.miss || Cat(VecInit(resp(i).bits.excp.pf).asUInt).orR), p"Itlb: vaddr:${Hexadecimal(RegNext(req(i).bits.vaddr))} paddr:${Hexadecimal(resp(i).bits.paddr)} should be equal\n")
-//       assert(!vmEnable || req(i).bits.vaddr===resp(i).bits.paddr || !resp(i).valid || resp(i).bits.miss || Cat(VecInit(resp(i).bits.excp.pf).asUInt).orR)
-//     }
-//   }
+  // assert(!io.ptw.resp.valid || io.ptw.resp.bits.entry.tag === io.ptw.resp.bits.entry.ppn, "Simple tlb debug requires vpn === ppn")
 }
 
 object TLB {