ICache : fix 2 potential rule violations according to TL specification (#1444)

* ReplacePipe: block miss until get ReleaseAck

* IPrefetch: cancle prefetch req when meet MSHR

* Fetch <perf>: add fetch bubble performance counters

src/main/scala/top/Configs.scala

@@ -92,7 +92,7 @@ class MinimalConfig(n: Int = 1) extends Config(
           dataECC = Some("parity"),
           replacer = Some("setplru"),
           nMissEntries = 2,
-          nReleaseEntries = 2,
+          nReleaseEntries = 1,
           nProbeEntries = 2,
           nPrefetchEntries = 2,
           hasPrefetch = false

src/main/scala/xiangshan/Parameters.scala

@@ -218,7 +218,6 @@ case class XSCoreParameters
     dataECC = Some("parity"),
     replacer = Some("setplru"),
     nMissEntries = 2,
-    nReleaseEntries = 2,
     nProbeEntries = 2,
     nPrefetchEntries = 2,
     hasPrefetch = true,

src/main/scala/xiangshan/frontend/IFU.scala

@@ -164,10 +164,14 @@ class NewIFU(implicit p: Parameters) extends XSModule
 
   /** <PERF> f0 fetch bubble */
 
-  XSPerfAccumulate("fetch_bubble_ftq_not_valid",   !f0_valid )
+  XSPerfAccumulate("fetch_bubble_ftq_not_valid",   !fromFtq.req.valid && fromFtq.req.ready  )
   XSPerfAccumulate("fetch_bubble_pipe_stall",    f0_valid && toICache(0).ready && toICache(1).ready && !f1_ready )
-  XSPerfAccumulate("fetch_bubble_sram_0_busy",   f0_valid && !toICache(0).ready  )
-  XSPerfAccumulate("fetch_bubble_sram_1_busy",   f0_valid && !toICache(1).ready  )
+  XSPerfAccumulate("fetch_bubble_icache_0_busy",   f0_valid && !toICache(0).ready  )
+  XSPerfAccumulate("fetch_bubble_icache_1_busy",   f0_valid && !toICache(1).ready  )
+  XSPerfAccumulate("fetch_flush_backend_redirect",   backend_redirect  )
+  XSPerfAccumulate("fetch_flush_wb_redirect",    wb_redirect  )
+  XSPerfAccumulate("fetch_flush_bpu_f1_flush",   from_bpu_f1_flush  )
+  XSPerfAccumulate("fetch_flush_bpu_f0_flush",   from_bpu_f0_flush  )
 
 
   /**
@@ -303,6 +307,8 @@ class NewIFU(implicit p: Parameters) extends XSModule
 
   val predecodeOutValid = WireInit(false.B)
 
+  XSPerfAccumulate("fetch_bubble_icache_not_resp",   f2_valid && !icacheRespAllValid )
+
 
   /**
     ******************************************************************************
@@ -599,6 +605,9 @@ class NewIFU(implicit p: Parameters) extends XSModule
 
   mmio_redirect := (f3_req_is_mmio && mmio_state === m_waitCommit && RegNext(fromUncache.fire())  && f3_mmio_use_seq_pc)
 
+  XSPerfAccumulate("fetch_bubble_ibuffer_not_ready",   io.toIbuffer.valid && !io.toIbuffer.ready )
+
+
   /**
     ******************************************************************************
     * IFU Write Back Stage
@@ -693,8 +702,6 @@ class NewIFU(implicit p: Parameters) extends XSModule
     ("hit_0_miss_1                 ", f3_perf_info.hit_0_miss_1     && io.toIbuffer.fire() ),
     ("miss_0_hit_1                 ", f3_perf_info.miss_0_hit_1     && io.toIbuffer.fire() ),
     ("miss_0_miss_1                ", f3_perf_info.miss_0_miss_1    && io.toIbuffer.fire() ),
-    // ("cross_line_block             ", io.toIbuffer.fire() && f3_situation(0)     ),
-    // ("fall_through_is_cacheline_end", io.toIbuffer.fire() && f3_situation(1)     ),
   )
   generatePerfEvent()
 

src/main/scala/xiangshan/frontend/icache/ICache.scala

@@ -39,7 +39,7 @@ case class ICacheParameters(
     dataECC: Option[String] = None,
     replacer: Option[String] = Some("random"),
     nMissEntries: Int = 2,
-    nReleaseEntries: Int = 2,
+    nReleaseEntries: Int = 1,
     nProbeEntries: Int = 2,
     nPrefetchEntries: Int = 4,
     hasPrefetch: Boolean = false,
@@ -484,7 +484,7 @@ class ICacheImp(outer: ICache) extends LazyModuleImp(outer) with HasICacheParame
   val mainPipe       = Module(new ICacheMainPipe)
   val missUnit      = Module(new ICacheMissUnit(edge))
   val releaseUnit    = Module(new ReleaseUnit(edge))
-  val replacePipe     = Module(new ReplacePipe)
+  val replacePipe     = Module(new ICacheReplacePipe)
   val probeQueue     = Module(new ICacheProbeQueue(edge))
   val prefetchPipe    = Module(new IPrefetchPipe)
 
@@ -564,6 +564,8 @@ class ICacheImp(outer: ICache) extends LazyModuleImp(outer) with HasICacheParame
 
   missUnit.io.prefetch_req <> prefetchPipe.io.toMissUnit.enqReq
 
+  prefetchPipe.io.fromMSHR <> missUnit.io.prefetch_check
+
   bus.b.ready := false.B
   bus.c.valid := false.B
   bus.c.bits  := DontCare
@@ -573,8 +575,8 @@ class ICacheImp(outer: ICache) extends LazyModuleImp(outer) with HasICacheParame
   bus.a <> missUnit.io.mem_acquire
   bus.e <> missUnit.io.mem_finish
 
-  releaseUnit.io.req(0)  <>  replacePipe.io.release_req
-  releaseUnit.io.req(1)  <>  DontCare//mainPipe.io.toReleaseUnit(1)
+  releaseUnit.io.req <>  replacePipe.io.release_req
+  replacePipe.io.release_finish := releaseUnit.io.finish
   bus.c <> releaseUnit.io.mem_release
 
   // connect bus d
@@ -606,12 +608,14 @@ class ICacheImp(outer: ICache) extends LazyModuleImp(outer) with HasICacheParame
 
   val hasConflict = VecInit(Seq(
         replacePipe.io.status.r1_set.valid,
-        replacePipe.io.status.r2_set.valid
+        replacePipe.io.status.r2_set.valid,
+        replacePipe.io.status.r3_set.valid
   ))
 
   val conflictIdx = VecInit(Seq(
         replacePipe.io.status.r1_set.bits,
-        replacePipe.io.status.r2_set.bits
+        replacePipe.io.status.r2_set.bits,
+        replacePipe.io.status.r3_set.bits
   ))
 
   val releaseShouldBlock = VecInit(hasConflict.zip(conflictIdx).map{case(valid, idx) =>  valid && releaseReqValid && idx === releaseReqVidx }).reduce(_||_)

src/main/scala/xiangshan/frontend/icache/ICacheMainPipe.scala

@@ -189,6 +189,7 @@ class ICacheMainPipe(implicit p: Parameters) extends ICacheModule
   // val tlb_status = RegInit(VecInit(Seq.fill(PortNumber)(t_idle)))
   // dontTouch(tlb_status)
 
+
   val tlb_miss_vec = VecInit((0 until PortNumber).map( i => toITLB(i).valid && fromITLB(i).bits.miss ))
   val tlb_has_miss = tlb_miss_vec.reduce(_||_)
   val tlb_resp = Wire(Vec(2, Bool()))
@@ -196,6 +197,8 @@ class ICacheMainPipe(implicit p: Parameters) extends ICacheModule
   tlb_resp(1) := !fromITLB(1).bits.miss || !s0_final_double_line
   val tlb_all_resp = tlb_resp.reduce(_&&_)
 
+  XSPerfAccumulate("icache_bubble_s0_tlb_miss",    s0_valid && tlb_has_miss )
+
   when(tlb_has_miss && !tlb_slot.valid){
     tlb_slot.valid := s0_valid
     tlb_slot.only_first := s0_only_first
@@ -319,8 +322,6 @@ class ICacheMainPipe(implicit p: Parameters) extends ICacheModule
     XSPerfAccumulate("line_1_victim_way_" + Integer.toString(w, 10),  s1_fire && s1_double_line && !s1_port_hit(1) && OHToUInt(s1_victim_oh(1))  === w.U)
   }
 
-  XSPerfAccumulate("ifu_bubble_s1_tlb_miss",    s1_valid && !tlbRespAllValid )
-
   /**
     ******************************************************************************
     * ICache Stage 2
@@ -656,6 +657,6 @@ class ICacheMainPipe(implicit p: Parameters) extends ICacheModule
 
   /** <PERF> fetch bubble generated by icache miss*/
 
-  XSPerfAccumulate("ifu_bubble_s2_miss",    s2_valid && !s2_fetch_finish )
+  XSPerfAccumulate("icache_bubble_s2_miss",    s2_valid && !s2_fetch_finish )
 
 }

src/main/scala/xiangshan/frontend/icache/ICacheMissUnit.scala

@@ -79,7 +79,9 @@ class ICacheMissEntry(edge: TLEdgeOut, id: Int)(implicit p: Parameters) extends
 
     val release_req    =  DecoupledIO(new ReplacePipeReq)
     val release_resp   =  Flipped(ValidIO(UInt(ReplaceIdWid.W)))
-    val victimInfor        =  Output(new ICacheVictimInfor())
+    val victimInfor    =  Output(new ICacheVictimInfor())
+
+    val toPrefetch    = ValidIO(UInt(PAddrBits.W))
 
   })
 
@@ -129,6 +131,9 @@ class ICacheMissEntry(edge: TLEdgeOut, id: Int)(implicit p: Parameters) extends
   io.mem_acquire.valid := (state === s_send_mem_aquire)
   io.release_req.valid := (state === s_send_replace)
 
+  io.toPrefetch.valid := (state =/= s_idle)
+  io.toPrefetch.bits  :=  addrAlign(req.paddr, blockBytes, PAddrBits)
+
   val grantack = RegEnable(edge.GrantAck(io.mem_grant.bits), io.mem_grant.fire())
   val grant_param = Reg(UInt(TLPermissions.bdWidth.W))
   val is_dirty = RegInit(false.B)
@@ -265,7 +270,9 @@ class ICacheMissUnit(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheMiss
 
     val victimInfor = Vec(PortNumber, Output(new ICacheVictimInfor()))
 
-    val prefetch_req =  Flipped(DecoupledIO(new PIQReq))
+    val prefetch_req          =  Flipped(DecoupledIO(new PIQReq))
+    val prefetch_check        =  Vec(PortNumber,ValidIO(UInt(PAddrBits.W)))
+
 
   })
   // assign default values to output signals
@@ -301,6 +308,7 @@ class ICacheMissUnit(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheMiss
     io.resp(i) <> entry.io.resp
 
     io.victimInfor(i) := entry.io.victimInfor
+    io.prefetch_check(i) <> entry.io.toPrefetch
 
     entry.io.release_resp <> io.release_resp
 

src/main/scala/xiangshan/frontend/icache/IPrefetch.scala

@@ -45,7 +45,8 @@ class IPredfetchIO(implicit p: Parameters) extends IPrefetchBundle {
   val pmp             =   new ICachePMPBundle
   val toIMeta         = Decoupled(new ICacheReadBundle)
   val fromIMeta       = Input(new ICacheMetaRespBundle)
-  val toMissUnit     = new IPrefetchToMissUnit
+  val toMissUnit      = new IPrefetchToMissUnit
+  val fromMSHR        = Flipped(Vec(PortNumber,ValidIO(UInt(PAddrBits.W))))
 
   val prefetchEnable = Input(Bool())
   val prefetchDisable = Input(Bool())
@@ -154,6 +155,9 @@ class IPrefetchPipe(implicit p: Parameters) extends  IPrefetchModule
   val p2_except_af = DataHoldBypass(pmpExcpAF, p2_pmp_fire) || RegEnable(next = tlb_resp_af, enable = p1_fire)
   val p2_mmio      = DataHoldBypass(io.pmp.resp.mmio && !p2_except_af && !p2_except_pf, p2_pmp_fire)
 
+  /*when a prefetch req meet with a miss req in MSHR cancle the prefetch req */
+  val p2_check_in_mshr = VecInit(io.fromMSHR.map(mshr => mshr.valid && mshr.bits === addrAlign(p2_paddr, blockBytes, PAddrBits))).reduce(_||_)
+
   //TODO wait PMP logic
   val p2_exception  = VecInit(Seq(pmpExcpAF, p2_mmio)).reduce(_||_)
 
@@ -170,10 +174,11 @@ class IPrefetchPipe(implicit p: Parameters) extends  IPrefetchModule
   val p3_valid =  generatePipeControl(lastFire = p2_fire, thisFire = p3_fire || p3_discard, thisFlush = false.B, lastFlush = false.B)
 
   val p3_paddr = RegEnable(next = p2_paddr,  enable = p2_fire)
+  val p3_check_in_mshr = RegEnable(next = p2_check_in_mshr,  enable = p2_fire)
 
   val p3_hit_dir = VecInit((0 until nPrefetchEntries).map(i => prefetch_dir(i).valid && prefetch_dir(i).paddr === p3_paddr )).reduce(_||_)
 
-  p3_discard := p3_hit_dir
+  p3_discard := p3_hit_dir || p3_check_in_mshr
 
   toMissUnit.enqReq.valid             := p3_valid && enableBit && !p3_discard
   toMissUnit.enqReq.bits.paddr        := p3_paddr

src/main/scala/xiangshan/frontend/icache/ReleaseUnit.scala

@@ -44,12 +44,13 @@ class RealeaseEntry(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheModul
   val io = IO(new Bundle {
     val id = Input(UInt())
     val req = Flipped(DecoupledIO(new ReleaseReq))
+    val finish = Output(Bool())
 
     val mem_release = DecoupledIO(new TLBundleC(edge.bundle))
     val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle)))
   })
 
-  val s_invalid :: s_release_req :: s_release_resp :: s_meta_write :: Nil = Enum(4)
+  val s_invalid :: s_release_req :: s_release_resp :: Nil = Enum(3)
   val state = RegInit(s_invalid)
 
   val req  = Reg(new ReleaseReq)
@@ -67,6 +68,7 @@ class RealeaseEntry(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheModul
 
   io.req.ready := state === s_invalid
   io.mem_grant.ready := false.B
+  io.finish := false.B
 
   when (io.req.fire()) {
     req        := io.req.bits
@@ -95,7 +97,7 @@ class RealeaseEntry(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheModul
     data = beat_data(beat)
   )
 
-    val voluntaryReleaseData = edge.Release(
+  val voluntaryReleaseData = edge.Release(
     fromSource = io.id,
     toAddress = addrAlign(req.addr, blockBytes, PAddrBits),
     lgSize = log2Ceil(blockBytes).U,
@@ -122,12 +124,9 @@ class RealeaseEntry(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheModul
   when (state === s_release_resp) {
     io.mem_grant.ready := true.B
     when (io.mem_grant.fire()) {
-      state := Mux(req.voluntary,s_meta_write,s_invalid)
-    }
-  }
-
-  when(state === s_meta_write) {
+      io.finish := true.B
       state := s_invalid
+    }
   }
 
 }
@@ -135,35 +134,35 @@ class RealeaseEntry(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheModul
 class ReleaseUnit(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheModule
 {
   val io = IO(new Bundle {
-    val req = Vec(2, Flipped(DecoupledIO(new ReleaseReq)))
+    val req = Flipped(DecoupledIO(new ReleaseReq))
+    val finish = Output(Bool())
     val mem_release = DecoupledIO(new TLBundleC(edge.bundle))
     val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle)))
   })
 
+  //more than 1 release entries may cause bug
+  //TODO: support multiple concurrent Releases 
+  require(cacheParams.nReleaseEntries == 1)
+
   val req = io.req
   io.mem_release.valid := false.B
   io.mem_release.bits  := DontCare
   io.mem_grant.ready   := false.B
 
-  val entries = (0 until cacheParams.nReleaseEntries) map { i =>
-    val entry = Module(new RealeaseEntry(edge))
-
-    entry.io.id := i.U
+  val entry = Module(new RealeaseEntry(edge))
 
-    // entry req
-    entry.io.req.valid := io.req(i).valid
-    entry.io.req.bits  := io.req(i).bits
-    io.req(i).ready    := entry.io.req.ready
+  entry.io.id := 0.U
 
-    entry.io.mem_grant.valid := (i.U === io.mem_grant.bits.source) && io.mem_grant.valid
-    entry.io.mem_grant.bits  := io.mem_grant.bits
-    when (i.U === io.mem_grant.bits.source) {
-      io.mem_grant.ready := entry.io.mem_grant.ready
-    }
+  // entry req
+  entry.io.req.valid := io.req.valid
+  entry.io.req.bits  := io.req.bits
+  io.req.ready    := entry.io.req.ready
 
+  entry.io.mem_grant.valid := (0.U === io.mem_grant.bits.source) && io.mem_grant.valid
+  entry.io.mem_grant.bits  := io.mem_grant.bits
+  io.mem_grant.ready := entry.io.mem_grant.ready
 
-    entry
-  }
-  TLArbiter.robin(edge, io.mem_release, entries.map(_.io.mem_release):_*)
+  io.mem_release <> entry.io.mem_release
+  io.finish := entry.io.finish
 
 }
\ No newline at end of file

src/main/scala/xiangshan/frontend/icache/ReplacePipe.scala

@@ -40,7 +40,7 @@ class ReplacePipeReq(implicit p: Parameters) extends ICacheBundle
 }
 
 
-class ReplacePipe(implicit p: Parameters) extends ICacheModule{
+class ICacheReplacePipe(implicit p: Parameters) extends ICacheModule{
   val io = IO(new Bundle{
     val pipe_req = Flipped(DecoupledIO(new ReplacePipeReq))
 
@@ -55,11 +55,12 @@ class ReplacePipe(implicit p: Parameters) extends ICacheModule{
     val meta_write = DecoupledIO(new ICacheMetaWriteBundle)
 
     val release_req = DecoupledIO(new ReleaseReq)
+    val release_finish = Input(Bool())
 
     val pipe_resp = ValidIO(UInt(ReplaceIdWid.W))
     
     val status = new Bundle() {
-      val r1_set, r2_set = ValidIO(UInt(idxBits.W))
+      val r1_set, r2_set, r3_set = ValidIO(UInt(idxBits.W))
     }
 
     val csr_parity_enable = Input(Bool())
@@ -71,7 +72,7 @@ class ReplacePipe(implicit p: Parameters) extends ICacheModule{
   val (metaError, dataError) = (io.meta_response.errors(0), io.data_response.errors(0))
 
   val r0_ready, r1_ready, r2_ready = WireInit(false.B)
-  val r0_fire,  r1_fire , r2_fire  = WireInit(false.B)
+  val r0_fire,  r1_fire , r2_fire, r3_fire  = WireInit(false.B)
 
   /**
     ******************************************************************************
@@ -209,10 +210,29 @@ class ReplacePipe(implicit p: Parameters) extends ICacheModule{
   io.release_req.bits.vidx      := DontCare
 
   //response to MissQueue
-  io.pipe_resp.valid := r2_fire && r2_req.isRelease
-  io.pipe_resp.bits  := r2_req.id
+  // io.pipe_resp.valid := r2_fire && r2_req.isRelease
+  // io.pipe_resp.bits  := r2_req.id
 
   io.status.r2_set.valid := r2_valid
   io.status.r2_set.bits  := r2_req.vidx
 
+  /**
+    ******************************************************************************
+    * ReplacePipe Stage 3
+    ******************************************************************************
+    */
+
+  val r3_valid          = generatePipeControl(lastFire = r2_fire && r2_req.isRelease, thisFire = r3_fire, thisFlush = false.B, lastFlush = false.B)
+  val r3_release_need_send = RegEnable(next = release_need_send, enable = r2_fire && r2_req.isRelease)
+
+  r3_fire       := (r3_valid && RegNext(io.release_finish) && r3_release_need_send) || (r3_valid && !r3_release_need_send) 
+
+  val r3_req = RegEnable(next = r2_req, enable = r2_fire && r2_req.isRelease)
+
+  io.pipe_resp.valid := r3_fire 
+  io.pipe_resp.bits  := r3_req.id
+
+  io.status.r3_set.valid := r3_valid
+  io.status.r3_set.bits  := r3_req.vidx
+
 }
\ No newline at end of file