Merge pull request #368 from RISCVERS/opt-sbuffer-timing

Opt sbuffer timing

src/main/scala/utils/Replacement.scala

@@ -6,6 +6,7 @@ package utils
 import chisel3._
 import chisel3.util._
 import chisel3.util.random.LFSR
+import xiangshan.{HasXSLog, XSCoreParameters}
 
 abstract class ReplacementPolicy {
   def way: UInt
@@ -206,9 +207,10 @@ class SbufferLRU(n_ways: Int) {
   // get the way which is valid and has the least 1
   def get_replace_way(state: UInt, sbufferState:Seq[Bool]): UInt = {
     val moreRecentVec = state.asTypeOf(Vec(n_ways, UInt(n_ways.W)))
-    val count = Wire(Vec(n_ways, UInt(log2Up(n_ways).W)))
+    val count = Wire(Vec(n_ways, UInt(log2Up(n_ways+1).W)))
     for(i <- 0 until n_ways){
-      count(i) := Mux(sbufferState(i), PopCount(moreRecentVec(i)), ((1<<n_ways)-1).U)
+      count(i) := Mux(sbufferState(i), PopCount(moreRecentVec(i)), n_ways.U)
+      //XSDebug("count %d\n",count(i))(" ")
     }
     count.zip((0 until n_ways).map(_.U))
     get_min_value(count.zip((0 until n_ways).map(_.U)))._2

src/main/scala/xiangshan/mem/sbuffer/NewSbuffer.scala

@@ -5,23 +5,19 @@ import chisel3.util._
 import xiangshan._
 import utils._
 import xiangshan.cache._
-import utils.ParallelAND
-import utils.TrueLRU
-
 
 trait HasSbufferCst extends HasXSParameter {
-  // def s_invalid :: s_valid :: s_inflight_req :: s_inflight_resp :: Nil = Enum(4)
 
   def s_invalid = 0.U(2.W)
   def s_valid = 1.U(2.W)
-  def s_inflight_req = 2.U(2.W)
-  def s_inflight_resp = 3.U(2.W)
+  def s_inflight = 2.U(2.W)
 
   val SbufferIndexWidth: Int = log2Up(StoreBufferSize)
   // paddr = tag + offset
   val CacheLineBytes: Int = CacheLineSize / 8
   val CacheLineWords: Int = CacheLineBytes / DataBytes
   val OffsetWidth: Int = log2Up(CacheLineBytes)
+  val WordsWidth: Int = log2Up(CacheLineWords)
   val TagWidth: Int = PAddrBits - OffsetWidth
 }
 
@@ -37,29 +33,95 @@ class SbufferLine extends SbufferBundle {
   }
 }
 
-class AbstractEvictor extends XSModule with HasSbufferCst{
+//class AbstractEvictor extends XSModule with HasSbufferCst{
+//  val io = IO(new Bundle{
+//    val states = Input(Vec(StoreBufferSize, UInt(s_invalid.getWidth.W)))
+//    val do_eviction = Output(Bool())
+//  })
+//}
+//
+//
+//class NaiveEvictor(threshold: Int) extends AbstractEvictor{
+//
+//  require(threshold >= 0 && threshold <= StoreBufferSize)
+//
+//  val entryCnt = PopCount(io.states.map(s => s=/=s_invalid))
+//
+//  io.do_eviction := entryCnt >= threshold.U((SbufferIndexWidth+1).W)
+//
+//  XSDebug("sbuffer entry cnt: %d\n", entryCnt)
+//
+//}
+
+class ChooseReplace(nWay: Int) extends XSModule {
   val io = IO(new Bundle{
-    val states = Input(Vec(StoreBufferSize, UInt(s_invalid.getWidth.W)))
-    val do_eviction = Output(Bool())
+    // val in = Vec(StorePipelineWidth, Input(UInt(nWay.W)))
+    val mask = Vec(StoreBufferSize, Input(Bool()))
+    val fire = Input(Bool())
+    val way = Output(UInt(nWay.W))
+    val flush = Input(Bool())
   })
-}
+  val wayReg = RegInit(0.U(log2Up(nWay).W))
+  val nextWay = (wayReg + 1.U)(log2Up(nWay)-1, 0)
+
+  io.way := wayReg
 
+  when(io.fire){
+    wayReg := Mux(io.mask(nextWay), nextWay, 0.U)
+  }
+
+  when(io.flush){
+    wayReg := 0.U
+  }
 
-class NaiveEvictor(threshold: Int) extends AbstractEvictor{
+}
 
-  require(threshold >= 0 && threshold <= StoreBufferSize)
+class SbufferLru(nWay: Int) extends XSModule {
+  val io = IO(new Bundle{
+    val in = Vec(StorePipelineWidth, Input(UInt(nWay.W)))
+    val mask = Vec(StoreBufferSize, Input(Bool()))
+    val way = Output(UInt(nWay.W))
+    val flush = Input(Bool())
+  })
 
-  val entryCnt = PopCount(io.states.map(s => s=/=s_invalid))
+  val lruRect = RegInit(VecInit(Seq.fill(StoreBufferSize)(0.U(nWay.W))))
+  val count = RegInit(VecInit(Seq.fill(StoreBufferSize)(0.U(log2Up(nWay+1).W))))
+  val idx = RegInit(VecInit(Seq.tabulate(StoreBufferSize)(i => i.U)))
+
+  //update
+  val updataMask = ParallelOR(io.in)
+  val updateValue = (~updataMask).asUInt()
+  for(i <- 0 until nWay){
+    val lruUpdate = Mux(updataMask(i), updateValue, lruRect(i) & updateValue)
+    lruRect(i) := lruUpdate
+    count(i) := PopCount(lruUpdate)
+  }
 
-  io.do_eviction := entryCnt >= threshold.U((SbufferIndexWidth+1).W)
+  // get evictionIdx
+  val maskCount = Wire(Vec(StoreBufferSize, UInt((log2Up(1 + nWay) + log2Up(nWay)).W)))    // (popcount, Idx)
+  val countZipIdx = maskCount.zip((0 until nWay).map(_.U))
+  for(i <- 0 until nWay){
+    val value = Mux(io.mask(i), count(i), nWay.U)
+    maskCount(i) := Cat(value, idx(i))
+  }
 
-  XSDebug("sbuffer entry cnt: %d\n", entryCnt)
+  io.way := ParallelMin(maskCount)(log2Up(nWay)-1,0)
 
+  // flush
+  when(io.flush){
+    for(i <- 0 until nWay){
+      lruRect(i) := 0.U
+      count(i) := nWay.U
+    }
+    XSDebug("drain sbuffer finish, flush lru\n")
+  }
 }
 
+
+
 class NewSbuffer extends XSModule with HasSbufferCst {
   val io = IO(new Bundle() {
-    val in = Vec(StorePipelineWidth, Flipped(Decoupled(new DCacheWordReq)))
+    val in = Vec(StorePipelineWidth, Flipped(Decoupled(new DCacheWordReq)))  //Todo: store logic only support Width == 2 now
     val dcache = new DCacheLineIO
     val forward = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO))
     val flush = new Bundle {
@@ -70,14 +132,6 @@ class NewSbuffer extends XSModule with HasSbufferCst {
 
   val buffer = Mem(StoreBufferSize, new SbufferLine)
   val stateVec = RegInit(VecInit(Seq.fill(StoreBufferSize)(s_invalid)))
-  //val lru = new SbufferLRU(StoreBufferSize)
-  val lru = new SbufferLRU(StoreBufferSize)
-  // 2 * enq + 1 * deq
-  val lruAccessWays = Wire(Vec(io.in.getWidth+1, new Valid(UInt(SbufferIndexWidth.W))))
-  for(w <- lruAccessWays){
-    w.bits := DontCare
-    w.valid := false.B
-  }
 
   /*
        idle --[flush]--> drian_sbuffer --[buf empty]--> idle
@@ -87,115 +141,160 @@ class NewSbuffer extends XSModule with HasSbufferCst {
   val sbuffer_state = RegInit(x_idle)
 
   // ---------------------- Store Enq Sbuffer ---------------------
-  // (state, lineBuf)
-  type SbufferEntry = (UInt, SbufferLine)
 
   def getTag(pa: UInt): UInt =
     pa(PAddrBits - 1, PAddrBits - TagWidth)
 
+  def getWord(pa: UInt): UInt =
+    pa(PAddrBits-1, 3)
+
+  def getWordOffset(pa: UInt): UInt =
+    pa(OffsetWidth-1, 3)
+
   def getAddr(tag: UInt): UInt =
     Cat(tag, 0.U((PAddrBits - TagWidth).W))
 
-  def getByteOffset(pa: UInt): UInt =
-    Cat(pa(OffsetWidth - 1, 3), 0.U(3.W))
-
-  def getWordOffset(pa: UInt): UInt = pa(OffsetWidth-1, 3)
+  def getByteOffset(offect: UInt): UInt =
+    Cat(offect(OffsetWidth - 1, 3), 0.U(3.W))
 
   def isOneOf(key: UInt, seq: Seq[UInt]): Bool =
     if(seq.isEmpty) false.B else Cat(seq.map(_===key)).orR()
 
   def widthMap[T <: Data](f: Int => T) = (0 until StoreBufferSize) map f
 
+  // sbuffer entry count
+  val invalidCount = RegInit(StoreBufferSize.U((log2Up(StoreBufferSize) + 1).W))
+  val validCount = RegInit(0.U((log2Up(StoreBufferSize) + 1).W))
+  val full = invalidCount === 0.U
+  val oneSpace = invalidCount === 1.U
+
+  val bufferRead  = VecInit((0 until StoreBufferSize).map(i => buffer(i)))
+  val stateRead   = VecInit((0 until StoreBufferSize).map(i => stateVec(i)))
+  val dataRead    = VecInit((0 until StoreBufferSize).map(i => bufferRead(i).data.asTypeOf(Vec(CacheLineWords, Vec(DataBytes, UInt(8.W))))))
+  val maskRead    = VecInit((0 until StoreBufferSize).map(i => bufferRead(i).mask.asTypeOf(Vec(CacheLineWords, Vec(DataBytes, Bool())))))
+  val tagRead     = VecInit((0 until StoreBufferSize).map(i => bufferRead(i).tag))
+
+  val dataUpdate  = WireInit(dataRead)
+  val maskUpdate  = WireInit(maskRead)
+  val tagUpdate   = WireInit(tagRead)
+  val stateUpdate = WireInit(stateRead)
+  val bufferUpdate = Wire(Vec(StoreBufferSize, new SbufferLine))
+  (0 until StoreBufferSize) foreach { i =>
+    bufferUpdate(i).tag := tagUpdate(i)
+    bufferUpdate(i).data := dataUpdate(i).asUInt()
+    bufferUpdate(i).mask := maskUpdate(i).asUInt()
+  }
+
+  val lru = Module(new ChooseReplace(StoreBufferSize))
+  // val lruAccessWays = WireInit(VecInit(Seq.fill(StorePipelineWidth)(0.U(StoreBufferSize.W))))  // 2 * enq
+  val evictionIdx = lru.io.way
+  // lru.io.in := lruAccessWays
+  lru.io.fire := false.B
+  lru.io.mask := stateRead.map(_ === s_valid)
+
+  val tags = io.in.map(in => getTag(in.bits.addr))
+  val sameTag = tags(0) === tags(1)
+  val firstWord = getWord(io.in(0).bits.addr)
+  val secondWord = getWord(io.in(1).bits.addr)
+  val sameWord = firstWord === secondWord
+
+
+  // merge condition
+  val mergeMask = Wire(Vec(StorePipelineWidth, Vec(StoreBufferSize, Bool())))
+  val mergeIdx = mergeMask.map(PriorityEncoder(_))
+  val canMerge = mergeMask.map(ParallelOR(_))
+
+
+  // merge should be block when the sameTag entry is inflight and will be inflight next cycle
+  for(i <- 0 until StorePipelineWidth){
+    mergeMask(i) := widthMap(j =>
+      Mux(tags(i) === tagRead(j) && stateRead(j) === s_valid , true.B, false.B))
+        //!(tagRead(j) === tagRead(evictionIdx) && io.dcache.req.fire()),    // entry should be block if it will be inflight next cycle
+//        true.B,
+//        false.B))
+  }
 
-  def maskData(mask: UInt, data: UInt): UInt = {
-    assert(mask.getWidth * 8 == data.getWidth)
-    Cat((0 until mask.getWidth).map(i => data(i*8+7, i*8) & Fill(8, mask(i))).reverse)
+  // insert confition
+  // firstInsert: the first invalid entry
+  // if first entry canMerge or second entry has the same tag with the first entry , secondInsert equal the first invalid entry, otherwise, the second invalid entry
+  val invalidMask = stateRead.map(s => s === s_invalid)
+  val firstInsertMask = PriorityEncoderOH(invalidMask)
+  val secondInsertMask = Wire(Vec(StoreBufferSize, Bool()))
+  for (i <- 0 until StoreBufferSize){
+    secondInsertMask(i) := Mux(canMerge(0) || sameTag, firstInsertMask(i), invalidMask(i) - firstInsertMask(i))
   }
 
-  def wordReqToBufLine(req: DCacheWordReq): SbufferLine = {
-    val bufLine = Wire(new SbufferLine)
-    val wordIdx = getWordOffset(req.addr)
-    val dataVec = Wire(Vec(CacheLineWords, Vec(DataBytes, UInt(8.W))))
-    val mask = VecInit(Seq.fill(CacheLineWords){
-      VecInit(Seq.fill(DataBytes)(false.B))
-    })
-    dataVec := DontCare
+  val (firstInsertIdx, firstCanInsert) = PriorityEncoderWithFlag(invalidMask)
+  val (secondInsertIdx, secondCanInsert) = PriorityEncoderWithFlag(secondInsertMask)
+
+
+  io.in(0).ready := firstCanInsert || canMerge(0)
+  io.in(1).ready := (secondCanInsert || canMerge(1)) && !sameWord && io.in(0).ready
+  //XSDebug(p"firstCanInsert[$firstCanInsert], secondCanInsert[$secondCanInsert], sameWord[$sameWord]\n")
+  //XSDebug(p"first ready[${io.in(0).ready}], senond ready[${io.in(1).ready}]\n")
+
+  def wordReqToBufLine(req: DCacheWordReq, tag: UInt, insertIdx: UInt, wordOffset: UInt, flushMask: Bool): Unit = {
+    stateUpdate(insertIdx) := s_valid
+    tagUpdate(insertIdx) := tag
+
+    when(flushMask){
+      for(j <- 0 until CacheLineWords){
+        for(i <- 0 until DataBytes){
+          maskUpdate(insertIdx)(j)(i) := false.B
+        }
+      }
+    }
+
     for(i <- 0 until DataBytes){
       when(req.mask(i)){
-        mask(wordIdx)(i) := true.B
-        dataVec(wordIdx)(i) := req.data(i*8+7, i*8)
+        maskUpdate(insertIdx)(wordOffset)(i) := true.B
+        dataUpdate(insertIdx)(wordOffset)(i) := req.data(i*8+7, i*8)
       }
     }
-    bufLine.tag := getTag(req.addr)
-    bufLine.mask := mask.asUInt()
-    bufLine.data := dataVec.asUInt()
-    bufLine
   }
 
-  def mergeWordReq(req: DCacheWordReq, oldLine: SbufferLine): SbufferLine = {
-    val newLine = WireInit(oldLine)
-    val wordIdx = getWordOffset(req.addr)
-    val mask = oldLine.mask.asTypeOf(Vec(CacheLineWords, Vec(DataBytes, Bool())))
-    val data = oldLine.data.asTypeOf(Vec(CacheLineWords, Vec(DataBytes, UInt(8.W))))
+  def mergeWordReq(req: DCacheWordReq, mergeIdx:UInt, wordOffset:UInt): Unit = {
     for(i <- 0 until DataBytes){
       when(req.mask(i)){
-        mask(wordIdx)(i) := true.B
-        data(wordIdx)(i) := req.data(i*8+7, i*8)
+        maskUpdate(mergeIdx)(wordOffset)(i) := true.B
+        dataUpdate(mergeIdx)(wordOffset)(i) := req.data(i*8+7, i*8)
       }
     }
-    newLine.mask := mask.asUInt()
-    newLine.data := data.asUInt()
-    newLine
   }
 
-  type ReqWithIdx = (DecoupledIO[DCacheWordReq], Int)
-
-  def enqSbuffer(buf: Seq[SbufferEntry], reqWithIdx: ReqWithIdx): Seq[SbufferEntry] = {
-    val req = reqWithIdx._1
-    val reqIdx = reqWithIdx._2
-    val state_old = VecInit(buf.map(_._1))
-    val mem_old = VecInit(buf.map(_._2))
-    val state_new = WireInit(state_old)
-    val mem_new = WireInit(mem_old)
-
-    def stateCanMerge(s: UInt): Bool = isOneOf(s, Seq(s_valid, s_inflight_req))
-
-    val mergeMask = widthMap(i =>
-      req.valid && stateCanMerge(state_old(i)) && getTag(req.bits.addr)===mem_old(i).tag
-    )
-    val canMerge = Cat(mergeMask).orR()
-    val invalidMask = state_old.map(s => s===s_invalid)
-    val notFull = Cat(invalidMask).orR()
-    req.ready := notFull || canMerge
-    val mergeIdx = PriorityEncoder(mergeMask)
-    val insertIdx = PriorityEncoder(invalidMask)
-    when(canMerge){
-      mem_new(mergeIdx) := mergeWordReq(req.bits, mem_old(mergeIdx))
-      lruAccessWays(reqIdx).valid := true.B
-      lruAccessWays(reqIdx).bits := mergeIdx
-      XSDebug(p"merge req $reqIdx to line [$mergeIdx]\n")
-    }.elsewhen(notFull && req.valid){
-      state_new(insertIdx) := s_valid
-      mem_new(insertIdx) := wordReqToBufLine(req.bits)
-      lruAccessWays(reqIdx).valid := true.B
-      lruAccessWays(reqIdx).bits := insertIdx
-      XSDebug(p"insert req $reqIdx to line[$insertIdx]\n")
+  // first store
+  when(io.in(0).fire()){
+    when(canMerge(0)){
+      mergeWordReq(io.in(0).bits, mergeIdx(0), firstWord)
+      // lruAccessWays(0).valid := true.B
+      // lruAccessWays(0) := Cat(mergeMask(0).reverse)
+      XSDebug(p"merge req 0 to line [${mergeIdx(0)}]\n")
+    }.elsewhen(firstCanInsert){
+      wordReqToBufLine(io.in(0).bits, tags(0), firstInsertIdx, firstWord, true.B)
+      //lruAccessWays(0).valid := true.B
+      // lruAccessWays(0) := Cat(firstInsertMask.reverse)
+      XSDebug(p"insert req 0 to line[$firstInsertIdx]\n")
     }
-    state_new.zip(mem_new)
   }
 
-  val bufferRead = VecInit((0 until StoreBufferSize) map (i => buffer(i)))
-  val initialSbuffer = stateVec.zip(bufferRead)
-  val updatedSbuffer = io.in.zipWithIndex.foldLeft[Seq[SbufferEntry]](initialSbuffer)(enqSbuffer)
-  val updatedState = updatedSbuffer.map(_._1)
-  val updatedSbufferLine = VecInit(updatedSbuffer.map(_._2))
-  when (!io.in(0).ready) {
-    io.in(1).ready := false.B
+  // second store
+  when(io.in(1).fire()){
+    when(canMerge(1)){
+      mergeWordReq(io.in(1).bits, mergeIdx(1), secondWord)
+      // lruAccessWays(1).valid := true.B
+      // lruAccessWays(1) := Cat(mergeMask(1).reverse)
+      XSDebug(p"merge req 1 to line [${mergeIdx(1)}]\n")
+    }.elsewhen(secondCanInsert){
+      wordReqToBufLine(io.in(1).bits, tags(1), secondInsertIdx, secondWord, !sameTag)
+      //lruAccessWays(1).valid := true.B
+      // lruAccessWays(1) := Cat(PriorityEncoderOH(secondInsertMask).reverse)
+      XSDebug(p"insert req 1 to line[$secondInsertIdx]\n")
+    }
   }
 
   for(i <- 0 until StoreBufferSize){
-    buffer.write(i.U, updatedSbufferLine(i))
-    stateVec(i) := updatedState(i)
+    buffer.write(i.U, bufferUpdate(i))
+    stateVec(i) := stateUpdate(i)
   }
 
   for(i <- 0 until StoreBufferSize){
@@ -221,14 +320,11 @@ class NewSbuffer extends XSModule with HasSbufferCst {
 
   val do_eviction = Wire(Bool())
   val empty = Cat(stateVec.map(s => s===s_invalid)).andR() && !Cat(io.in.map(_.valid)).orR()
-  val replaceIdx = lru.way(stateVec.map(s => s===s_valid))
-  val firstValidEntry = PriorityEncoder(stateVec.map(s => s===s_valid))
 
-  val evictor = Module(new NaiveEvictor(StoreBufferSize-4))
-  evictor.io.states := stateVec
-  do_eviction := evictor.io.do_eviction
+  do_eviction := validCount >= 12.U
 
   io.flush.empty := empty
+  lru.io.flush := sbuffer_state === x_drain_sbuffer && empty
   switch(sbuffer_state){
     is(x_idle){
       when(io.flush.valid){
@@ -252,18 +348,12 @@ class NewSbuffer extends XSModule with HasSbufferCst {
   }
   XSDebug(p"sbuffer state:${sbuffer_state} do eviction:${do_eviction} empty:${empty}\n")
 
-  //XSDebug(p"replaceIdx:${replaceIdx}\n")
-  //val evictionIdxWire = replaceIdx
-  val evictionIdxWire = Mux(stateVec(replaceIdx)===s_valid, replaceIdx, firstValidEntry)
-  val evictionIdxEnqReq = Wire(DecoupledIO(UInt(SbufferIndexWidth.W)))
-  val evictionIdxQueue = Module(new Queue(UInt(SbufferIndexWidth.W), StoreBufferSize, pipe = true, flow = false))
-
   def noSameBlockInflight(idx: UInt): Bool = {
-    val tag = updatedSbufferLine(idx).tag
+    val tag = tagRead(idx)
     !Cat(widthMap(i => {
       // stateVec(idx) itself must not be s_inflight*
-      isOneOf(stateVec(i), Seq(s_inflight_req, s_inflight_resp)) &&
-        tag===updatedSbufferLine(i).tag
+      stateRead(i) === s_inflight &&
+        tag === tagRead(i)
     })).orR()
   }
 
@@ -271,59 +361,61 @@ class NewSbuffer extends XSModule with HasSbufferCst {
       If there is a inflight dcache req which has same tag with evictionIdx's tag,
       current eviction should be blocked.
    */
-  evictionIdxEnqReq.valid :=
-    isOneOf(sbuffer_state, Seq(x_drain_sbuffer, x_replace)) &&
-      stateVec(evictionIdxWire)===s_valid &&
-      noSameBlockInflight(evictionIdxWire)
-
-  evictionIdxEnqReq.bits := evictionIdxWire
-  evictionIdxQueue.io.enq <> evictionIdxEnqReq
-
-  when(evictionIdxEnqReq.fire()){
-    stateVec(evictionIdxWire) := s_inflight_req
-    lruAccessWays.last.valid := true.B
-    lruAccessWays.last.bits := evictionIdxWire
-  }
-
-  // update lru
-  lru.access(lruAccessWays)
-  when(sbuffer_state === x_drain_sbuffer && empty){
-    lru.flush()
-    XSDebug("drain sbuffer finish, flush lru\n")
-  }
-
-
-  val wbIdx = evictionIdxQueue.io.deq.bits
-  val wbLine = updatedSbufferLine(wbIdx)
-  io.dcache.req.valid := evictionIdxQueue.io.deq.valid
-  io.dcache.req.bits.addr := getAddr(wbLine.tag)
-  io.dcache.req.bits.data := wbLine.data
-  io.dcache.req.bits.mask := wbLine.mask
+//  val evictionEntry = Wire(DecoupledIO(UInt(SbufferIndexWidth.W)))
+//
+//  evictionEntry.valid :=
+//    do_eviction && sbuffer_state === x_replace || sbuffer_state === x_drain_sbuffer &&
+//      stateVec(evictionIdx)===s_valid &&
+//      noSameBlockInflight(evictionIdx)
+//
+//  evictionEntry.bits := evictionIdx
+
+  val tagConflict = tagRead(evictionIdx) === tags(0) || tagRead(evictionIdx) === tags(1)
+
+  io.dcache.req.valid :=
+    ((do_eviction && sbuffer_state === x_replace) || (sbuffer_state === x_drain_sbuffer)) &&
+    stateVec(evictionIdx)===s_valid &&
+    noSameBlockInflight(evictionIdx) &&
+    !tagConflict
+
+  io.dcache.req.bits.addr := getAddr(tagRead(evictionIdx))
+  io.dcache.req.bits.data := bufferRead(evictionIdx).data
+  io.dcache.req.bits.mask := bufferRead(evictionIdx).mask
   io.dcache.req.bits.cmd := MemoryOpConstants.M_XWR
   io.dcache.req.bits.meta := DontCare
-  io.dcache.req.bits.meta.id := wbIdx
-  when(io.dcache.req.fire()){ stateVec(wbIdx) := s_inflight_resp }
-  evictionIdxQueue.io.deq.ready := io.dcache.req.ready
+  io.dcache.req.bits.meta.id := evictionIdx
+  when(io.dcache.req.fire()){
+    lru.io.fire := true.B
+    stateVec(evictionIdx) := s_inflight
+  }
+//  evictionEntry.ready := io.dcache.req.ready
 
   XSDebug(io.dcache.req.fire(),
-    p"send buf [$wbIdx] to Dcache, req fire\n"
+    p"send buf [$evictionIdx] to Dcache, req fire\n"
   )
 
   io.dcache.resp.ready := true.B // sbuffer always ready to recv dcache resp
   val respId = io.dcache.resp.bits.meta.id
   when(io.dcache.resp.fire()){
+    XSDebug("")
     stateVec(respId) := s_invalid
-    assert(stateVec(respId) === s_inflight_resp)
+    assert(stateVec(respId) === s_inflight)
     XSDebug(p"recv cache resp: id=[$respId]\n")
   }
 
+  val needSpace = (io.in(0).fire && !canMerge(0)) +& (io.in(1).fire && !canMerge(1) && !sameTag)
+  invalidCount := invalidCount - needSpace + io.dcache.resp.fire()
+  validCount := validCount + needSpace - io.dcache.req.fire()
+
+  XSDebug(p"needSpace[$needSpace] invalidCount[$invalidCount]  validCount[$validCount]\n")
+
   // ---------------------- Load Data Forward ---------------------
 
   for ((forward, i) <- io.forward.zipWithIndex) {
-    val tag_matches = widthMap(i => bufferRead(i).tag===getTag(forward.paddr))
-    val valid_tag_matches = widthMap(i => tag_matches(i) && stateVec(i)===s_valid)
+    val tag_matches = widthMap(i => tagRead(i) === getTag(forward.paddr))
+    val valid_tag_matches = widthMap(i => tag_matches(i) && stateVec(i) === s_valid)
     val inflight_tag_matches = widthMap(i =>
-      tag_matches(i) && (stateVec(i)===s_inflight_req || stateVec(i)===s_inflight_resp)
+      tag_matches(i) && (stateVec(i) === s_inflight)
     )
     val line_offset_mask = UIntToOH(getWordOffset(forward.paddr))
 
@@ -344,19 +436,15 @@ class NewSbuffer extends XSModule with HasSbufferCst {
       forward.forwardData(j) := DontCare
 
       // valid entries have higher priority than inflight entries
-      when (selectedInflightMask(j)) {
+      when(selectedInflightMask(j)) {
         forward.forwardMask(j) := true.B
         forward.forwardData(j) := selectedInflightData(j)
       }
-      when (selectedValidMask(j)) {
+      when(selectedValidMask(j)) {
         forward.forwardMask(j) := true.B
         forward.forwardData(j) := selectedValidData(j)
       }
     }
-
-    XSDebug(Cat(inflight_tag_matches).orR || Cat(valid_tag_matches).orR,
-      p"[$i] forward paddr:${Hexadecimal(forward.paddr)}\n"
-    )
   }
 }