Merge remote-tracking branch 'origin/master' into opt-brq

src/main/scala/utils/CircularQueuePtr.scala

@@ -45,6 +45,8 @@ trait HasCircularQueuePtrHelper {
     }
 
     final def === (that_ptr: T): Bool = ptr.asUInt()===that_ptr.asUInt()
+
+    final def =/= (that_ptr: T): Bool = ptr.asUInt()=/=that_ptr.asUInt()
   }
 
 

src/main/scala/xiangshan/XSCore.scala

@@ -62,7 +62,6 @@ case class XSCoreParameters
   StoreQueueSize: Int = 48,
   RoqSize: Int = 192,
   dpParams: DispatchParameters = DispatchParameters(
-    DqEnqWidth = 4,
     IntDqSize = 24,
     FpDqSize = 24,
     LsDqSize = 24,

src/main/scala/xiangshan/backend/CtrlBlock.scala

@@ -12,6 +12,7 @@ import xiangshan.backend.exu._
 import xiangshan.backend.exu.Exu.exuConfigs
 import xiangshan.backend.regfile.RfReadPort
 import xiangshan.backend.roq.{Roq, RoqPtr, RoqCSRIO}
+import xiangshan.mem.LsqEnqIO
 
 class CtrlToIntBlockIO extends XSBundle {
   val enqIqCtrl = Vec(exuParameters.IntExuCnt, DecoupledIO(new MicroOp))
@@ -30,11 +31,7 @@ class CtrlToFpBlockIO extends XSBundle {
 class CtrlToLsBlockIO extends XSBundle {
   val enqIqCtrl = Vec(exuParameters.LsExuCnt, DecoupledIO(new MicroOp))
   val enqIqData = Vec(exuParameters.LsExuCnt, Output(new ExuInput))
-  val enqLsq = new Bundle() {
-    val canAccept = Input(Bool())
-    val req = Vec(RenameWidth, ValidIO(new MicroOp))
-    val resp = Vec(RenameWidth, Input(new LSIdx))
-  }
+  val enqLsq = Flipped(new LsqEnqIO)
   val redirect = ValidIO(new Redirect)
 }
 

src/main/scala/xiangshan/backend/dispatch/Dispatch.scala

@@ -6,12 +6,12 @@ import xiangshan._
 import utils._
 import xiangshan.backend.regfile.RfReadPort
 import chisel3.ExcitingUtils._
-import xiangshan.backend.roq.RoqPtr
+import xiangshan.backend.roq.{RoqPtr, RoqEnqIO}
 import xiangshan.backend.rename.RenameBypassInfo
+import xiangshan.mem.LsqEnqIO
 
 case class DispatchParameters
 (
-  DqEnqWidth: Int,
   IntDqSize: Int,
   FpDqSize: Int,
   LsDqSize: Int,
@@ -30,19 +30,9 @@ class Dispatch extends XSModule {
     // to busytable: set pdest to busy (not ready) when they are dispatched
     val allocPregs = Vec(RenameWidth, Output(new ReplayPregReq))
     // enq Roq
-    val enqRoq = new Bundle {
-      val canAccept = Input(Bool())
-      val isEmpty = Input(Bool())
-      val extraWalk = Vec(RenameWidth, Output(Bool()))
-      val req = Vec(RenameWidth, ValidIO(new MicroOp))
-      val resp = Vec(RenameWidth, Input(new RoqPtr))
-    }
+    val enqRoq = Flipped(new RoqEnqIO)
     // enq Lsq
-    val enqLsq = new Bundle() {
-      val canAccept = Input(Bool())
-      val req = Vec(RenameWidth, ValidIO(new MicroOp))
-      val resp = Vec(RenameWidth, Input(new LSIdx))
-    }
+    val enqLsq = Flipped(new LsqEnqIO)
     // read regfile
     val readIntRf = Vec(NRIntReadPorts, Flipped(new RfReadPort))
     val readFpRf = Vec(NRFpReadPorts, Flipped(new RfReadPort))
@@ -56,9 +46,9 @@ class Dispatch extends XSModule {
   })
 
   val dispatch1 = Module(new Dispatch1)
-  val intDq = Module(new DispatchQueue(dpParams.IntDqSize, dpParams.DqEnqWidth, dpParams.IntDqDeqWidth))
-  val fpDq = Module(new DispatchQueue(dpParams.FpDqSize, dpParams.DqEnqWidth, dpParams.FpDqDeqWidth))
-  val lsDq = Module(new DispatchQueue(dpParams.LsDqSize, dpParams.DqEnqWidth, dpParams.LsDqDeqWidth))
+  val intDq = Module(new DispatchQueue(dpParams.IntDqSize, RenameWidth, dpParams.IntDqDeqWidth))
+  val fpDq = Module(new DispatchQueue(dpParams.FpDqSize, RenameWidth, dpParams.FpDqDeqWidth))
+  val lsDq = Module(new DispatchQueue(dpParams.LsDqSize, RenameWidth, dpParams.LsDqDeqWidth))
 
   // pipeline between rename and dispatch
   // accepts all at once
@@ -68,15 +58,12 @@ class Dispatch extends XSModule {
   }
 
   // dispatch 1: accept uops from rename and dispatch them to the three dispatch queues
-  dispatch1.io.redirect <> io.redirect
+  // dispatch1.io.redirect <> io.redirect
   dispatch1.io.renameBypass := RegEnable(io.renameBypass, io.fromRename(0).valid && dispatch1.io.fromRename(0).ready)
   dispatch1.io.enqRoq <> io.enqRoq
   dispatch1.io.enqLsq <> io.enqLsq
-  dispatch1.io.toIntDqReady <> intDq.io.enqReady
   dispatch1.io.toIntDq <> intDq.io.enq
-  dispatch1.io.toFpDqReady <> fpDq.io.enqReady
   dispatch1.io.toFpDq <> fpDq.io.enq
-  dispatch1.io.toLsDqReady <> lsDq.io.enqReady
   dispatch1.io.toLsDq <> lsDq.io.enq
   dispatch1.io.allocPregs <> io.allocPregs
 

src/main/scala/xiangshan/backend/dispatch/Dispatch1.scala

@@ -5,40 +5,35 @@ import chisel3.util._
 import chisel3.ExcitingUtils._
 import xiangshan._
 import utils.{XSDebug, XSError, XSInfo}
-import xiangshan.backend.roq.RoqPtr
+import xiangshan.backend.roq.{RoqPtr, RoqEnqIO}
 import xiangshan.backend.rename.RenameBypassInfo
+import xiangshan.mem.LsqEnqIO
 
 // read rob and enqueue
 class Dispatch1 extends XSModule {
   val io = IO(new Bundle() {
-    val redirect = Flipped(ValidIO(new Redirect))
     // from rename
     val fromRename = Vec(RenameWidth, Flipped(DecoupledIO(new MicroOp)))
     val renameBypass = Input(new RenameBypassInfo)
     val recv = Output(Vec(RenameWidth, Bool()))
     // enq Roq
-    val enqRoq = new Bundle {
+    val enqRoq = Flipped(new RoqEnqIO)
+    // enq Lsq
+    val enqLsq = Flipped(new LsqEnqIO)
+    val allocPregs = Vec(RenameWidth, Output(new ReplayPregReq))
+    // to dispatch queue
+    val toIntDq = new Bundle {
       val canAccept = Input(Bool())
-      val isEmpty = Input(Bool())
-      // if set, Roq needs extra walk
-      val extraWalk = Vec(RenameWidth, Output(Bool()))
       val req = Vec(RenameWidth, ValidIO(new MicroOp))
-      val resp = Vec(RenameWidth, Input(new RoqPtr))
     }
-    // enq Lsq
-    val enqLsq = new Bundle() {
+    val toFpDq = new Bundle {
+      val canAccept = Input(Bool())
+      val req = Vec(RenameWidth, ValidIO(new MicroOp))
+    }
+    val toLsDq = new Bundle {
       val canAccept = Input(Bool())
       val req = Vec(RenameWidth, ValidIO(new MicroOp))
-      val resp = Vec(RenameWidth, Input(new LSIdx))
     }
-    val allocPregs = Vec(RenameWidth, Output(new ReplayPregReq))
-    // to dispatch queue
-    val toIntDqReady = Input(Bool())
-    val toIntDq = Vec(dpParams.DqEnqWidth, ValidIO(new MicroOp))
-    val toFpDqReady = Input(Bool())
-    val toFpDq = Vec(dpParams.DqEnqWidth, ValidIO(new MicroOp))
-    val toLsDqReady = Input(Bool())
-    val toLsDq = Vec(dpParams.DqEnqWidth, ValidIO(new MicroOp))
   })
 
 
@@ -50,28 +45,10 @@ class Dispatch1 extends XSModule {
   val isFp    = VecInit(io.fromRename.map(req => FuType.isFpExu (req.bits.ctrl.fuType)))
   val isLs    = VecInit(io.fromRename.map(req => FuType.isMemExu(req.bits.ctrl.fuType)))
   val isStore = VecInit(io.fromRename.map(req => FuType.isStoreExu(req.bits.ctrl.fuType)))
+  val isAMO = VecInit(io.fromRename.map(req => req.bits.ctrl.fuType === FuType.mou))
   val isBlockBackward = VecInit(io.fromRename.map(_.bits.ctrl.blockBackward))
   val isNoSpecExec    = VecInit(io.fromRename.map(_.bits.ctrl.noSpecExec))
 
-  // generate index mapping
-  val intIndex = Module(new IndexMapping(RenameWidth, dpParams.DqEnqWidth, false))
-  val fpIndex  = Module(new IndexMapping(RenameWidth, dpParams.DqEnqWidth, false))
-  val lsIndex  = Module(new IndexMapping(RenameWidth, dpParams.DqEnqWidth, false))
-  for (i <- 0 until RenameWidth) {
-    intIndex.io.validBits(i) := isInt(i) && io.fromRename(i).valid
-    fpIndex.io.validBits(i)  := isFp(i)  && io.fromRename(i).valid
-    lsIndex.io.validBits(i)  := isLs(i)  && io.fromRename(i).valid
-  }
-  intIndex.io.priority := DontCare
-  fpIndex.io.priority  := DontCare
-  lsIndex.io.priority  := DontCare
-
-  if (!env.FPGAPlatform) {
-    val dispatchNotEmpty = Cat(io.fromRename.map(_.valid)).orR
-    ExcitingUtils.addSource(!dispatchNotEmpty, "perfCntCondDp1Empty", Perf)
-  }
-
-
   /**
     * Part 2:
     *   Update commitType, psrc1, psrc2, psrc3, old_pdest for the uops
@@ -84,7 +61,7 @@ class Dispatch1 extends XSModule {
   val updatedOldPdest = Wire(Vec(RenameWidth, UInt(PhyRegIdxWidth.W)))
 
   for (i <- 0 until RenameWidth) {
-    updatedCommitType(i) := Cat(isLs(i), isStore(i) | isFp(i))
+    updatedCommitType(i) := Cat(isLs(i) && !isAMO(i), isStore(i) | isFp(i))
     updatedPsrc1(i) := io.fromRename.take(i).map(_.bits.pdest)
       .zip(if (i == 0) Seq() else io.renameBypass.lsrc1_bypass(i-1).asBools)
       .foldLeft(io.fromRename(i).bits.psrc1) {
@@ -122,110 +99,85 @@ class Dispatch1 extends XSModule {
     *   acquire ROQ (all), LSQ (load/store only) and dispatch queue slots
     *   only set valid when all of them provides enough entries
     */
-  val redirectValid = io.redirect.valid// && !io.redirect.bits.isReplay
-  val allResourceReady = io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDqReady && io.toFpDqReady && io.toLsDqReady
+  val allResourceReady = io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
 
   // Instructions should enter dispatch queues in order.
-  // When RenameWidth > DqEnqWidth, it's possible that some instructions cannot enter dispatch queue
-  // because previous instructions cannot enter dispatch queue.
-  // The reason is that although ROB and LSQ have enough empty slots, dispatch queue has limited enqueue ports.
-  // Thus, for i >= dpParams.DqEnqWidth, we have to check whether it's previous instructions (and the instruction itself) can enqueue.
-  // However, since, for instructions with indices less than dpParams.DqEnqWidth,
-  // they can always enter dispatch queue when ROB and LSQ are ready, we don't need to check whether they can enqueue.
   // thisIsBlocked: this instruction is blocked by itself (based on noSpecExec)
-  // thisCanOut: this instruction can enqueue (based on resource)
-  // nextCanOut: next instructions can out (based on blockBackward and previous instructions)
+  // nextCanOut: next instructions can out (based on blockBackward)
   // notBlockedByPrevious: previous instructions can enqueue
   val thisIsBlocked = VecInit((0 until RenameWidth).map(i => {
     // for i > 0, when Roq is empty but dispatch1 have valid instructions to enqueue, it's blocked
     if (i > 0) isNoSpecExec(i) && (!io.enqRoq.isEmpty || Cat(io.fromRename.take(i).map(_.valid)).orR)
     else isNoSpecExec(i) && !io.enqRoq.isEmpty
   }))
-  val thisCanOut = VecInit((0 until RenameWidth).map(i => {
-    // For i in [0, DqEnqWidth), they can always enqueue when ROB and LSQ are ready
-    if (i < dpParams.DqEnqWidth) true.B
-    else Cat(Seq(intIndex, fpIndex, lsIndex).map(_.io.reverseMapping(i).valid)).orR
-  }))
   val nextCanOut = VecInit((0 until RenameWidth).map(i =>
-    (thisCanOut(i) && !isNoSpecExec(i) && !isBlockBackward(i)) || !io.fromRename(i).valid
+    (!isNoSpecExec(i) && !isBlockBackward(i)) || !io.fromRename(i).valid
   ))
   val notBlockedByPrevious = VecInit((0 until RenameWidth).map(i =>
     if (i == 0) true.B
     else Cat((0 until i).map(j => nextCanOut(j))).andR
   ))
 
+  // for noSpecExec: (roqEmpty || !this.noSpecExec) && !previous.noSpecExec
+  // For blockBackward:
   // this instruction can actually dequeue: 3 conditions
   // (1) resources are ready
   // (2) previous instructions are ready
-  val thisCanActualOut = (0 until RenameWidth).map(i => allResourceReady && thisCanOut(i) && !thisIsBlocked(i) && notBlockedByPrevious(i))
+  val thisCanActualOut = (0 until RenameWidth).map(i => !thisIsBlocked(i) && notBlockedByPrevious(i))
 
   // input for ROQ and LSQ
-  // note that LSQ needs roqIdx
+  // (1) LSQ needs roqIdx; (2) DPQ needs roqIdx and lsIdx
+  val updateUopWithIndex = Wire(Vec(RenameWidth, new MicroOp))
   for (i <- 0 until RenameWidth) {
-    io.enqRoq.extraWalk(i) := io.fromRename(i).valid && !thisCanActualOut(i)
-    io.enqRoq.req(i).valid := io.fromRename(i).valid && thisCanActualOut(i)
+    io.enqRoq.needAlloc(i) := io.fromRename(i).valid
+    io.enqRoq.req(i).valid := io.fromRename(i).valid && thisCanActualOut(i) && io.enqLsq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
     io.enqRoq.req(i).bits := updatedUop(i)
+    XSDebug(io.enqRoq.req(i).valid, p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)} receives nroq ${io.enqRoq.resp(i)}\n")
 
-    val shouldEnqLsq = isLs(i) && io.fromRename(i).bits.ctrl.fuType =/= FuType.mou
-    io.enqLsq.req(i).valid := io.fromRename(i).valid && shouldEnqLsq && !redirectValid && thisCanActualOut(i)
+    val shouldEnqLsq = isLs(i) && !isAMO(i)
+    io.enqLsq.needAlloc(i) := io.fromRename(i).valid && shouldEnqLsq
+    io.enqLsq.req(i).valid := io.fromRename(i).valid && shouldEnqLsq && thisCanActualOut(i) && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
     io.enqLsq.req(i).bits := updatedUop(i)
     io.enqLsq.req(i).bits.roqIdx := io.enqRoq.resp(i)
-
     XSDebug(io.enqLsq.req(i).valid,
       p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)} receives lq ${io.enqLsq.resp(i).lqIdx} sq ${io.enqLsq.resp(i).sqIdx}\n")
 
-    XSDebug(io.enqRoq.req(i).valid, p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)} receives nroq ${io.enqRoq.resp(i)}\n")
-  }
-
-
-  /**
-    * Part 4:
-    *   append ROQ and LSQ indexed to uop, and send them to dispatch queue
-    */
-  val updateUopWithIndex = Wire(Vec(RenameWidth, new MicroOp))
-  for (i <- 0 until RenameWidth) {
     updateUopWithIndex(i)        := updatedUop(i)
     updateUopWithIndex(i).roqIdx := io.enqRoq.resp(i)
     updateUopWithIndex(i).lqIdx  := io.enqLsq.resp(i).lqIdx
     updateUopWithIndex(i).sqIdx  := io.enqLsq.resp(i).sqIdx
-  }
 
-  // send uops with correct indexes to dispatch queues
-  // Note that if one of their previous instructions cannot enqueue, they should not enter dispatch queue.
-  // We use notBlockedByPrevious here since mapping(i).valid implies there's a valid instruction that can enqueue,
-  // thus we don't need to check thisCanOut.
-  for (i <- 0 until dpParams.DqEnqWidth) {
-    io.toIntDq(i).bits  := updateUopWithIndex(intIndex.io.mapping(i).bits)
-    io.toIntDq(i).valid := intIndex.io.mapping(i).valid && allResourceReady &&
-                           !thisIsBlocked(intIndex.io.mapping(i).bits) && notBlockedByPrevious(intIndex.io.mapping(i).bits)
-
-    // NOTE: floating point instructions are not noSpecExec currently
-    // remove commit /**/ when fp instructions are possible to be noSpecExec
-    io.toFpDq(i).bits   := updateUopWithIndex(fpIndex.io.mapping(i).bits)
-    io.toFpDq(i).valid  := fpIndex.io.mapping(i).valid && allResourceReady &&
-                           /*!thisIsBlocked(fpIndex.io.mapping(i).bits) && */notBlockedByPrevious(fpIndex.io.mapping(i).bits)
-
-    io.toLsDq(i).bits   := updateUopWithIndex(lsIndex.io.mapping(i).bits)
-    io.toLsDq(i).valid  := lsIndex.io.mapping(i).valid && allResourceReady &&
-                           !thisIsBlocked(lsIndex.io.mapping(i).bits) && notBlockedByPrevious(lsIndex.io.mapping(i).bits)
-
-    XSDebug(io.toIntDq(i).valid, p"pc 0x${Hexadecimal(io.toIntDq(i).bits.cf.pc)} int index $i\n")
-    XSDebug(io.toFpDq(i).valid , p"pc 0x${Hexadecimal(io.toFpDq(i).bits.cf.pc )} fp  index $i\n")
-    XSDebug(io.toLsDq(i).valid , p"pc 0x${Hexadecimal(io.toLsDq(i).bits.cf.pc )} ls  index $i\n")
+    // send uops to dispatch queues
+    // Note that if one of their previous instructions cannot enqueue, they should not enter dispatch queue.
+    // We use notBlockedByPrevious here.
+    io.toIntDq.req(i).bits  := updateUopWithIndex(i)
+    io.toIntDq.req(i).valid := io.fromRename(i).valid && isInt(i) && thisCanActualOut(i) &&
+                           io.enqLsq.canAccept && io.enqRoq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
+
+    io.toFpDq.req(i).bits   := updateUopWithIndex(i)
+    io.toFpDq.req(i).valid  := io.fromRename(i).valid && isFp(i) && thisCanActualOut(i) &&
+                           io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toLsDq.canAccept
+
+    io.toLsDq.req(i).bits   := updateUopWithIndex(i)
+    io.toLsDq.req(i).valid  := io.fromRename(i).valid && isLs(i) && thisCanActualOut(i) &&
+                           io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept
+
+    XSDebug(io.toIntDq.req(i).valid, p"pc 0x${Hexadecimal(io.toIntDq.req(i).bits.cf.pc)} int index $i\n")
+    XSDebug(io.toFpDq.req(i).valid , p"pc 0x${Hexadecimal(io.toFpDq.req(i).bits.cf.pc )} fp  index $i\n")
+    XSDebug(io.toLsDq.req(i).valid , p"pc 0x${Hexadecimal(io.toLsDq.req(i).bits.cf.pc )} ls  index $i\n")
   }
 
   /**
-    * Part 3: send response to rename when dispatch queue accepts the uop
+    * Part 4: send response to rename when dispatch queue accepts the uop
     */
-  val readyVector = (0 until RenameWidth).map(i => !io.fromRename(i).valid || io.recv(i))
+  val hasSpecialInstr = Cat((0 until RenameWidth).map(i => io.fromRename(i).valid && (isBlockBackward(i) || isNoSpecExec(i)))).orR
   for (i <- 0 until RenameWidth) {
-    io.recv(i) := thisCanActualOut(i)
-    io.fromRename(i).ready := Cat(readyVector).andR()
+    io.recv(i) := thisCanActualOut(i) && io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
+    io.fromRename(i).ready := !hasSpecialInstr && io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
 
-    XSInfo(io.recv(i),
+    XSInfo(io.recv(i) && io.fromRename(i).valid,
       p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)}, type(${isInt(i)}, ${isFp(i)}, ${isLs(i)}), " +
-      p"roq ${updateUopWithIndex(i).roqIdx}, lq ${updateUopWithIndex(i).lqIdx}, sq ${updateUopWithIndex(i).sqIdx}, " +
-      p"(${intIndex.io.reverseMapping(i).bits}, ${fpIndex.io.reverseMapping(i).bits}, ${lsIndex.io.reverseMapping(i).bits})\n"
+      p"roq ${updateUopWithIndex(i).roqIdx}, lq ${updateUopWithIndex(i).lqIdx}, sq ${updateUopWithIndex(i).sqIdx})\n"
     )
 
     io.allocPregs(i).isInt := io.fromRename(i).valid && io.fromRename(i).bits.ctrl.rfWen && (io.fromRename(i).bits.ctrl.ldest =/= 0.U)
@@ -233,6 +185,8 @@ class Dispatch1 extends XSModule {
     io.allocPregs(i).preg  := io.fromRename(i).bits.pdest
   }
   val renameFireCnt = PopCount(io.recv)
-  val enqFireCnt = PopCount(io.toIntDq.map(_.valid && io.toIntDqReady)) + PopCount(io.toFpDq.map(_.valid && io.toFpDqReady)) + PopCount(io.toLsDq.map(_.valid && io.toLsDqReady))
+  val enqFireCnt = PopCount(io.toIntDq.req.map(_.valid && io.toIntDq.canAccept)) +
+    PopCount(io.toFpDq.req.map(_.valid && io.toFpDq.canAccept)) +
+    PopCount(io.toLsDq.req.map(_.valid && io.toLsDq.canAccept))
   XSError(enqFireCnt > renameFireCnt, "enqFireCnt should not be greater than renameFireCnt\n")
 }

src/main/scala/xiangshan/backend/dispatch/DispatchQueue.scala

@@ -7,8 +7,10 @@ import xiangshan._
 import xiangshan.backend.roq.RoqPtr
 
 class DispatchQueueIO(enqnum: Int, deqnum: Int) extends XSBundle {
-  val enq = Vec(enqnum, Flipped(ValidIO(new MicroOp)))
-  val enqReady = Output(Bool())
+  val enq = new Bundle {
+    val canAccept = Output(Bool())
+    val req = Vec(enqnum, Flipped(ValidIO(new MicroOp)))
+  }
   val deq = Vec(deqnum, DecoupledIO(new MicroOp))
   val redirect = Flipped(ValidIO(new Redirect))
   override def cloneType: DispatchQueueIO.this.type =
@@ -27,22 +29,16 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
   val stateEntries = RegInit(VecInit(Seq.fill(size)(s_invalid)))
 
   // head: first valid entry (dispatched entry)
-  val headPtr = RegInit(0.U.asTypeOf(new CircularQueuePtr(size)))
-  val headPtrMask = UIntToMask(headPtr.value, size)
+  val headPtr = RegInit(VecInit((0 until deqnum).map(_.U.asTypeOf(new CircularQueuePtr(size)))))
+  val headPtrMask = UIntToMask(headPtr(0).value, size)
   // tail: first invalid entry (free entry)
-  val tailPtr = RegInit(0.U.asTypeOf(new CircularQueuePtr(size)))
-  val tailPtrMask = UIntToMask(tailPtr.value, size)
-
-  // TODO: make ptr a vector to reduce latency?
-  // deq: starting from head ptr
-  val deqIndex = (0 until deqnum).map(i => headPtr + i.U).map(_.value)
-  // enq: starting from tail ptr
-  val enqIndex = (0 until enqnum).map(i => tailPtr + i.U).map(_.value)
+  val tailPtr = RegInit(VecInit((0 until enqnum).map(_.U.asTypeOf(new CircularQueuePtr(size)))))
+  val tailPtrMask = UIntToMask(tailPtr(0).value, size)
 
-  val validEntries = distanceBetween(tailPtr, headPtr)
+  val validEntries = distanceBetween(tailPtr(0), headPtr(0))
   val isTrueEmpty = ~Cat((0 until size).map(i => stateEntries(i) === s_valid)).orR
   val canEnqueue = validEntries <= (size - enqnum).U
-  val canActualEnqueue = canEnqueue && !(io.redirect.valid /*&& !io.redirect.bits.isReplay*/)
+  val canActualEnqueue = canEnqueue && !io.redirect.valid
 
   /**
     * Part 1: update states and uops when enqueue, dequeue, commit, redirect/replay
@@ -57,25 +53,26 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
     * (5) redirect (replay): from s_dispatched to s_valid (re-dispatch)
     */
   // enqueue: from s_invalid to s_valid
-  io.enqReady := canEnqueue
+  io.enq.canAccept := canEnqueue
   for (i <- 0 until enqnum) {
-    when (io.enq(i).valid && canActualEnqueue) {
-      uopEntries(enqIndex(i)) := io.enq(i).bits
-      stateEntries(enqIndex(i)) := s_valid
+    when (io.enq.req(i).valid && canActualEnqueue) {
+      val sel = if (i == 0) 0.U else PopCount(io.enq.req.take(i).map(_.valid))
+      uopEntries(tailPtr(sel).value) := io.enq.req(i).bits
+      stateEntries(tailPtr(sel).value) := s_valid
     }
   }
 
   // dequeue: from s_valid to s_dispatched
   for (i <- 0 until deqnum) {
     when (io.deq(i).fire() && !io.redirect.valid) {
-      stateEntries(deqIndex(i)) := s_invalid
+      stateEntries(headPtr(i).value) := s_invalid
 
-      XSError(stateEntries(deqIndex(i)) =/= s_valid, "state of the dispatch entry is not s_valid\n")
+      XSError(stateEntries(headPtr(i).value) =/= s_valid, "state of the dispatch entry is not s_valid\n")
     }
   }
 
   // redirect: cancel uops currently in the queue
-  val mispredictionValid = io.redirect.valid //&& io.redirect.bits.isMisPred
+  val mispredictionValid = io.redirect.valid
   val exceptionValid = io.redirect.valid && io.redirect.bits.isException
   val flushPipeValid = io.redirect.valid && io.redirect.bits.isFlushPipe
   val roqNeedFlush = Wire(Vec(size, Bool()))
@@ -106,12 +103,15 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
     // For dequeue, the first entry should never be s_invalid
     // Otherwise, there should be a redirect and tail walks back
     // in this case, we set numDeq to 0
-    !deq.fire() && (if (i == 0) true.B else stateEntries(deqIndex(i)) =/= s_invalid)
+    !deq.fire() && (if (i == 0) true.B else stateEntries(headPtr(i).value) =/= s_invalid)
   } :+ true.B)
   val numDeq = Mux(numDeqTry > numDeqFire, numDeqFire, numDeqTry)
   // agreement with reservation station: don't dequeue when redirect.valid
-  val headPtrNext = Mux(mispredictionValid, headPtr, headPtr + numDeq)
-  headPtr := Mux(exceptionValid, 0.U.asTypeOf(new CircularQueuePtr(size)), headPtrNext)
+  for (i <- 0 until deqnum) {
+    headPtr(i) := Mux(exceptionValid,
+      i.U.asTypeOf(new CircularQueuePtr(size)),
+      Mux(mispredictionValid, headPtr(i), headPtr(i) + numDeq))
+  }
 
   // For branch mis-prediction or memory violation replay,
   // we delay updating the indices for one clock cycle.
@@ -124,18 +124,31 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
   val flippedFlag = loValidBitVec.orR
   val lastOneIndex = size.U - PriorityEncoder(Mux(loValidBitVec.orR, loValidBitVec, hiValidBitVec))
   val walkedTailPtr = Wire(new CircularQueuePtr(size))
-  walkedTailPtr.flag := flippedFlag ^ headPtr.flag
+  walkedTailPtr.flag := flippedFlag ^ headPtr(0).flag
   walkedTailPtr.value := lastOneIndex
 
   // enqueue
-  val numEnq = Mux(canActualEnqueue, PriorityEncoder(io.enq.map(!_.valid) :+ true.B), 0.U)
-  XSError(numEnq =/= 0.U && (mispredictionValid || exceptionValid), "should not enqueue when redirect\n")
-  tailPtr := Mux(exceptionValid,
+  val numEnq = Mux(io.enq.canAccept, PopCount(io.enq.req.map(_.valid)), 0.U)
+  tailPtr(0) := Mux(exceptionValid,
     0.U.asTypeOf(new CircularQueuePtr(size)),
-    Mux(lastCycleMisprediction,
-      Mux(isTrueEmpty, headPtr, walkedTailPtr),
-      tailPtr + numEnq)
+    Mux(io.redirect.valid,
+      tailPtr(0),
+      Mux(lastCycleMisprediction,
+        Mux(isTrueEmpty, headPtr(0), walkedTailPtr),
+        tailPtr(0) + numEnq))
   )
+  val lastCycleException = RegNext(exceptionValid)
+  val lastLastCycleMisprediction = RegNext(lastCycleMisprediction)
+  for (i <- 1 until enqnum) {
+    tailPtr(i) := Mux(exceptionValid,
+      i.U.asTypeOf(new CircularQueuePtr(size)),
+      Mux(io.redirect.valid,
+        tailPtr(i),
+        Mux(lastLastCycleMisprediction,
+          tailPtr(0) + i.U,
+          tailPtr(i) + numEnq))
+      )
+  }
 
 
   /**
@@ -143,13 +156,13 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
     */
   // TODO: remove this when replay moves to roq
   for (i <- 0 until deqnum) {
-    io.deq(i).bits := uopEntries(deqIndex(i))
+    io.deq(i).bits := uopEntries(headPtr(i).value)
     // do not dequeue when io.redirect valid because it may cause dispatchPtr work improperly
-    io.deq(i).valid := stateEntries(deqIndex(i)) === s_valid && !lastCycleMisprediction// && !io.redirect.valid
+    io.deq(i).valid := stateEntries(headPtr(i).value) === s_valid && !lastCycleMisprediction
   }
 
   // debug: dump dispatch queue states
-  XSDebug(p"head: $headPtr, tail: $tailPtr\n")
+  XSDebug(p"head: ${headPtr(0)}, tail: ${tailPtr(0)}\n")
   XSDebug(p"state: ")
   stateEntries.reverse.foreach { s =>
     XSDebug(false, s === s_invalid, "-")
@@ -158,11 +171,11 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
   XSDebug(false, true.B, "\n")
   XSDebug(p"ptr:   ")
   (0 until size).reverse.foreach { i =>
-    val isPtr = i.U === headPtr.value || i.U === tailPtr.value
+    val isPtr = i.U === headPtr(0).value || i.U === tailPtr(0).value
     XSDebug(false, isPtr, "^")
     XSDebug(false, !isPtr, " ")
   }
   XSDebug(false, true.B, "\n")
 
-  XSError(isAfter(headPtr, tailPtr), p"assert greaterOrEqualThan(tailPtr: $tailPtr, headPtr: $headPtr) failed\n")
+  XSError(isAfter(headPtr(0), tailPtr(0)), p"assert greaterOrEqualThan(tailPtr: ${tailPtr(0)}, headPtr: ${headPtr(0)}) failed\n")
 }

src/main/scala/xiangshan/backend/fu/CSR.scala

@@ -828,7 +828,6 @@ class CSR extends FunctionUnit with HasCSRConst
     "RoqWaitFp"   -> (0xb11, "perfCntCondRoqWaitFp"   ),
     "RoqWaitLoad" -> (0xb12, "perfCntCondRoqWaitLoad" ),
     "RoqWaitStore"-> (0xb13, "perfCntCondRoqWaitStore"),
-    "Dp1Empty"    -> (0xb14, "perfCntCondDp1Empty"    ),
     "DTlbReqCnt0" -> (0xb15, "perfCntDtlbReqCnt0"     ),
     "DTlbReqCnt1" -> (0xb16, "perfCntDtlbReqCnt1"     ),
     "DTlbReqCnt2" -> (0xb17, "perfCntDtlbReqCnt2"     ),

src/main/scala/xiangshan/backend/roq/Roq.scala

@@ -38,17 +38,20 @@ class RoqCSRIO extends XSBundle {
   val dirty_fs = Output(Bool())
 }
 
+class RoqEnqIO extends XSBundle {
+  val canAccept = Output(Bool())
+  val isEmpty = Output(Bool())
+  // valid vector, for roqIdx gen and walk
+  val needAlloc = Vec(RenameWidth, Input(Bool()))
+  val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
+  val resp = Vec(RenameWidth, Output(new RoqPtr))
+}
+
 class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   val io = IO(new Bundle() {
     val brqRedirect = Input(Valid(new Redirect))
     val memRedirect = Input(Valid(new Redirect))
-    val enq = new Bundle {
-      val canAccept = Output(Bool())
-      val isEmpty = Output(Bool())
-      val extraWalk = Vec(RenameWidth, Input(Bool()))
-      val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
-      val resp = Vec(RenameWidth, Output(new RoqPtr))
-    }
+    val enq = new RoqEnqIO
     val redirect = Output(Valid(new Redirect))
     val exception = Output(new MicroOp)
     // exu + brq
@@ -108,14 +111,7 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   io.roqDeqPtr := deqPtrExt
 
   // common signal
-  val enqPtrValPlus = Wire(Vec(RenameWidth, UInt(log2Up(RoqSize).W)))
-  val enqPtrFlagPlus = Wire(Vec(RenameWidth, Bool()))
-  for (i <- 0 until RenameWidth) {
-    val offset = PopCount(io.enq.req.map(_.valid).take(i))
-    val roqIdxExt = enqPtrExt + offset
-    enqPtrValPlus(i) := roqIdxExt.value
-    enqPtrFlagPlus(i) := roqIdxExt.flag
-  }
+  val enqPtrVec = WireInit(VecInit((0 until RenameWidth).map(i => enqPtrExt + PopCount(io.enq.needAlloc.take(i)))))
 
   val deqPtrExtPlus = Wire(Vec(RenameWidth, UInt(log2Up(RoqSize).W)))
   for(i <- 0 until CommitWidth){
@@ -138,14 +134,9 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   val walkNoSpecExec = io.commits.isWalk && Cat((0 until CommitWidth).map(i => io.commits.valid(i) && io.commits.uop(i).ctrl.noSpecExec)).orR
   XSError(state =/= s_extrawalk && walkNoSpecExec, "noSpecExec should not walk\n")
 
-  val validDispatch = io.enq.req.map(_.valid)
   for (i <- 0 until RenameWidth) {
-    val offset = PopCount(validDispatch.take(i))
-    val roqIdxExt = enqPtrExt + offset
-    val roqIdx = roqIdxExt.value
-
-    when(io.enq.req(i).valid) {
-      microOp(roqIdx) := io.enq.req(i).bits
+    when(io.enq.req(i).valid && io.enq.canAccept) {
+      microOp(enqPtrVec(i).value) := io.enq.req(i).bits
       when(io.enq.req(i).bits.ctrl.blockBackward) {
         hasBlockBackward := true.B
       }
@@ -153,19 +144,18 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
         hasNoSpecExec := true.B
       }
     }
-    io.enq.resp(i) := roqIdxExt
+    io.enq.resp(i) := enqPtrVec(i)
   }
 
   val validEntries = distanceBetween(enqPtrExt, deqPtrExt)
-  val firedDispatch = Cat(io.enq.req.map(_.valid))
+  val firedDispatch = Mux(io.enq.canAccept, PopCount(Cat(io.enq.req.map(_.valid))), 0.U)
   io.enq.canAccept := (validEntries <= (RoqSize - RenameWidth).U) && !hasBlockBackward
   io.enq.isEmpty   := isEmpty
-  XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(firedDispatch)}\n")
+  XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n")
 
-  val dispatchCnt = PopCount(firedDispatch)
-  enqPtrExt := enqPtrExt + dispatchCnt
-  when (firedDispatch.orR) {
-    XSInfo("dispatched %d insts\n", dispatchCnt)
+  enqPtrExt := enqPtrExt + firedDispatch
+  when (firedDispatch =/= 0.U) {
+    XSInfo("dispatched %d insts\n", firedDispatch)
   }
 
   // Writeback
@@ -224,7 +214,7 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
 
   // extra space is used weh roq has no enough space, but mispredict recovery needs such info to walk regmap
   val needExtraSpaceForMPR = WireInit(VecInit(
-    List.tabulate(RenameWidth)(i => io.brqRedirect.valid && io.enq.extraWalk(i))
+    List.tabulate(RenameWidth)(i => io.brqRedirect.valid && io.enq.needAlloc(i))
   ))
   val extraSpaceForMPR = Reg(Vec(RenameWidth, new MicroOp))
   val usedSpaceForMPR = Reg(Vec(RenameWidth, Bool()))
@@ -345,12 +335,12 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   // when redirect, walk back roq entries
   when(io.brqRedirect.valid){ // TODO: need check if consider exception redirect?
     state := s_walk
-    val nextEnqPtr = (enqPtrExt - 1.U) + dispatchCnt
+    val walkPtrStart = enqPtrExt - 1.U
     walkPtrExt := Mux(state === s_walk,
       walkPtrExt - Mux(walkFinished, walkCounter, CommitWidth.U),
-      Mux(state === s_extrawalk, walkPtrExt, nextEnqPtr))
+      Mux(state === s_extrawalk, walkPtrExt, walkPtrStart))
     // walkTgtExt := io.brqRedirect.bits.roqIdx
-    val currentWalkPtr = Mux(state === s_walk || state === s_extrawalk, walkPtrExt, nextEnqPtr)
+    val currentWalkPtr = Mux(state === s_walk || state === s_extrawalk, walkPtrExt, walkPtrStart)
     walkCounter := distanceBetween(currentWalkPtr, io.brqRedirect.bits.roqIdx) - Mux(state === s_walk, commitCnt, 0.U)
     enqPtrExt := io.brqRedirect.bits.roqIdx + 1.U
   }
@@ -375,8 +365,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   // write
   // enqueue logic writes 6 valid
   for (i <- 0 until RenameWidth) {
-    when(io.enq.req(i).fire()){
-      valid(enqPtrValPlus(i)) := true.B
+    when(io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid){
+      valid(enqPtrVec(i).value) := true.B
     }
   }
   // dequeue/walk logic writes 6 valid, dequeue and walk will not happen at the same time
@@ -412,8 +402,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   // write
   // enqueue logic set 6 writebacked to false
   for (i <- 0 until RenameWidth) {
-    when(io.enq.req(i).fire()){
-      writebacked(enqPtrValPlus(i)) := false.B
+    when(io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid){
+      writebacked(enqPtrVec(i).value) := false.B
     }
   }
   // writeback logic set numWbPorts writebacked to true
@@ -443,8 +433,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   // write: update when enqueue
   // enqueue logic set 6 flagBkup at most
   for (i <- 0 until RenameWidth) {
-    when(io.enq.req(i).fire()){
-      flagBkup(enqPtrValPlus(i)) := enqPtrFlagPlus(i)
+    when(io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid){
+      flagBkup(enqPtrVec(i).value) := enqPtrVec(i).flag
     }
   }
   // read: used in rollback logic

src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala

@@ -53,7 +53,7 @@ class LSQueueData(size: Int, nchannel: Int) extends XSModule with HasDCacheParam
     val needForward = Input(Vec(nchannel, Vec(2, UInt(size.W))))
     val forward = Vec(nchannel, Flipped(new LoadForwardQueryIO))
     val rdata = Output(Vec(size, new LsqEntry))
-    
+
     // val debug = new Bundle() {
     //   val debug_data = Vec(LoadQueueSize, new LsqEntry)
     // }
@@ -76,7 +76,7 @@ class LSQueueData(size: Int, nchannel: Int) extends XSModule with HasDCacheParam
       this.needForward(channel)(1) := needForward2
       this.forward(channel).paddr := paddr
     }
-    
+
     // def refillWrite(ldIdx: Int): Unit = {
     // }
     // use "this.refill.wen(ldIdx) := true.B" instead
@@ -229,14 +229,17 @@ class InflightBlockInfo extends XSBundle {
   val valid = Bool()
 }
 
+class LsqEnqIO extends XSBundle {
+  val canAccept = Output(Bool())
+  val needAlloc = Vec(RenameWidth, Input(Bool()))
+  val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
+  val resp = Vec(RenameWidth, Output(new LSIdx))
+}
+
 // Load / Store Queue Wrapper for XiangShan Out of Order LSU
 class LsqWrappper extends XSModule with HasDCacheParameters {
   val io = IO(new Bundle() {
-    val enq = new Bundle() {
-      val canAccept = Output(Bool())
-      val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
-      val resp = Vec(RenameWidth, Output(new LSIdx))
-    }
+    val enq = new LsqEnqIO
     val brqRedirect = Input(Valid(new Redirect))
     val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LsPipelineBundle)))
     val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle)))
@@ -261,14 +264,17 @@ class LsqWrappper extends XSModule with HasDCacheParameters {
   io.enq.canAccept := loadQueue.io.enq.canAccept && storeQueue.io.enq.canAccept
   for (i <- 0 until RenameWidth) {
     val isStore = CommitType.lsInstIsStore(io.enq.req(i).bits.ctrl.commitType)
+
+    loadQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i) && !isStore
     loadQueue.io.enq.req(i).valid  := !isStore && io.enq.req(i).valid
-    storeQueue.io.enq.req(i).valid :=  isStore && io.enq.req(i).valid
     loadQueue.io.enq.req(i).bits  := io.enq.req(i).bits
+
+    storeQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i) && isStore
+    storeQueue.io.enq.req(i).valid :=  isStore && io.enq.req(i).valid
     storeQueue.io.enq.req(i).bits := io.enq.req(i).bits
+
     io.enq.resp(i).lqIdx := loadQueue.io.enq.resp(i)
     io.enq.resp(i).sqIdx := storeQueue.io.enq.resp(i)
-
-    XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not enqueue LSQ when not")
   }
 
   // load queue wiring
@@ -294,7 +300,7 @@ class LsqWrappper extends XSModule with HasDCacheParameters {
   storeQueue.io.exceptionAddr.lsIdx := io.exceptionAddr.lsIdx
   storeQueue.io.exceptionAddr.isStore := DontCare
 
-  loadQueue.io.forward <> io.forward
+  loadQueue.io.load_s1 <> io.forward
   storeQueue.io.forward <> io.forward // overlap forwardMask & forwardData, DO NOT CHANGE SEQUENCE
 
   io.exceptionAddr.vaddr := Mux(io.exceptionAddr.isStore, storeQueue.io.exceptionAddr.vaddr, loadQueue.io.exceptionAddr.vaddr)

src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala

@@ -23,27 +23,28 @@ object LqPtr extends HasXSParameter {
   }
 }
 
+class LqEnqIO extends XSBundle {
+  val canAccept = Output(Bool())
+  val needAlloc = Vec(RenameWidth, Input(Bool()))
+  val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
+  val resp = Vec(RenameWidth, Output(new LqPtr))
+}
 
 // Load Queue
 class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper {
   val io = IO(new Bundle() {
-    val enq = new Bundle() {
-      val canAccept = Output(Bool())
-      val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
-      val resp = Vec(RenameWidth, Output(new LqPtr))
-    }
+    val enq = new LqEnqIO
     val brqRedirect = Input(Valid(new Redirect))
     val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LsPipelineBundle)))
     val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // FIXME: Valid() only
     val ldout = Vec(2, DecoupledIO(new ExuOutput)) // writeback load
-    val forward = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO))
+    val load_s1 = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO))
     val commits = Flipped(new RoqCommitIO)
     val rollback = Output(Valid(new Redirect)) // replay now starts from load instead of store
     val dcache = new DCacheLineIO
     val uncache = new DCacheWordIO
     val roqDeqPtr = Input(new RoqPtr)
     val exceptionAddr = new ExceptionAddrIO
-    // val refill = Flipped(Valid(new DCacheLineReq ))
   })
 
   val uop = Reg(Vec(LoadQueueSize, new MicroOp))
@@ -58,11 +59,11 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
   val listening = Reg(Vec(LoadQueueSize, Bool())) // waiting for refill result
   val pending = Reg(Vec(LoadQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of roq
 
-  val enqPtrExt = RegInit(0.U.asTypeOf(new LqPtr))
+  val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new LqPtr))))
   val deqPtrExt = RegInit(0.U.asTypeOf(new LqPtr))
-  val enqPtr = enqPtrExt.value
+  val enqPtr = enqPtrExt(0).value
   val deqPtr = deqPtrExt.value
-  val sameFlag = enqPtrExt.flag === deqPtrExt.flag
+  val sameFlag = enqPtrExt(0).flag === deqPtrExt.flag
   val isEmpty = enqPtr === deqPtr && sameFlag
   val isFull = enqPtr === deqPtr && !sameFlag
   val allowIn = !isFull
@@ -72,19 +73,21 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
 
   val deqMask = UIntToMask(deqPtr, LoadQueueSize)
   val enqMask = UIntToMask(enqPtr, LoadQueueSize)
-  val enqDeqMask1 = deqMask ^ enqMask
-  val enqDeqMask = Mux(sameFlag, enqDeqMask1, ~enqDeqMask1)
 
-  // Enqueue at dispatch
-  val validEntries = distanceBetween(enqPtrExt, deqPtrExt)
+  /**
+    * Enqueue at dispatch
+    *
+    * Currently, LoadQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth)
+    */
+  val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt)
   val firedDispatch = io.enq.req.map(_.valid)
   io.enq.canAccept := validEntries <= (LoadQueueSize - RenameWidth).U
   XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n")
   for (i <- 0 until RenameWidth) {
-    val offset = if (i == 0) 0.U else PopCount((0 until i).map(firedDispatch(_)))
-    val lqIdx = enqPtrExt + offset
+    val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i))
+    val lqIdx = enqPtrExt(offset)
     val index = lqIdx.value
-    when(io.enq.req(i).valid) {
+    when (io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid) {
       uop(index) := io.enq.req(i).bits
       allocated(index) := true.B
       datavalid(index) := false.B
@@ -95,17 +98,28 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
       pending(index) := false.B
     }
     io.enq.resp(i) := lqIdx
-
-    XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not valid when not ready\n")
   }
 
-  when(Cat(firedDispatch).orR) {
-    enqPtrExt := enqPtrExt + PopCount(firedDispatch)
-    XSInfo("dispatched %d insts to lq\n", PopCount(firedDispatch))
+  // when io.brqRedirect.valid, we don't allow eneuque even though it may fire.
+  when (Cat(firedDispatch).orR && io.enq.canAccept && !io.brqRedirect.valid) {
+    val enqNumber = PopCount(firedDispatch)
+    enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber))
+    XSInfo("dispatched %d insts to lq\n", enqNumber)
   }
 
-  // writeback load
-  (0 until LoadPipelineWidth).map(i => {
+  /**
+    * Writeback load from load units
+    *
+    * Most load instructions writeback to regfile at the same time.
+    * However,
+    *   (1) For an mmio instruction with exceptions, it writes back to ROB immediately.
+    *   (2) For an mmio instruction without exceptions, it does not write back.
+    * The mmio instruction will be sent to lower level when it reaches ROB's head.
+    * After uncache response, it will write back through arbiter with loadUnit.
+    *   (3) For cache misses, it is marked miss and sent to dcache later.
+    * After cache refills, it will write back through arbiter with loadUnit.
+    */
+  for (i <- 0 until LoadPipelineWidth) {
     dataModule.io.wb(i).wen := false.B
     when(io.loadIn(i).fire()) {
       when(io.loadIn(i).bits.miss) {
@@ -140,7 +154,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
         val loadWbIndex = io.loadIn(i).bits.uop.lqIdx.value
         datavalid(loadWbIndex) := !io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio
         writebacked(loadWbIndex) := !io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio
-        allocated(loadWbIndex) := !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR
 
         val loadWbData = Wire(new LsqEntry)
         loadWbData.paddr := io.loadIn(i).bits.paddr
@@ -155,13 +168,20 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
         dataModule.io.wb(i).wen := true.B
 
         val dcacheMissed = io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio
-        miss(loadWbIndex) := dcacheMissed
+        miss(loadWbIndex) := dcacheMissed && !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR
         listening(loadWbIndex) := dcacheMissed
-        pending(loadWbIndex) := io.loadIn(i).bits.mmio
+        pending(loadWbIndex) := io.loadIn(i).bits.mmio && !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR
       }
-    })
+    }
 
-  // cache miss request
+  /**
+    * Cache miss request
+    *
+    * (1) writeback: miss
+    * (2) send to dcache: listing
+    * (3) dcache response: datavalid
+    * (4) writeback to ROB: writeback
+    */
   val inflightReqs = RegInit(VecInit(Seq.fill(cfg.nLoadMissEntries)(0.U.asTypeOf(new InflightBlockInfo))))
   val inflightReqFull = inflightReqs.map(req => req.valid).reduce(_&&_)
   val reqBlockIndex = PriorityEncoder(~VecInit(inflightReqs.map(req => req.valid)).asUInt)
@@ -303,21 +323,18 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
     }
   })
 
-  // move tailPtr
-  // allocatedMask: dequeuePtr can go to the next 1-bit
-  val allocatedMask = VecInit((0 until LoadQueueSize).map(i => allocated(i) || !enqDeqMask(i)))
-  // find the first one from deqPtr (deqPtr)
-  val nextTail1 = getFirstOneWithFlag(allocatedMask, deqMask, deqPtrExt.flag)
-  val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, enqPtrExt)
-  deqPtrExt := nextTail
-
-  // When load commited, mark it as !allocated, this entry will be recycled later
+  /**
+    * Load commits
+    *
+    * When load commited, mark it as !allocated and move deqPtrExt forward.
+    */
   (0 until CommitWidth).map(i => {
     when(loadCommit(i)) {
       allocated(mcommitIdx(i)) := false.B
       XSDebug("load commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc)
     }
   })
+  deqPtrExt := deqPtrExt + PopCount(loadCommit)
 
   def getFirstOne(mask: Vec[Bool], startMask: UInt) = {
     val length = mask.length
@@ -326,15 +343,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
     PriorityEncoder(Mux(highBitsUint.orR(), highBitsUint, mask.asUInt))
   }
 
-  def getFirstOneWithFlag(mask: Vec[Bool], startMask: UInt, startFlag: Bool) = {
-    val length = mask.length
-    val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
-    val highBitsUint = Cat(highBits.reverse)
-    val changeDirection = !highBitsUint.orR()
-    val index = PriorityEncoder(Mux(!changeDirection, highBitsUint, mask.asUInt))
-    LqPtr(startFlag ^ changeDirection, index)
-  }
-
   def getOldestInTwo(valid: Seq[Bool], uop: Seq[MicroOp]) = {
     assert(valid.length == uop.length)
     assert(valid.length == 2)
@@ -355,25 +363,25 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
     })
   }
 
-  def rangeMask(start: LqPtr, end: LqPtr): UInt = {
-    val startMask = (1.U((LoadQueueSize + 1).W) << start.value).asUInt - 1.U
-    val endMask = (1.U((LoadQueueSize + 1).W) << end.value).asUInt - 1.U
-    val xorMask = startMask(LoadQueueSize - 1, 0) ^ endMask(LoadQueueSize - 1, 0)
-    Mux(start.flag === end.flag, xorMask, ~xorMask)
-  }
-
-  // ignore data forward
-  (0 until LoadPipelineWidth).foreach(i => {
-    io.forward(i).forwardMask := DontCare
-    io.forward(i).forwardData := DontCare
-  })
-
-  // store backward query and rollback
+  /**
+    * Memory violation detection
+    *
+    * When store writes back, it searches LoadQueue for younger load instructions
+    * with the same load physical address. They loaded wrong data and need re-execution.
+    *
+    * Cycle 0: Store Writeback
+    *   Generate match vector for store address with rangeMask(stPtr, enqPtr).
+    *   Besides, load instructions in LoadUnit_S1 and S2 are also checked.
+    * Cycle 1: Redirect Generation
+    *   There're three possible types of violations. Choose the oldest load.
+    *   Set io.redirect according to the detected violation.
+    */
+  io.load_s1 := DontCare
   def detectRollback(i: Int) = {
     val startIndex = io.storeIn(i).bits.uop.lqIdx.value
     val lqIdxMask = UIntToMask(startIndex, LoadQueueSize)
     val xorMask = lqIdxMask ^ enqMask
-    val sameFlag = io.storeIn(i).bits.uop.lqIdx.flag === enqPtrExt.flag
+    val sameFlag = io.storeIn(i).bits.uop.lqIdx.flag === enqPtrExt(0).flag
     val toEnqPtrMask = Mux(sameFlag, xorMask, ~xorMask)
 
     // check if load already in lq needs to be rolledback
@@ -405,13 +413,13 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
 
     // check if rollback is needed for load in l1
     val l1ViolationVec = RegNext(VecInit((0 until LoadPipelineWidth).map(j => {
-      io.forward(j).valid && // L1 valid
-        isAfter(io.forward(j).uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) &&
-        io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.forward(j).paddr(PAddrBits - 1, 3) &&
-        (io.storeIn(i).bits.mask & io.forward(j).mask).orR
+      io.load_s1(j).valid && // L1 valid
+        isAfter(io.load_s1(j).uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) &&
+        io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.load_s1(j).paddr(PAddrBits - 1, 3) &&
+        (io.storeIn(i).bits.mask & io.load_s1(j).mask).orR
     })))
     val l1Violation = l1ViolationVec.asUInt().orR()
-    val l1ViolationUop = getOldestInTwo(l1ViolationVec, RegNext(VecInit(io.forward.map(_.uop))))
+    val l1ViolationUop = getOldestInTwo(l1ViolationVec, RegNext(VecInit(io.load_s1.map(_.uop))))
     XSDebug(l1Violation, p"${Binary(Cat(l1ViolationVec))}, $l1ViolationUop\n")
 
     val rollbackValidVec = Seq(lqViolation, wbViolation, l1Violation)
@@ -465,10 +473,10 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
   val rollbackSelected = ParallelOperation(rollback, rollbackSel)
   val lastCycleRedirect = RegNext(io.brqRedirect)
 
-  io.rollback := DontCare
   // Note that we use roqIdx - 1.U to flush the load instruction itself.
   // Thus, here if last cycle's roqIdx equals to this cycle's roqIdx, it still triggers the redirect.
-  io.rollback.valid := rollbackSelected.valid && (!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx)) &&
+  io.rollback.valid := rollbackSelected.valid &&
+    (!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx)) &&
     !(lastCycleRedirect.valid && (lastCycleRedirect.bits.isFlushPipe || lastCycleRedirect.bits.isException))
 
   io.rollback.bits.roqIdx := rollbackSelected.bits.roqIdx - 1.U
@@ -476,13 +484,18 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
   io.rollback.bits.isMisPred := false.B
   io.rollback.bits.isException := false.B
   io.rollback.bits.isFlushPipe := false.B
+  io.rollback.bits.pc := DontCare
   io.rollback.bits.target := rollbackSelected.bits.cf.pc
   io.rollback.bits.brTag := rollbackSelected.bits.brTag
 
-  // Memory mapped IO / other uncached operations
+  when(io.rollback.valid) {
+    XSDebug("Mem rollback: pc %x roqidx %d\n", io.rollback.bits.pc, io.rollback.bits.roqIdx.asUInt)
+  }
 
-  // setup misc mem access req
-  // mask / paddr / data can be get from lq.data
+  /**
+    * Memory mapped IO / other uncached operations
+    *
+    */
   val commitType = io.commits.uop(0).ctrl.commitType
   io.uncache.req.valid := pending(deqPtr) && allocated(deqPtr) &&
     commitType === CommitType.LOAD &&
@@ -494,11 +507,11 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
   io.uncache.req.bits.data := dataModule.io.rdata(deqPtr).data
   io.uncache.req.bits.mask := dataModule.io.rdata(deqPtr).mask
 
-  io.uncache.req.bits.meta.id       := DontCare // TODO: // FIXME
+  io.uncache.req.bits.meta.id       := DontCare
   io.uncache.req.bits.meta.vaddr    := DontCare
   io.uncache.req.bits.meta.paddr    := dataModule.io.rdata(deqPtr).paddr
   io.uncache.req.bits.meta.uop      := uop(deqPtr)
-  io.uncache.req.bits.meta.mmio     := true.B // dataModule.io.rdata(deqPtr).mmio
+  io.uncache.req.bits.meta.mmio     := true.B
   io.uncache.req.bits.meta.tlb_miss := false.B
   io.uncache.req.bits.meta.mask     := dataModule.io.rdata(deqPtr).mask
   io.uncache.req.bits.meta.replay   := false.B
@@ -507,17 +520,7 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
 
   when (io.uncache.req.fire()) {
     pending(deqPtr) := false.B
-  }
 
-  dataModule.io.uncache.wen := false.B
-  when(io.uncache.resp.fire()){
-    datavalid(deqPtr) := true.B
-    dataModule.io.uncacheWrite(deqPtr, io.uncache.resp.bits.data(XLEN-1, 0))
-    dataModule.io.uncache.wen := true.B
-    // TODO: write back exception info
-  }
-
-  when(io.uncache.req.fire()){
     XSDebug("uncache req: pc %x addr %x data %x op %x mask %x\n",
       uop(deqPtr).cf.pc,
       io.uncache.req.bits.addr,
@@ -527,7 +530,12 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
     )
   }
 
+  dataModule.io.uncache.wen := false.B
   when(io.uncache.resp.fire()){
+    datavalid(deqPtr) := true.B
+    dataModule.io.uncacheWrite(deqPtr, io.uncache.resp.bits.data(XLEN-1, 0))
+    dataModule.io.uncache.wen := true.B
+
     XSDebug("uncache resp: data %x\n", io.dcache.resp.bits.data)
   }
 
@@ -539,29 +547,19 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
   val needCancel = Wire(Vec(LoadQueueSize, Bool()))
   for (i <- 0 until LoadQueueSize) {
     needCancel(i) := uop(i).roqIdx.needFlush(io.brqRedirect) && allocated(i) && !commited(i)
-    when(needCancel(i)) {
-      // when(io.brqRedirect.bits.isReplay){
-      //   valid(i) := false.B
-      //   writebacked(i) := false.B
-      //   listening(i) := false.B
-      //   miss(i) := false.B
-      //   pending(i) := false.B
-      // }.otherwise{
+    when (needCancel(i)) {
         allocated(i) := false.B
-      // }
     }
   }
-  when (io.brqRedirect.valid && io.brqRedirect.bits.isMisPred) {
-    enqPtrExt := enqPtrExt - PopCount(needCancel)
-  }
-
-  // assert(!io.rollback.valid)
-  when(io.rollback.valid) {
-    XSDebug("Mem rollback: pc %x roqidx %d\n", io.rollback.bits.pc, io.rollback.bits.roqIdx.asUInt)
+  // we recover the pointers in the next cycle after redirect
+  val needCancelReg = RegNext(needCancel)
+  when (lastCycleRedirect.valid) {
+    val cancelCount = PopCount(needCancelReg)
+    enqPtrExt := VecInit(enqPtrExt.map(_ - cancelCount))
   }
 
   // debug info
-  XSDebug("head %d:%d tail %d:%d\n", enqPtrExt.flag, enqPtr, deqPtrExt.flag, deqPtr)
+  XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt.flag, deqPtr)
 
   def PrintFlag(flag: Bool, name: String): Unit = {
     when(flag) {

src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala

@@ -21,14 +21,17 @@ object SqPtr extends HasXSParameter {
   }
 }
 
+class SqEnqIO extends XSBundle {
+  val canAccept = Output(Bool())
+  val needAlloc = Vec(RenameWidth, Input(Bool()))
+  val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
+  val resp = Vec(RenameWidth, Output(new SqPtr))
+}
+
 // Store Queue
 class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper {
   val io = IO(new Bundle() {
-    val enq = new Bundle() {
-      val canAccept = Output(Bool())
-      val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
-      val resp = Vec(RenameWidth, Output(new SqPtr))
-    }
+    val enq = new SqEnqIO
     val brqRedirect = Input(Valid(new Redirect))
     val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle)))
     val sbuffer = Vec(StorePipelineWidth, Decoupled(new DCacheWordReq))
@@ -51,33 +54,29 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
   val commited = Reg(Vec(StoreQueueSize, Bool())) // inst has been commited by roq
   val pending = Reg(Vec(StoreQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of roq
 
-  val enqPtrExt = RegInit(0.U.asTypeOf(new SqPtr))
-  val deqPtrExt = RegInit(0.U.asTypeOf(new SqPtr))
-  val enqPtr = enqPtrExt.value
-  val deqPtr = deqPtrExt.value
-  val sameFlag = enqPtrExt.flag === deqPtrExt.flag
-  val isEmpty = enqPtr === deqPtr && sameFlag
-  val isFull = enqPtr === deqPtr && !sameFlag
-  val allowIn = !isFull
-
-  val storeCommit = (0 until CommitWidth).map(i => io.commits.valid(i) && !io.commits.isWalk && io.commits.uop(i).ctrl.commitType === CommitType.STORE)
-  val mcommitIdx = (0 until CommitWidth).map(i => io.commits.uop(i).sqIdx.value)
+  require(StoreQueueSize > RenameWidth)
+  val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new SqPtr))))
+  val deqPtrExt = RegInit(VecInit((0 until StorePipelineWidth).map(_.U.asTypeOf(new SqPtr))))
+  val enqPtr = enqPtrExt(0).value
+  val deqPtr = deqPtrExt(0).value
 
   val tailMask = UIntToMask(deqPtr, StoreQueueSize)
   val headMask = UIntToMask(enqPtr, StoreQueueSize)
-  val enqDeqMask1 = tailMask ^ headMask
-  val enqDeqMask = Mux(sameFlag, enqDeqMask1, ~enqDeqMask1)
 
-  // Enqueue at dispatch
-  val validEntries = distanceBetween(enqPtrExt, deqPtrExt)
+  /**
+    * Enqueue at dispatch
+    *
+    * Currently, StoreQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth)
+    */
+  val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt(0))
   val firedDispatch = io.enq.req.map(_.valid)
   io.enq.canAccept := validEntries <= (StoreQueueSize - RenameWidth).U
   XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n")
   for (i <- 0 until RenameWidth) {
-    val offset = if (i == 0) 0.U else PopCount((0 until i).map(firedDispatch(_)))
-    val sqIdx = enqPtrExt + offset
+    val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i))
+    val sqIdx = enqPtrExt(offset)
     val index = sqIdx.value
-    when(io.enq.req(i).valid) {
+    when (io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid) {
       uop(index) := io.enq.req(i).bits
       allocated(index) := true.B
       datavalid(index) := false.B
@@ -86,17 +85,28 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
       pending(index) := false.B
     }
     io.enq.resp(i) := sqIdx
-
-    XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not valid when not ready\n")
   }
 
-  when(Cat(firedDispatch).orR) {
-    enqPtrExt := enqPtrExt + PopCount(firedDispatch)
-    XSInfo("dispatched %d insts to sq\n", PopCount(firedDispatch))
+  when (Cat(firedDispatch).orR && io.enq.canAccept && !io.brqRedirect.valid) {
+    val enqNumber = PopCount(firedDispatch)
+    enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber))
+    XSInfo("dispatched %d insts to sq\n", enqNumber)
   }
 
-  // writeback store
-  (0 until StorePipelineWidth).map(i => {
+  /**
+    * Writeback store from store units
+    *
+    * Most store instructions writeback to regfile in the previous cycle.
+    * However,
+    *   (1) For an mmio instruction with exceptions, we need to mark it as datavalid
+    * (in this way it will trigger an exception when it reaches ROB's head)
+    * instead of pending to avoid sending them to lower level.
+    *   (2) For an mmio instruction without exceptions, we mark it as pending.
+    * When the instruction reaches ROB's head, StoreQueue sends it to uncache channel.
+    * Upon receiving the response, StoreQueue writes back the instruction 
+    * through arbiter with store units. It will later commit as normal.
+    */
+  for (i <- 0 until StorePipelineWidth) {
     dataModule.io.wb(i).wen := false.B
     when(io.storeIn(i).fire()) {
       val stWbIndex = io.storeIn(i).bits.uop.sqIdx.value
@@ -129,94 +139,16 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
         io.storeIn(i).bits.uop.cf.exceptionVec.asUInt
         )
     }
-  })
-
-  def getFirstOne(mask: Vec[Bool], startMask: UInt) = {
-    val length = mask.length
-    val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
-    val highBitsUint = Cat(highBits.reverse)
-    PriorityEncoder(Mux(highBitsUint.orR(), highBitsUint, mask.asUInt))
-  }
-
-  def getFirstOneWithFlag(mask: Vec[Bool], startMask: UInt, startFlag: Bool) = {
-    val length = mask.length
-    val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
-    val highBitsUint = Cat(highBits.reverse)
-    val changeDirection = !highBitsUint.orR()
-    val index = PriorityEncoder(Mux(!changeDirection, highBitsUint, mask.asUInt))
-    SqPtr(startFlag ^ changeDirection, index)
-  }
-
-  def selectFirstTwo(valid: Vec[Bool], startMask: UInt) = {
-    val selVec = Wire(Vec(2, UInt(log2Up(StoreQueueSize).W)))
-    val selValid = Wire(Vec(2, Bool()))
-    selVec(0) := getFirstOne(valid, startMask)
-    val firstSelMask = UIntToOH(selVec(0))
-    val secondSelVec = VecInit((0 until valid.length).map(i => valid(i) && !firstSelMask(i)))
-    selVec(1) := getFirstOne(secondSelVec, startMask)
-    selValid(0) := Cat(valid).orR
-    selValid(1) := Cat(secondSelVec).orR
-    (selValid, selVec)
-  }
-
-  def selectFirstTwoRoughly(valid: Vec[Bool]) = {
-    // TODO: do not select according to seq, just select 2 valid bit randomly
-    val firstSelVec = valid
-    val notFirstVec = Wire(Vec(valid.length, Bool()))
-    (0 until valid.length).map(i =>
-      notFirstVec(i) := (if(i != 0) { valid(i) || !notFirstVec(i) } else { false.B })
-    )
-    val secondSelVec = VecInit((0 until valid.length).map(i => valid(i) && !notFirstVec(i)))
-
-    val selVec = Wire(Vec(2, UInt(log2Up(valid.length).W)))
-    val selValid = Wire(Vec(2, Bool()))
-    selVec(0) := PriorityEncoder(firstSelVec)
-    selVec(1) := PriorityEncoder(secondSelVec)
-    selValid(0) := Cat(firstSelVec).orR
-    selValid(1) := Cat(secondSelVec).orR
-    (selValid, selVec)
   }
 
-  // writeback finished mmio store
-  io.mmioStout.bits.uop := uop(deqPtr)
-  io.mmioStout.bits.uop.sqIdx := deqPtrExt
-  io.mmioStout.bits.uop.cf.exceptionVec := dataModule.io.rdata(deqPtr).exception.asBools
-  io.mmioStout.bits.data := dataModule.io.rdata(deqPtr).data
-  io.mmioStout.bits.redirectValid := false.B
-  io.mmioStout.bits.redirect := DontCare
-  io.mmioStout.bits.brUpdate := DontCare
-  io.mmioStout.bits.debug.isMMIO := true.B
-  io.mmioStout.bits.fflags := DontCare
-  io.mmioStout.valid := allocated(deqPtr) && datavalid(deqPtr) && !writebacked(deqPtr) // finished mmio store
-  when(io.mmioStout.fire()) {
-    writebacked(deqPtr) := true.B
-    allocated(deqPtr) := false.B // potential opt: move deqPtr immediately
-  }
-
-  // remove retired insts from sq, add retired store to sbuffer
-
-  // move tailPtr
-  // TailPtr slow recovery: recycle bubbles in store queue
-  // allocatedMask: dequeuePtr can go to the next 1-bit
-  val allocatedMask = VecInit((0 until StoreQueueSize).map(i => allocated(i) || !enqDeqMask(i)))
-  // find the first one from deqPtr (deqPtr)
-  val nextTail1 = getFirstOneWithFlag(allocatedMask, tailMask, deqPtrExt.flag)
-  val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, enqPtrExt)
-  deqPtrExt := nextTail
-
-  // TailPtr fast recovery
-  // val tailRecycle = VecInit(List(
-  //   io.uncache.resp.fire() || io.sbuffer(0).fire(),
-  //   io.sbuffer(1).fire()
-  // ))
-
-  when(io.sbuffer(0).fire()){
-    deqPtrExt := deqPtrExt + Mux(io.sbuffer(1).fire(), 2.U, 1.U)
-  }
-
-  // load forward query
+  /**
+    * load forward query
+    *
+    * Check store queue for instructions that is older than the load.
+    * The response will be valid at the next cycle after req.
+    */
   // check over all lq entries and forward data from the first matched store
-  (0 until LoadPipelineWidth).map(i => {
+  for (i <- 0 until LoadPipelineWidth) {
     io.forward(i).forwardMask := 0.U(8.W).asBools
     io.forward(i).forwardData := DontCare
 
@@ -226,8 +158,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     // Forward1: Mux(same_flag, range(tail, sqIdx), range(tail, LoadQueueSize))
     // Forward2: Mux(same_flag, 0.U,                   range(0, sqIdx)    )
     // i.e. forward1 is the target entries with the same flag bits and forward2 otherwise
-
-    val differentFlag = deqPtrExt.flag =/= io.forward(i).sqIdx.flag
+    val differentFlag = deqPtrExt(0).flag =/= io.forward(i).sqIdx.flag
     val forwardMask = UIntToMask(io.forward(i).sqIdx.value, StoreQueueSize)
     val storeWritebackedVec = WireInit(VecInit(Seq.fill(StoreQueueSize)(false.B)))
     for (j <- 0 until StoreQueueSize) {
@@ -236,7 +167,9 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     val needForward1 = Mux(differentFlag, ~tailMask, tailMask ^ forwardMask) & storeWritebackedVec.asUInt
     val needForward2 = Mux(differentFlag, forwardMask, 0.U(StoreQueueSize.W)) & storeWritebackedVec.asUInt
 
-    XSDebug("" + i + " f1 %b f2 %b sqIdx %d pa %x\n", needForward1, needForward2, io.forward(i).sqIdx.asUInt, io.forward(i).paddr)
+    XSDebug(p"$i f1 ${Binary(needForward1)} f2 ${Binary(needForward2)} " +
+      p"sqIdx ${io.forward(i).sqIdx} pa ${Hexadecimal(io.forward(i).paddr)}\n"
+    )
 
     // do real fwd query
     dataModule.io.forwardQuery(
@@ -248,40 +181,19 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
 
     io.forward(i).forwardMask := dataModule.io.forward(i).forwardMask
     io.forward(i).forwardData := dataModule.io.forward(i).forwardData
-  })
-
-  // When store commited, mark it as commited (will not be influenced by redirect),
-  (0 until CommitWidth).map(i => {
-    when(storeCommit(i)) {
-      commited(mcommitIdx(i)) := true.B
-      XSDebug("store commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc)
-    }
-  })
-
-  (0 until 2).map(i => {
-    val ptr = (deqPtrExt + i.U).value
-    val mmio = dataModule.io.rdata(ptr).mmio
-    io.sbuffer(i).valid := allocated(ptr) && commited(ptr) && !mmio
-    io.sbuffer(i).bits.cmd  := MemoryOpConstants.M_XWR
-    io.sbuffer(i).bits.addr := dataModule.io.rdata(ptr).paddr
-    io.sbuffer(i).bits.data := dataModule.io.rdata(ptr).data
-    io.sbuffer(i).bits.mask := dataModule.io.rdata(ptr).mask
-    io.sbuffer(i).bits.meta          := DontCare
-    io.sbuffer(i).bits.meta.tlb_miss := false.B
-    io.sbuffer(i).bits.meta.uop      := DontCare
-    io.sbuffer(i).bits.meta.mmio     := mmio
-    io.sbuffer(i).bits.meta.mask     := dataModule.io.rdata(ptr).mask
-
-    when(io.sbuffer(i).fire()) {
-      allocated(ptr) := false.B
-      XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr)
-    }
-  })
-
-  // Memory mapped IO / other uncached operations
+  }
 
-  // setup misc mem access req
-  // mask / paddr / data can be get from sq.data
+  /**
+    * Memory mapped IO / other uncached operations
+    *
+    * States:
+    * (1) writeback from store units: mark as pending
+    * (2) when they reach ROB's head, they can be sent to uncache channel
+    * (3) response from uncache channel: mark as datavalid
+    * (4) writeback to ROB (and other units): mark as writebacked
+    * (5) ROB commits the instruction: same as normal instructions
+    */
+  //(2) when they reach ROB's head, they can be sent to uncache channel
   val commitType = io.commits.uop(0).ctrl.commitType
   io.uncache.req.valid := pending(deqPtr) && allocated(deqPtr) &&
     commitType === CommitType.STORE &&
@@ -297,30 +209,88 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
   io.uncache.req.bits.meta.vaddr    := DontCare
   io.uncache.req.bits.meta.paddr    := dataModule.io.rdata(deqPtr).paddr
   io.uncache.req.bits.meta.uop      := uop(deqPtr)
-  io.uncache.req.bits.meta.mmio     := true.B // dataModule.io.rdata(deqPtr).mmio
+  io.uncache.req.bits.meta.mmio     := true.B
   io.uncache.req.bits.meta.tlb_miss := false.B
   io.uncache.req.bits.meta.mask     := dataModule.io.rdata(deqPtr).mask
   io.uncache.req.bits.meta.replay   := false.B
 
-  io.uncache.resp.ready := true.B
-
   when(io.uncache.req.fire()){
     pending(deqPtr) := false.B
+
+    XSDebug(
+      p"uncache req: pc ${Hexadecimal(uop(deqPtr).cf.pc)} " +
+      p"addr ${Hexadecimal(io.uncache.req.bits.addr)} " +
+      p"data ${Hexadecimal(io.uncache.req.bits.data)} " +
+      p"op ${Hexadecimal(io.uncache.req.bits.cmd)} " +
+      p"mask ${Hexadecimal(io.uncache.req.bits.mask)}\n"
+    )
   }
 
-  when(io.uncache.resp.fire()){
-    datavalid(deqPtr) := true.B // will be writeback to CDB in the next cycle
-    // TODO: write back exception info
+  // (3) response from uncache channel: mark as datavalid
+  io.uncache.resp.ready := true.B
+  when (io.uncache.resp.fire()) {
+    datavalid(deqPtr) := true.B
   }
 
-  when(io.uncache.req.fire()){
-    XSDebug("uncache req: pc %x addr %x data %x op %x mask %x\n",
-      uop(deqPtr).cf.pc,
-      io.uncache.req.bits.addr,
-      io.uncache.req.bits.data,
-      io.uncache.req.bits.cmd,
-      io.uncache.req.bits.mask
-    )
+  // (4) writeback to ROB (and other units): mark as writebacked
+  io.mmioStout.valid := allocated(deqPtr) && datavalid(deqPtr) && !writebacked(deqPtr)
+  io.mmioStout.bits.uop := uop(deqPtr)
+  io.mmioStout.bits.uop.sqIdx := deqPtrExt(0)
+  io.mmioStout.bits.uop.cf.exceptionVec := dataModule.io.rdata(deqPtr).exception.asBools
+  io.mmioStout.bits.data := dataModule.io.rdata(deqPtr).data
+  io.mmioStout.bits.redirectValid := false.B
+  io.mmioStout.bits.redirect := DontCare
+  io.mmioStout.bits.brUpdate := DontCare
+  io.mmioStout.bits.debug.isMMIO := true.B
+  io.mmioStout.bits.fflags := DontCare
+  when (io.mmioStout.fire()) {
+    writebacked(deqPtr) := true.B
+    allocated(deqPtr) := false.B
+    deqPtrExt := VecInit(deqPtrExt.map(_ + 1.U))
+  }
+
+  /**
+    * ROB commits store instructions (mark them as commited)
+    *
+    * (1) When store commits, mark it as commited.
+    * (2) They will not be cancelled and can be sent to lower level.
+    */
+  for (i <- 0 until CommitWidth) {
+    val storeCommit = !io.commits.isWalk && io.commits.valid(i) && io.commits.uop(i).ctrl.commitType === CommitType.STORE
+    when (storeCommit) {
+      commited(io.commits.uop(i).sqIdx.value) := true.B
+      XSDebug("store commit %d: idx %d %x\n", i.U, io.commits.uop(i).sqIdx.value, io.commits.uop(i).cf.pc)
+    }
+  }
+
+  // Commited stores will not be cancelled and can be sent to lower level.
+  // remove retired insts from sq, add retired store to sbuffer
+  for (i <- 0 until StorePipelineWidth) {
+    val ptr = deqPtrExt(i).value
+    val mmio = dataModule.io.rdata(ptr).mmio
+    io.sbuffer(i).valid := allocated(ptr) && commited(ptr) && !mmio
+    io.sbuffer(i).bits.cmd  := MemoryOpConstants.M_XWR
+    io.sbuffer(i).bits.addr := dataModule.io.rdata(ptr).paddr
+    io.sbuffer(i).bits.data := dataModule.io.rdata(ptr).data
+    io.sbuffer(i).bits.mask := dataModule.io.rdata(ptr).mask
+    io.sbuffer(i).bits.meta          := DontCare
+    io.sbuffer(i).bits.meta.tlb_miss := false.B
+    io.sbuffer(i).bits.meta.uop      := DontCare
+    io.sbuffer(i).bits.meta.mmio     := mmio
+    io.sbuffer(i).bits.meta.mask     := dataModule.io.rdata(ptr).mask
+
+    when (io.sbuffer(i).fire()) {
+      allocated(ptr) := false.B
+      XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr)
+    }
+  }
+  // note that sbuffer will not accept req(1) if req(0) is not accepted.
+  when (Cat(io.sbuffer.map(_.fire())).orR) {
+    val stepForward = Mux(io.sbuffer(1).fire(), 2.U, 1.U)
+    deqPtrExt := VecInit(deqPtrExt.map(_ + stepForward))
+    when (io.sbuffer(1).fire()) {
+      assert(io.sbuffer(0).fire())
+    }
   }
 
   // Read vaddr for mem exception
@@ -331,22 +301,19 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
   val needCancel = Wire(Vec(StoreQueueSize, Bool()))
   for (i <- 0 until StoreQueueSize) {
     needCancel(i) := uop(i).roqIdx.needFlush(io.brqRedirect) && allocated(i) && !commited(i)
-    when(needCancel(i)) {
-      // when(io.brqRedirect.bits.isReplay){
-      //   datavalid(i) := false.B
-      //   writebacked(i) := false.B
-      //   pending(i) := false.B
-      // }.otherwise{
+    when (needCancel(i)) {
         allocated(i) := false.B
-      // }
     }
   }
-  when (io.brqRedirect.valid && io.brqRedirect.bits.isMisPred) {
-    enqPtrExt := enqPtrExt - PopCount(needCancel)
+  // we recover the pointers in the next cycle after redirect
+  val lastCycleRedirectValid = RegNext(io.brqRedirect.valid)
+  val needCancelCount = PopCount(RegNext(needCancel))
+  when (lastCycleRedirectValid) {
+    enqPtrExt := VecInit(enqPtrExt.map(_ - needCancelCount))
   }
 
   // debug info
-  XSDebug("head %d:%d tail %d:%d\n", enqPtrExt.flag, enqPtr, deqPtrExt.flag, deqPtr)
+  XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt(0).flag, deqPtr)
 
   def PrintFlag(flag: Bool, name: String): Unit = {
     when(flag) {

src/main/scala/xiangshan/mem/sbuffer/NewSbuffer.scala

@@ -189,6 +189,9 @@ class NewSbuffer extends XSModule with HasSbufferCst {
   val updatedSbuffer = io.in.zipWithIndex.foldLeft[Seq[SbufferEntry]](initialSbuffer)(enqSbuffer)
   val updatedState = updatedSbuffer.map(_._1)
   val updatedSbufferLine = VecInit(updatedSbuffer.map(_._2))
+  when (!io.in(0).ready) {
+    io.in(1).ready := false.B
+  }
 
   for(i <- 0 until StoreBufferSize){
     buffer.write(i.U, updatedSbufferLine(i))