rs,status: simplify logic to optimize timing (#1020)

This commit simplifies status logic in reservations stations. Module
StatusArray is mostly rewritten.

The following optimizations are applied:

* Wakeup now has higher priority than enqueue. This reduces the length
of the critical path of ALU back-to-back wakeup.

* Don't compare fpWen/rfWen if the reservation station does not have
float/int operands.

* Ignore status.valid or redirect for srcState update. For data capture,
these are necessary and not changed.

* Remove blocked and scheduled conditions in issue logic when the
reservation station does not have loadWait bit and feedback.

src/main/scala/xiangshan/Bundle.scala

@@ -215,7 +215,7 @@ class MicroOp(implicit p: Parameters) extends CfCtrl {
     }
   }
   def srcIsReady: Vec[Bool] = {
-    VecInit(ctrl.srcType.zip(srcState).map{ case (t, s) => SrcType.isPcImm(t) || s === SrcState.rdy })
+    VecInit(ctrl.srcType.zip(srcState).map{ case (t, s) => SrcType.isPcOrImm(t) || s === SrcState.rdy })
   }
   def doWriteIntRf: Bool = ctrl.rfWen && ctrl.ldest =/= 0.U
   def doWriteFpRf: Bool = ctrl.fpWen

src/main/scala/xiangshan/backend/issue/ReservationStation.scala

@@ -54,6 +54,7 @@ case class RSParams
   def allWakeup: Int = numFastWakeup + numWakeup
   def indexWidth: Int = log2Up(numEntries)
   def oldestFirst: Boolean = exuCfg.get != AluExeUnitCfg
+  def needScheduledBit: Boolean = hasFeedback || delayedRf
 
   override def toString: String = {
     s"type ${exuCfg.get.name}, size $numEntries, enq $numEnq, deq $numDeq, numSrc $numSrc, fast $numFastWakeup, wakeup $numWakeup"
@@ -274,16 +275,17 @@ class ReservationStation(params: RSParams)(implicit p: Parameters) extends XSMod
     */
   // enqueue from dispatch
   select.io.validVec := statusArray.io.isValid
-  val doEnqueue = Wire(Vec(params.numEnq, Bool()))
-  val needFpSource = Wire(Vec(params.numEnq, Bool()))
+  // agreement with dispatch: don't enqueue when io.redirect.valid
+  val doEnqueue = io.fromDispatch.map(_.fire && !io.redirect.valid && !io.flush)
+  val enqShouldNotFlushed = io.fromDispatch.map(d => d.fire && !d.bits.roqIdx.needFlush(io.redirect, io.flush))
+  XSPerfAccumulate("wrong_stall", Mux(io.redirect.valid, PopCount(enqShouldNotFlushed), 0.U))
+  val needFpSource = io.fromDispatch.map(_.bits.needRfRPort(1, 1, false))
   for (i <- 0 until params.numEnq) {
     io.fromDispatch(i).ready := select.io.allocate(i).valid
-    // agreement with dispatch: don't enqueue when io.redirect.valid
-    doEnqueue(i) := io.fromDispatch(i).fire() && !io.redirect.valid && !io.flush
-    statusArray.io.update(i).enable := doEnqueue(i)
+    // for better timing, we update statusArray no matter there's a flush or not
+    statusArray.io.update(i).enable := io.fromDispatch(i).fire()
     statusArray.io.update(i).addr := select.io.allocate(i).bits
     statusArray.io.update(i).data.valid := true.B
-    needFpSource(i) := io.fromDispatch(i).bits.needRfRPort(1, 1, false)
     statusArray.io.update(i).data.scheduled := (if (params.delayedRf) needFpSource(i) else false.B)
     statusArray.io.update(i).data.blocked := (if (params.checkWaitBit) io.fromDispatch(i).bits.cf.loadWaitBit else false.B)
     statusArray.io.update(i).data.credit := (if (params.delayedRf) Mux(needFpSource(i), 2.U, 0.U) else 0.U)
@@ -293,11 +295,12 @@ class ReservationStation(params: RSParams)(implicit p: Parameters) extends XSMod
     statusArray.io.update(i).data.roqIdx := io.fromDispatch(i).bits.roqIdx
     statusArray.io.update(i).data.sqIdx := io.fromDispatch(i).bits.sqIdx
     statusArray.io.update(i).data.isFirstIssue := true.B
+    // for better power, we don't write payload array when there's a redirect
     payloadArray.io.write(i).enable := doEnqueue(i)
     payloadArray.io.write(i).addr := select.io.allocate(i).bits
     payloadArray.io.write(i).data := io.fromDispatch(i).bits
   }
-  val enqVec = VecInit(doEnqueue.zip(select.io.allocate.map(_.bits)).map{ case (d, b) => Mux(d, b, 0.U) })
+
   // when config.checkWaitBit is set, we need to block issue until the corresponding store issues
   if (params.checkWaitBit) {
     statusArray.io.stIssuePtr := io.checkwait.get.stIssuePtr
@@ -320,6 +323,7 @@ class ReservationStation(params: RSParams)(implicit p: Parameters) extends XSMod
   // Option 1: normal selection (do not care about the age)
   select.io.request := statusArray.io.canIssue
   // Option 2: select the oldest
+  val enqVec = VecInit(doEnqueue.zip(select.io.allocate.map(_.bits)).map{ case (d, b) => Mux(d, b, 0.U) })
   val oldestSel = AgeDetector(params.numEntries, enqVec, statusArray.io.flushed, statusArray.io.canIssue)
 
   val issueVec = Wire(Vec(params.numDeq, Valid(UInt(params.numEntries.W))))
@@ -428,7 +432,7 @@ class ReservationStation(params: RSParams)(implicit p: Parameters) extends XSMod
   dataArray.io.multiWrite.zipWithIndex.foreach { case (w, i) =>
     w.enable := broadcastValid(i)
     for (j <- 0 until params.numSrc) {
-      w.addr(j) := VecInit(slowWakeupMatchVec.map(_ (j)(i))).asUInt
+      w.addr(j) := VecInit(slowWakeupMatchVec.map(_(j)(i))).asUInt
     }
     w.data := broadcastValue(i)
   }

src/main/scala/xiangshan/backend/issue/StatusArray.scala

@@ -53,6 +53,12 @@ class StatusEntry(params: RSParams)(implicit p: Parameters) extends XSBundle {
   // misc
   val isFirstIssue = Bool()
 
+  def canIssue: Bool = {
+    val scheduledCond = if (params.needScheduledBit) !scheduled else true.B
+    val blockedCond = if (params.checkWaitBit) !blocked else true.B
+    srcState.asUInt.andR && scheduledCond && blockedCond
+  }
+
   override def cloneType: StatusEntry.this.type =
     new StatusEntry(params).asInstanceOf[this.type]
   override def toPrintable: Printable = {
@@ -92,88 +98,124 @@ class StatusArray(params: RSParams)(implicit p: Parameters) extends XSModule
   }
 
   // instruction is ready for issue
-  val readyVec = VecInit(statusArray.map(s => s.srcState.asUInt.andR && !s.scheduled && !s.blocked))
-  val readyVecNext = VecInit(statusArrayNext.map(s => s.srcState.asUInt.andR && !s.scheduled && !s.blocked))
+  val readyVec = VecInit(statusArray.map(_.canIssue))
+  val readyVecNext = VecInit(statusArrayNext.map(_.canIssue))
 
   // update srcState when enqueue, wakeup
-  def wakeupMatch(psrc: UInt, srcType: UInt) = {
-    val matchVec = VecInit(io.wakeup.map(w =>
-      w.valid && w.bits.pdest === psrc && (SrcType.isReg(srcType) && w.bits.ctrl.rfWen && psrc =/= 0.U || SrcType.isFp(srcType) && w.bits.ctrl.fpWen)
-    ))
-    XSError(PopCount(matchVec) > 1.U, p"matchVec ${Binary(matchVec.asUInt)} should be one-hot\n")
-    matchVec.asUInt
+  // For better timing, we use different conditions for data write and srcState update
+  def wakeupMatch(srcInfo: (UInt, UInt)): (Bool, UInt) = {
+    val (psrc, srcType) = srcInfo
+    val (stateMatchVec, dataMatchVec) = io.wakeup.map(w => {
+      val pdestMatch = w.valid && w.bits.pdest === psrc
+      val rfStateMatch = if (params.exuCfg.get.readIntRf) w.bits.ctrl.rfWen else false.B
+      val rfDataMatch = if (params.exuCfg.get.readIntRf) w.bits.ctrl.rfWen && psrc =/= 0.U else false.B
+      val fpMatch = if (params.exuCfg.get.readFpRf) w.bits.ctrl.fpWen else false.B
+      // For state condition: only pdest is used for matching.
+      // If the exu needs both int and fp sources, we need to check which type of source it is.
+      // Otherwise, no need to check the source type (does not matter if it is imm).
+      val bothIntFp = params.exuCfg.get.readIntRf && params.exuCfg.get.readFpRf
+      val bothStateMatch = (rfStateMatch && !SrcType.regIsFp(srcType)) || (fpMatch && SrcType.regIsFp(srcType))
+      val stateCond = pdestMatch && (if (bothIntFp) bothStateMatch else rfStateMatch || fpMatch)
+      // For data condition: types are matched and int pdest is not $zero.
+      val bothDataMatch = (rfDataMatch && SrcType.isReg(srcType)) || (fpMatch && SrcType.isFp(srcType))
+      val dataCond = pdestMatch && bothDataMatch
+      (stateCond, dataCond)
+    }).unzip
+    val stateMatch = VecInit(stateMatchVec).asUInt.orR
+    val dataMatch = VecInit(dataMatchVec).asUInt
+    XSError(PopCount(dataMatchVec) > 1.U, p"matchVec ${Binary(dataMatch)} should be one-hot\n")
+    (stateMatch, dataMatch)
   }
+
   def deqRespSel(i: Int) : (Bool, Bool, UInt) = {
     val mask = VecInit(io.deqResp.map(resp => resp.valid && resp.bits.rsMask(i)))
     XSError(PopCount(mask) > 1.U, p"feedbackVec ${Binary(mask.asUInt)} should be one-hot\n")
+    val deqValid = mask.asUInt.orR
+    XSError(deqValid && !statusArray(i).valid, p"should not deq an invalid entry $i\n")
+    if (params.hasFeedback) {
+      XSError(deqValid && !statusArray(i).scheduled, p"should not deq an un-scheduled entry $i\n")
+    }
     val successVec = io.deqResp.map(_.bits.success)
     val respTypeVec = io.deqResp.map(_.bits.resptype)
     (mask.asUInt.orR, Mux1H(mask, successVec), Mux1H(mask, respTypeVec))
   }
+
+  def enqUpdate(i: Int): (Bool, StatusEntry) = {
+    val updateVec = VecInit(io.update.map(u => u.enable && u.addr(i)))
+    val updateStatus = Mux1H(updateVec, io.update.map(_.data))
+    XSError(PopCount(updateVec) > 1.U, "should not update the same entry\n")
+    (updateVec.asUInt.orR, updateStatus)
+  }
+
   val flushedVec = Wire(Vec(params.numEntries, Bool()))
+
+  val (updateValid, updateVal) = statusArray.indices.map(enqUpdate).unzip
+  val deqResp = statusArray.indices.map(deqRespSel)
+
   for (((status, statusNext), i) <- statusArray.zip(statusArrayNext).zipWithIndex) {
-    val selVec = VecInit(io.update.map(u => u.enable && u.addr(i)))
-    XSError(PopCount(selVec) > 1.U, "should not update the same entry\n")
-    val updateEn = selVec.asUInt.orR
-
-    when (updateEn) {
-      val updateStatus = Mux1H(selVec, io.update.map(_.data))
-      val wakeupEnVec = VecInit(updateStatus.psrc.zip(updateStatus.srcType).map{ case (p, t) => wakeupMatch(p, t) })
-      val wakeupEn = wakeupEnVec.map(_.orR)
-      io.wakeupMatch(i) := wakeupEnVec
-      statusNext.valid := true.B
-      flushedVec(i) := DontCare
-      statusNext.srcState := VecInit(updateStatus.srcState.zip(wakeupEn).map {
-        case (update, wakeup) => update || wakeup
-      })
-      statusNext.scheduled := updateStatus.scheduled
-      statusNext.blocked := false.B
-      statusNext.credit := updateStatus.credit
-      statusNext.psrc := updateStatus.psrc
-      statusNext.srcType := updateStatus.srcType
-      statusNext.roqIdx := updateStatus.roqIdx
-      statusNext.sqIdx := updateStatus.sqIdx
-      statusNext.isFirstIssue := true.B
-      if (params.checkWaitBit) {
-        statusNext.blocked := updateStatus.blocked && isAfter(updateStatus.sqIdx, io.stIssuePtr)
-      }
-      XSError(status.valid, p"should not update a valid entry $i\n")
-    }.otherwise {
-      val hasIssued = VecInit(io.issueGranted.map(iss => iss.valid && iss.bits(i))).asUInt.orR
-      val (deqResp, deqGrant, deqRespType) = deqRespSel(i)
-      XSError(deqResp && !status.valid, p"should not deq an invalid entry $i\n")
-      if (params.hasFeedback) {
-        XSError(deqResp && !status.scheduled, p"should not deq an un-scheduled entry $i\n")
-      }
-      val wakeupEnVec = VecInit(status.psrc.zip(status.srcType).map{ case (p, t) => wakeupMatch(p, t) })
-      val wakeupEn = wakeupEnVec.map(_.orR)
-      io.wakeupMatch(i) := wakeupEnVec
-      statusNext.valid := Mux(deqResp && deqGrant, false.B, status.valid && !status.roqIdx.needFlush(io.redirect, io.flush))
-      flushedVec(i) := (deqResp && deqGrant) || status.roqIdx.needFlush(io.redirect, io.flush)
-      // (1) when deq is not granted, unset its scheduled bit; (2) set scheduled if issued
-      statusNext.scheduled := Mux(deqResp && !deqGrant || status.credit === 1.U, false.B, status.scheduled || hasIssued)
-      XSError(hasIssued && !status.valid, p"should not issue an invalid entry $i\n")
-      statusNext.blocked := false.B
-      if (params.checkWaitBit) {
-        statusNext.blocked := status.blocked && isAfter(status.sqIdx, io.stIssuePtr)
-        when (deqResp && !deqGrant && deqRespType === RSFeedbackType.dataInvalid) {
-          statusNext.blocked := true.B
-          XSError(status.valid && !isAfter(status.sqIdx, RegNext(RegNext(io.stIssuePtr))),
-            "Previous store instructions are all issued. Should not trigger dataInvalid.\n")
-        }
-      }
-      statusNext.credit := Mux(status.credit > 0.U, status.credit - 1.U, status.credit)
-      XSError(status.valid && status.credit > 0.U && !status.scheduled,
-        p"instructions $i with credit ${status.credit} must not be scheduled\n")
-      statusNext.srcState := VecInit(status.srcState.zip(wakeupEn).map {
-        case (current, wakeup) => current || wakeup
-      })
-      // when the entry is not granted to leave the RS, set isFirstIssue to false.B
-      when (deqResp && !deqGrant) {
-        statusNext.isFirstIssue := false.B
+    // valid: when the entry holds a valid instruction, mark it true.
+    // Set when (1) not (flushed or deq); AND (2) update.
+    val isFlushed = status.valid && status.roqIdx.needFlush(io.redirect, io.flush)
+    val (deqRespValid, deqRespSucc, deqRespType) = deqResp(i)
+    flushedVec(i) := isFlushed || (deqRespValid && deqRespSucc)
+    val realUpdateValid = updateValid(i) && !io.redirect.valid && !io.flush
+    statusNext.valid := !flushedVec(i) && (realUpdateValid || status.valid)
+    XSError(updateValid(i) && status.valid, p"should not update a valid entry $i\n")
+
+    // scheduled: when the entry is scheduled for issue, mark it true.
+    // Set when (1) scheduled for issue; (2) enq blocked.
+    // Reset when (1) deq is not granted (it needs to be scheduled again); (2) only one credit left.
+    val hasIssued = VecInit(io.issueGranted.map(iss => iss.valid && iss.bits(i))).asUInt.orR
+    val deqNotGranted = deqRespValid && !deqRespSucc
+    statusNext.scheduled := true.B
+    if (params.needScheduledBit) {
+      // An entry keeps in the scheduled state until its credit comes to zero or deqFailed.
+      val noCredit = status.valid && status.credit === 1.U
+      val keepScheduled = status.scheduled && !deqNotGranted && !noCredit
+      statusNext.scheduled := Mux(updateValid(i), updateVal(i).scheduled, hasIssued || keepScheduled)
+    }
+    XSError(hasIssued && !status.valid, p"should not issue an invalid entry $i\n")
+
+    // blocked: indicate whether the entry is blocked for issue until certain conditions meet.
+    statusNext.blocked := false.B
+    if (params.checkWaitBit) {
+      val blockReleased = isAfter(statusNext.sqIdx, io.stIssuePtr)
+      statusNext.blocked := Mux(updateValid(i), updateVal(i).blocked, status.blocked) && blockReleased
+      when (deqNotGranted && deqRespType === RSFeedbackType.dataInvalid) {
+        statusNext.blocked := true.B
+        XSError(status.valid && !isAfter(status.sqIdx, RegNext(RegNext(io.stIssuePtr))),
+          "Previous store instructions are all issued. Should not trigger dataInvalid.\n")
       }
     }
 
+    // credit: the number of cycles this entry needed until it can be scheduled
+    val creditStep = Mux(status.credit > 0.U, status.credit - 1.U, status.credit)
+    statusNext.credit := Mux(updateValid(i), updateVal(i).credit, creditStep)
+    XSError(status.valid && status.credit > 0.U && !status.scheduled,
+      p"instructions $i with credit ${status.credit} must not be scheduled\n")
+
+    // srcState: indicate whether the operand is ready for issue
+    val (stateWakeupEn, dataWakeupEnVec) = statusNext.psrc.zip(statusNext.srcType).map(wakeupMatch).unzip
+    io.wakeupMatch(i) := dataWakeupEnVec
+    // For best timing of srcState, we don't care whether the instruction is valid or not.
+    // We also don't care whether the instruction can really enqueue.
+    val updateSrcState = updateVal(i).srcState
+    val wakeupSrcState = stateWakeupEn
+    statusNext.srcState := VecInit(status.srcState.zip(updateSrcState).zip(wakeupSrcState).map {
+      // When the instruction enqueues, we always use the wakeup result.
+      case ((current, update), wakeup) => wakeup || Mux(updateValid(i), update, current)
+    })
+
+    // static data fields (only updated when instructions enqueue)
+    statusNext.psrc := Mux(updateValid(i), updateVal(i).psrc, status.psrc)
+    statusNext.srcType := Mux(updateValid(i), updateVal(i).srcType, status.srcType)
+    statusNext.roqIdx := Mux(updateValid(i), updateVal(i).roqIdx, status.roqIdx)
+    statusNext.sqIdx := Mux(updateValid(i), updateVal(i).sqIdx, status.sqIdx)
+
+    // isFirstIssue: indicate whether the entry has been issued before
+    // When the entry is not granted to leave the RS, set isFirstIssue to false.B
+    statusNext.isFirstIssue := Mux(deqNotGranted, false.B, updateValid(i) || status.isFirstIssue)
+
     XSDebug(status.valid, p"entry[$i]: $status\n")
   }
 
@@ -204,7 +246,5 @@ class StatusArray(params: RSParams)(implicit p: Parameters) extends XSModule
       val wakeup_j_i = io.wakeupMatch.map(_(i)(j)).zip(statusArray.map(_.valid)).map(p => p._1 && p._2)
       XSPerfAccumulate(s"wakeup_${j}_$i", PopCount(wakeup_j_i).asUInt)
     }
-    // val wakeup_j = io.wakeupMatch.map(m => PopCount(m.map(_(j)))).reduce(_ +& _)
-    // XSPerfHistogram(s"wakeup_$j", wakeup_j, true.B, 0, params.numEntries, 1)
   }
 }

src/main/scala/xiangshan/package.scala

@@ -37,8 +37,9 @@ package object xiangshan {
     def isPc(srcType: UInt) = srcType===pc
     def isImm(srcType: UInt) = srcType===imm
     def isFp(srcType: UInt) = srcType===fp
-    def isPcImm(srcType: UInt) = srcType(0)
-    def isRegFp(srcType: UInt) = !srcType(0)
+    def isPcOrImm(srcType: UInt) = srcType(0)
+    def isRegOrFp(srcType: UInt) = !srcType(1)
+    def regIsFp(srcType: UInt) = srcType(1)
 
     def apply() = UInt(2.W)
   }