backend,fu: allow early arbitration via fastUopOut (#962)

This commit adds a fastUopOut option to function units. This allows the
function units to give valid and uop one cycle before its output data is
ready. FastUopOut lets writeback arbitration happen one cycle before
data is ready and helps optimize the timing.

Since some function units are not ready for this new feature, this
commit adds a fastImplemented option to allow function units to have
fastUopOut but the data is still at the same cycle as uop. This option
will delay the data for one cycle and may cause performance degradation.
FastImplemented should be true after function units support fastUopOut.

src/main/scala/xiangshan/XSCore.scala

@@ -20,7 +20,7 @@ import chisel3._
 import chisel3.util._
 import xiangshan.backend._
 import xiangshan.backend.fu.HasExceptionNO
-import xiangshan.backend.exu.{ExuConfig, Wb}
+import xiangshan.backend.exu.{ExuConfig, WbArbiter}
 import xiangshan.frontend._
 import xiangshan.cache.mmu._
 import xiangshan.cache.L1plusCacheWrapper
@@ -69,12 +69,12 @@ abstract class XSCoreBase()(implicit p: config.Parameters) extends LazyModule
   val ptw = LazyModule(new PTWWrapper())
 
   val intConfigs = exuConfigs.filter(_.writeIntRf)
-  val intArbiter = LazyModule(new Wb(intConfigs, NRIntWritePorts, isFp = false))
+  val intArbiter = LazyModule(new WbArbiter(intConfigs, NRIntWritePorts, isFp = false))
   val intWbPorts = intArbiter.allConnections.map(c => c.map(intConfigs(_)))
   val numIntWbPorts = intWbPorts.length
 
   val fpConfigs = exuConfigs.filter(_.writeFpRf)
-  val fpArbiter = LazyModule(new Wb(fpConfigs, NRFpWritePorts, isFp = true))
+  val fpArbiter = LazyModule(new WbArbiter(fpConfigs, NRFpWritePorts, isFp = true))
   val fpWbPorts = fpArbiter.allConnections.map(c => c.map(fpConfigs(_)))
   val numFpWbPorts = fpWbPorts.length
 

src/main/scala/xiangshan/backend/FUBlock.scala

@@ -158,7 +158,10 @@ class FUBlock(configs: Seq[(ExuConfig, Int)])(implicit p: Parameters) extends XS
       }
 
       // out
-      io.writeback(i).bits.data := Mux(exu.io.out.bits.uop.ctrl.fpWen,
+      // TODO: remove this conversion after record is removed
+      val fpWen = exu.io.out.bits.uop.ctrl.fpWen
+      val dataIsFp = if (exu.config.hasFastUopOut) RegNext(fpWen) else fpWen
+      io.writeback(i).bits.data := Mux(dataIsFp,
         ieee(exu.io.out.bits.data),
         exu.io.out.bits.data
       )

src/main/scala/xiangshan/backend/exu/Exu.scala

@@ -66,6 +66,7 @@ case class ExuConfig
   val writeIntRf = fuConfigs.map(_.writeIntRf).reduce(_ || _)
   val writeFpRf = fuConfigs.map(_.writeFpRf).reduce(_ || _)
   val hasRedirect = fuConfigs.map(_.hasRedirect).reduce(_ || _)
+  val hasFastUopOut = fuConfigs.map(_.fastUopOut).reduce(_ || _)
 
   val latency: HasFuLatency = {
     val lats = fuConfigs.map(_.latency)
@@ -147,34 +148,50 @@ abstract class Exu(val config: ExuConfig)(implicit p: Parameters) extends XSModu
 
   val needArbiter = !(config.latency.latencyVal.nonEmpty && (config.latency.latencyVal.get == 0))
 
-  def writebackArb(in: Seq[DecoupledIO[FuOutput]], out: DecoupledIO[ExuOutput]): Arbiter[FuOutput] = {
+  def writebackArb(in: Seq[DecoupledIO[FuOutput]], out: DecoupledIO[ExuOutput]): Seq[Bool] = {
     if (needArbiter) {
       if(in.size == 1){
+        require(!config.hasFastUopOut)
         in.head.ready := out.ready
         out.bits.data := in.head.bits.data
         out.bits.uop := in.head.bits.uop
         out.valid := in.head.valid
-        null
       } else {
-        val arb = Module(new Arbiter(new FuOutput(in.head.bits.len), in.size))
-        arb.io.in <> in
-        arb.io.out.ready := out.ready
-        out.bits.data := arb.io.out.bits.data
-        out.bits.uop := arb.io.out.bits.uop
-        out.valid := arb.io.out.valid
-        arb
+        val arb = Module(new Arbiter(new ExuOutput, in.size))
+        in.zip(arb.io.in).foreach{ case (l, r) =>
+          l.ready := r.ready
+          r.valid := l.valid
+          r.bits := DontCare
+          r.bits.uop := l.bits.uop
+          r.bits.data := l.bits.data
+        }
+        arb.io.out <> out
       }
     } else {
+      require(!config.hasFastUopOut)
       in.foreach(_.ready := out.ready)
       val sel = Mux1H(in.map(x => x.valid -> x))
       out.bits.data := sel.bits.data
       out.bits.uop := sel.bits.uop
       out.valid := sel.valid
-      null
     }
+    in.map(_.fire)
   }
 
-  val arb = writebackArb(functionUnits.map(_.io.out), io.out)
+  val arbSel = writebackArb(functionUnits.map(_.io.out), io.out)
+
+  val arbSelReg = arbSel.map(RegNext(_))
+  val dataRegVec = functionUnits.map(_.io.out.bits.data).zip(config.fuConfigs).map{ case (i, cfg) =>
+    if (config.hasFastUopOut && (!cfg.fastUopOut || !cfg.fastImplemented)) {
+      println(s"WARNING: fast not implemented!! ${cfg.name} will be delayed for one cycle.")
+    }
+    (if (cfg.fastUopOut && cfg.fastImplemented) i else RegNext(i))
+  }
+  val dataReg = Mux1H(arbSelReg, dataRegVec)
+
+  if (config.hasFastUopOut) {
+    io.out.bits.data := dataReg
+  }
 
   val readIntFu = config.fuConfigs
     .zip(functionUnits.zip(fuSel))

src/main/scala/xiangshan/backend/exu/FmiscExeUnit.scala

@@ -39,17 +39,14 @@ class FmiscExeUnit(implicit p: Parameters) extends ExeUnit(FmiscExeUnitCfg) {
     module.asInstanceOf[FPUSubModule].rm := Mux(instr_rm =/= 7.U, instr_rm, frm.get)
   }
 
-  io.out.bits.fflags := MuxCase(
-    0.U,
-    fus.map(x => x.io.out.fire() -> x.fflags)
-  )
-  val fpOutCtrl = io.out.bits.uop.ctrl.fpu
-  io.out.bits.data := Mux(!io.out.bits.uop.ctrl.fpWen,
-    arb.io.out.bits.data,
-    Mux(fpOutCtrl.typeTagOut === S,
-      box(arb.io.out.bits.data, FType.S),
-      sanitizeNaN(arb.io.out.bits.data, FType.D)
+  require(config.hasFastUopOut)
+  io.out.bits.fflags := Mux1H(arbSelReg, fus.map(x => x.fflags))
+  val arbUop = RegNext(io.out.bits.uop)
+  io.out.bits.data := Mux(!arbUop.ctrl.fpWen,
+    dataReg,
+    Mux(arbUop.ctrl.fpu.typeTagOut === S,
+      box(dataReg, FType.S),
+      sanitizeNaN(dataReg, FType.D)
     )
   )
-  // io.out.bits.data := box(arb.io.out.bits.data, fpOutCtrl.typeTagOut)
 }

src/main/scala/xiangshan/backend/exu/Wb.scala → src/main/scala/xiangshan/backend/exu/WbArbiter.scala

@@ -62,7 +62,7 @@ class ExuWbArbiter(n: Int)(implicit p: Parameters) extends XSModule {
   assert(ctrl_arb.io.out.valid === data_arb.io.out.valid)
 }
 
-class Wb(cfgs: Seq[ExuConfig], numOut: Int, isFp: Boolean)(implicit p: Parameters) extends LazyModule {
+class WbArbiter(cfgs: Seq[ExuConfig], numOut: Int, isFp: Boolean)(implicit p: Parameters) extends LazyModule {
   val priorities = cfgs.map(c => if(isFp) c.wbFpPriority else c.wbIntPriority)
 
   // NOTE:
@@ -99,10 +99,18 @@ class Wb(cfgs: Seq[ExuConfig], numOut: Int, isFp: Boolean)(implicit p: Parameter
   val otherConnections = splitN(otherPorts, sharedPorts.length)
   val sharedConnections = sharedPorts.zip(otherConnections).map{ case (s, o) => s +: o }
   val allConnections: Seq[Seq[Int]] = exclusivePorts.map(Seq(_)) ++ sharedConnections
+  val hasFastUopOutVec = allConnections.map(_.map(cfgs(_).hasFastUopOut))
+  val hasFastUopOut: Seq[Boolean] = hasFastUopOutVec.map(_.reduce(_ || _))
+  hasFastUopOutVec.zip(hasFastUopOut).foreach{ case (vec, fast) =>
+    if (fast && vec.contains(false)) {
+      println("Warning: some exu does not have fastUopOut. It has extra one-cycle latency.")
+    }
+  }
 
   val sb = new StringBuffer(s"\n${if(isFp) "fp" else "int"} wb arbiter:\n")
   for ((port, i) <- exclusivePorts.zipWithIndex) {
-    sb.append(s"[ ${cfgs(port).name} ] -> out #$i\n")
+    val hasFastUopOutS = if (hasFastUopOut(i)) s" (hasFastUopOut)" else ""
+    sb.append(s"[ ${cfgs(port).name} ] -> out$hasFastUopOutS #$i\n")
   }
   for ((port, i) <- sharedPorts.zipWithIndex) {
     sb.append(s"[ ${cfgs(port).name} ")
@@ -110,14 +118,15 @@ class Wb(cfgs: Seq[ExuConfig], numOut: Int, isFp: Boolean)(implicit p: Parameter
     for (req <- otherConnections(i)) {
       sb.append(s"${cfgs(req).name} ")
     }
-    sb.append(s"] -> ${if(useArb) "arb ->" else ""} out #${exclusivePorts.size + i}\n")
+    val hasFastUopOutS = if (hasFastUopOut(i + exclusivePorts.length)) s" (hasFastUopOut)" else ""
+    sb.append(s"] -> ${if(useArb) "arb ->" else ""} out$hasFastUopOutS #${exclusivePorts.size + i}\n")
   }
   println(sb)
 
-  lazy val module = new WbImp(this)
+  lazy val module = new WbArbiterImp(this)
 }
 
-class WbImp(outer: Wb)(implicit p: Parameters) extends LazyModuleImp(outer) {
+class WbArbiterImp(outer: WbArbiter)(implicit p: Parameters) extends LazyModuleImp(outer) {
 
   val io = IO(new Bundle() {
     val in = Vec(outer.numInPorts, Flipped(DecoupledIO(new ExuOutput)))
@@ -128,23 +137,35 @@ class WbImp(outer: Wb)(implicit p: Parameters) extends LazyModuleImp(outer) {
   val sharedIn = outer.sharedPorts.map(io.in(_))
 
   // exclusive ports are connected directly
-  io.out.take(exclusiveIn.size).zip(exclusiveIn).foreach{
-    case (o, i) =>
-      val arb = Module(new ExuWbArbiter(1))
-      arb.io.in.head <> i
-      o.bits := arb.io.out.bits
-      o.valid := arb.io.out.valid
-      arb.io.out.ready := true.B
+  io.out.take(exclusiveIn.size).zip(exclusiveIn).zipWithIndex.foreach{
+    case ((out, in), i) =>
+      val hasFastUopOut = outer.hasFastUopOut(i)
+      out.valid := (if (hasFastUopOut) RegNext(in.valid) else in.valid)
+      out.bits := in.bits
+      in.ready := true.B
   }
 
   // shared ports are connected with an arbiter
   for (i <- sharedIn.indices) {
     val out = io.out(exclusiveIn.size + i)
     val shared = outer.sharedConnections(i).map(io.in(_))
+    val hasFastUopOut = outer.hasFastUopOut(i + exclusiveIn.length)
     val arb = Module(new ExuWbArbiter(shared.size))
     arb.io.in <> shared
     out.valid := arb.io.out.valid
     out.bits := arb.io.out.bits
+    if (hasFastUopOut) {
+      out.valid := RegNext(arb.io.out.valid)
+      // When hasFastUopOut, only uop comes at the same cycle with valid.
+      // Other bits like data, fflags come at the next cycle after valid,
+      // and they need to be selected with the fireVec.
+      val fastVec = outer.hasFastUopOutVec(i + exclusiveIn.length)
+      val dataVec = VecInit(shared.map(_.bits).zip(fastVec).map{ case (d, f) => if (f) d else RegNext(d) })
+      val sel = VecInit(arb.io.in.map(_.fire)).asUInt
+      out.bits := Mux1H(RegNext(sel), dataVec)
+      // uop comes at the same cycle with valid and only RegNext is needed.
+      out.bits.uop := RegNext(arb.io.out.bits.uop)
+    }
     arb.io.out.ready := true.B
   }
 

src/main/scala/xiangshan/backend/fu/FunctionUnit.scala

@@ -48,6 +48,8 @@ case class FuConfig
   writeFpRf: Boolean,
   hasRedirect: Boolean,
   latency: HasFuLatency = CertainLatency(0),
+  fastUopOut: Boolean = false,
+  fastImplemented: Boolean = false
 ) {
   def srcCnt: Int = math.max(numIntSrc, numFpSrc)
 }

src/main/scala/xiangshan/backend/issue/ReservationStation.scala

@@ -85,7 +85,7 @@ class ReservationStationWrapper(implicit p: Parameters) extends LazyModule with
       params.checkWaitBit = true
     }
     if (cfg.hasCertainLatency) {
-      params.fixedLatency = if (cfg == MulDivExeUnitCfg) 2 else cfg.latency.latencyVal.get
+      params.fixedLatency = if (cfg == MulDivExeUnitCfg) mulCfg.latency.latencyVal.get else cfg.latency.latencyVal.get
     }
   }
 

src/main/scala/xiangshan/package.scala

@@ -507,7 +507,9 @@ package object xiangshan {
     writeIntRf = true,
     writeFpRf = false,
     hasRedirect = false,
-    UncertainLatency()
+    latency = UncertainLatency(),
+    fastUopOut = true,
+    fastImplemented = false
   )
 
   val mulCfg = FuConfig(
@@ -520,7 +522,10 @@ package object xiangshan {
     writeIntRf = true,
     writeFpRf = false,
     hasRedirect = false,
-    CertainLatency(2)
+    // TODO: change this back to 2 when mul is ready for fastUopOut
+    latency = CertainLatency(3),
+    fastUopOut = true,
+    fastImplemented = false
   )
 
   val bmuCfg = FuConfig(
@@ -533,7 +538,9 @@ package object xiangshan {
     writeIntRf = true,
     writeFpRf = false,
     hasRedirect = false,
-    CertainLatency(1)
+    latency = CertainLatency(1),
+    fastUopOut = true,
+    fastImplemented = false
  )
 
   val fmacCfg = FuConfig(
@@ -547,21 +554,24 @@ package object xiangshan {
     name = "f2i",
     fuGen = f2iGen,
     fuSel = f2iSel,
-    FuType.fmisc, 0, 1, writeIntRf = true, writeFpRf = false, hasRedirect = false, CertainLatency(2)
+    FuType.fmisc, 0, 1, writeIntRf = true, writeFpRf = false, hasRedirect = false, CertainLatency(2),
+    fastUopOut = true, fastImplemented = false
   )
 
   val f2fCfg = FuConfig(
     name = "f2f",
     fuGen = f2fGen,
     fuSel = f2fSel,
-    FuType.fmisc, 0, 1, writeIntRf = false, writeFpRf = true, hasRedirect = false, CertainLatency(2)
+    FuType.fmisc, 0, 1, writeIntRf = false, writeFpRf = true, hasRedirect = false, CertainLatency(2),
+    fastUopOut = true, fastImplemented = false
   )
 
   val fdivSqrtCfg = FuConfig(
     name = "fdivSqrt",
     fuGen = fdivSqrtGen,
     fuSel = fdivSqrtSel,
-    FuType.fDivSqrt, 0, 2, writeIntRf = false, writeFpRf = true, hasRedirect = false, UncertainLatency()
+    FuType.fDivSqrt, 0, 2, writeIntRf = false, writeFpRf = true, hasRedirect = false, UncertainLatency(),
+    fastUopOut = true, fastImplemented = false
   )
 
   val lduCfg = FuConfig(