diff --git a/src/main/scala/utils/CircularQueuePtr.scala b/src/main/scala/utils/CircularQueuePtr.scala index dacfb714edd36d7cc61c45bcb5b08901ab38fd8f..b0921c1454dfbfa32252a8ebee75c6578b88ec9d 100644 --- a/src/main/scala/utils/CircularQueuePtr.scala +++ b/src/main/scala/utils/CircularQueuePtr.scala @@ -45,6 +45,8 @@ trait HasCircularQueuePtrHelper { } final def === (that_ptr: T): Bool = ptr.asUInt()===that_ptr.asUInt() + + final def =/= (that_ptr: T): Bool = ptr.asUInt()=/=that_ptr.asUInt() } diff --git a/src/main/scala/xiangshan/XSCore.scala b/src/main/scala/xiangshan/XSCore.scala index 84b922ac69602daf45860306890ca76e3c9e440e..39c21fe3468619c54f3d4bd39240414f7817d437 100644 --- a/src/main/scala/xiangshan/XSCore.scala +++ b/src/main/scala/xiangshan/XSCore.scala @@ -62,7 +62,6 @@ case class XSCoreParameters StoreQueueSize: Int = 48, RoqSize: Int = 192, dpParams: DispatchParameters = DispatchParameters( - DqEnqWidth = 4, IntDqSize = 24, FpDqSize = 24, LsDqSize = 24, diff --git a/src/main/scala/xiangshan/backend/CtrlBlock.scala b/src/main/scala/xiangshan/backend/CtrlBlock.scala index b1eb4f88e579c8bee017f58c2f41d7ffec1a8550..1e4d84041c8fe0ef85cf906e5e5069cbb67cd183 100644 --- a/src/main/scala/xiangshan/backend/CtrlBlock.scala +++ b/src/main/scala/xiangshan/backend/CtrlBlock.scala @@ -12,6 +12,7 @@ import xiangshan.backend.exu._ import xiangshan.backend.exu.Exu.exuConfigs import xiangshan.backend.regfile.RfReadPort import xiangshan.backend.roq.{Roq, RoqPtr, RoqCSRIO} +import xiangshan.mem.LsqEnqIO class CtrlToIntBlockIO extends XSBundle { val enqIqCtrl = Vec(exuParameters.IntExuCnt, DecoupledIO(new MicroOp)) @@ -30,11 +31,7 @@ class CtrlToFpBlockIO extends XSBundle { class CtrlToLsBlockIO extends XSBundle { val enqIqCtrl = Vec(exuParameters.LsExuCnt, DecoupledIO(new MicroOp)) val enqIqData = Vec(exuParameters.LsExuCnt, Output(new ExuInput)) - val enqLsq = new Bundle() { - val canAccept = Input(Bool()) - val req = Vec(RenameWidth, ValidIO(new MicroOp)) - val resp = Vec(RenameWidth, Input(new LSIdx)) - } + val enqLsq = Flipped(new LsqEnqIO) val redirect = ValidIO(new Redirect) } diff --git a/src/main/scala/xiangshan/backend/dispatch/Dispatch.scala b/src/main/scala/xiangshan/backend/dispatch/Dispatch.scala index 0e48173fb7de560c40b040695787cc3cd36e7869..c360aa1eac8b4b86854531750c842c970c432b8f 100644 --- a/src/main/scala/xiangshan/backend/dispatch/Dispatch.scala +++ b/src/main/scala/xiangshan/backend/dispatch/Dispatch.scala @@ -6,12 +6,12 @@ import xiangshan._ import utils._ import xiangshan.backend.regfile.RfReadPort import chisel3.ExcitingUtils._ -import xiangshan.backend.roq.RoqPtr +import xiangshan.backend.roq.{RoqPtr, RoqEnqIO} import xiangshan.backend.rename.RenameBypassInfo +import xiangshan.mem.LsqEnqIO case class DispatchParameters ( - DqEnqWidth: Int, IntDqSize: Int, FpDqSize: Int, LsDqSize: Int, @@ -30,19 +30,9 @@ class Dispatch extends XSModule { // to busytable: set pdest to busy (not ready) when they are dispatched val allocPregs = Vec(RenameWidth, Output(new ReplayPregReq)) // enq Roq - val enqRoq = new Bundle { - val canAccept = Input(Bool()) - val isEmpty = Input(Bool()) - val extraWalk = Vec(RenameWidth, Output(Bool())) - val req = Vec(RenameWidth, ValidIO(new MicroOp)) - val resp = Vec(RenameWidth, Input(new RoqPtr)) - } + val enqRoq = Flipped(new RoqEnqIO) // enq Lsq - val enqLsq = new Bundle() { - val canAccept = Input(Bool()) - val req = Vec(RenameWidth, ValidIO(new MicroOp)) - val resp = Vec(RenameWidth, Input(new LSIdx)) - } + val enqLsq = Flipped(new LsqEnqIO) // read regfile val readIntRf = Vec(NRIntReadPorts, Flipped(new RfReadPort)) val readFpRf = Vec(NRFpReadPorts, Flipped(new RfReadPort)) @@ -56,9 +46,9 @@ class Dispatch extends XSModule { }) val dispatch1 = Module(new Dispatch1) - val intDq = Module(new DispatchQueue(dpParams.IntDqSize, dpParams.DqEnqWidth, dpParams.IntDqDeqWidth)) - val fpDq = Module(new DispatchQueue(dpParams.FpDqSize, dpParams.DqEnqWidth, dpParams.FpDqDeqWidth)) - val lsDq = Module(new DispatchQueue(dpParams.LsDqSize, dpParams.DqEnqWidth, dpParams.LsDqDeqWidth)) + val intDq = Module(new DispatchQueue(dpParams.IntDqSize, RenameWidth, dpParams.IntDqDeqWidth)) + val fpDq = Module(new DispatchQueue(dpParams.FpDqSize, RenameWidth, dpParams.FpDqDeqWidth)) + val lsDq = Module(new DispatchQueue(dpParams.LsDqSize, RenameWidth, dpParams.LsDqDeqWidth)) // pipeline between rename and dispatch // accepts all at once @@ -68,15 +58,12 @@ class Dispatch extends XSModule { } // dispatch 1: accept uops from rename and dispatch them to the three dispatch queues - dispatch1.io.redirect <> io.redirect + // dispatch1.io.redirect <> io.redirect dispatch1.io.renameBypass := RegEnable(io.renameBypass, io.fromRename(0).valid && dispatch1.io.fromRename(0).ready) dispatch1.io.enqRoq <> io.enqRoq dispatch1.io.enqLsq <> io.enqLsq - dispatch1.io.toIntDqReady <> intDq.io.enqReady dispatch1.io.toIntDq <> intDq.io.enq - dispatch1.io.toFpDqReady <> fpDq.io.enqReady dispatch1.io.toFpDq <> fpDq.io.enq - dispatch1.io.toLsDqReady <> lsDq.io.enqReady dispatch1.io.toLsDq <> lsDq.io.enq dispatch1.io.allocPregs <> io.allocPregs diff --git a/src/main/scala/xiangshan/backend/dispatch/Dispatch1.scala b/src/main/scala/xiangshan/backend/dispatch/Dispatch1.scala index e1634e92e490474d3489bce8fa905d3223804699..8b011d74396c3ead10d582e8386486827bd0bf66 100644 --- a/src/main/scala/xiangshan/backend/dispatch/Dispatch1.scala +++ b/src/main/scala/xiangshan/backend/dispatch/Dispatch1.scala @@ -5,40 +5,35 @@ import chisel3.util._ import chisel3.ExcitingUtils._ import xiangshan._ import utils.{XSDebug, XSError, XSInfo} -import xiangshan.backend.roq.RoqPtr +import xiangshan.backend.roq.{RoqPtr, RoqEnqIO} import xiangshan.backend.rename.RenameBypassInfo +import xiangshan.mem.LsqEnqIO // read rob and enqueue class Dispatch1 extends XSModule { val io = IO(new Bundle() { - val redirect = Flipped(ValidIO(new Redirect)) // from rename val fromRename = Vec(RenameWidth, Flipped(DecoupledIO(new MicroOp))) val renameBypass = Input(new RenameBypassInfo) val recv = Output(Vec(RenameWidth, Bool())) // enq Roq - val enqRoq = new Bundle { + val enqRoq = Flipped(new RoqEnqIO) + // enq Lsq + val enqLsq = Flipped(new LsqEnqIO) + val allocPregs = Vec(RenameWidth, Output(new ReplayPregReq)) + // to dispatch queue + val toIntDq = new Bundle { val canAccept = Input(Bool()) - val isEmpty = Input(Bool()) - // if set, Roq needs extra walk - val extraWalk = Vec(RenameWidth, Output(Bool())) val req = Vec(RenameWidth, ValidIO(new MicroOp)) - val resp = Vec(RenameWidth, Input(new RoqPtr)) } - // enq Lsq - val enqLsq = new Bundle() { + val toFpDq = new Bundle { + val canAccept = Input(Bool()) + val req = Vec(RenameWidth, ValidIO(new MicroOp)) + } + val toLsDq = new Bundle { val canAccept = Input(Bool()) val req = Vec(RenameWidth, ValidIO(new MicroOp)) - val resp = Vec(RenameWidth, Input(new LSIdx)) } - val allocPregs = Vec(RenameWidth, Output(new ReplayPregReq)) - // to dispatch queue - val toIntDqReady = Input(Bool()) - val toIntDq = Vec(dpParams.DqEnqWidth, ValidIO(new MicroOp)) - val toFpDqReady = Input(Bool()) - val toFpDq = Vec(dpParams.DqEnqWidth, ValidIO(new MicroOp)) - val toLsDqReady = Input(Bool()) - val toLsDq = Vec(dpParams.DqEnqWidth, ValidIO(new MicroOp)) }) @@ -50,28 +45,10 @@ class Dispatch1 extends XSModule { val isFp = VecInit(io.fromRename.map(req => FuType.isFpExu (req.bits.ctrl.fuType))) val isLs = VecInit(io.fromRename.map(req => FuType.isMemExu(req.bits.ctrl.fuType))) val isStore = VecInit(io.fromRename.map(req => FuType.isStoreExu(req.bits.ctrl.fuType))) + val isAMO = VecInit(io.fromRename.map(req => req.bits.ctrl.fuType === FuType.mou)) val isBlockBackward = VecInit(io.fromRename.map(_.bits.ctrl.blockBackward)) val isNoSpecExec = VecInit(io.fromRename.map(_.bits.ctrl.noSpecExec)) - // generate index mapping - val intIndex = Module(new IndexMapping(RenameWidth, dpParams.DqEnqWidth, false)) - val fpIndex = Module(new IndexMapping(RenameWidth, dpParams.DqEnqWidth, false)) - val lsIndex = Module(new IndexMapping(RenameWidth, dpParams.DqEnqWidth, false)) - for (i <- 0 until RenameWidth) { - intIndex.io.validBits(i) := isInt(i) && io.fromRename(i).valid - fpIndex.io.validBits(i) := isFp(i) && io.fromRename(i).valid - lsIndex.io.validBits(i) := isLs(i) && io.fromRename(i).valid - } - intIndex.io.priority := DontCare - fpIndex.io.priority := DontCare - lsIndex.io.priority := DontCare - - if (!env.FPGAPlatform) { - val dispatchNotEmpty = Cat(io.fromRename.map(_.valid)).orR - ExcitingUtils.addSource(!dispatchNotEmpty, "perfCntCondDp1Empty", Perf) - } - - /** * Part 2: * Update commitType, psrc1, psrc2, psrc3, old_pdest for the uops @@ -84,7 +61,7 @@ class Dispatch1 extends XSModule { val updatedOldPdest = Wire(Vec(RenameWidth, UInt(PhyRegIdxWidth.W))) for (i <- 0 until RenameWidth) { - updatedCommitType(i) := Cat(isLs(i), isStore(i) | isFp(i)) + updatedCommitType(i) := Cat(isLs(i) && !isAMO(i), isStore(i) | isFp(i)) updatedPsrc1(i) := io.fromRename.take(i).map(_.bits.pdest) .zip(if (i == 0) Seq() else io.renameBypass.lsrc1_bypass(i-1).asBools) .foldLeft(io.fromRename(i).bits.psrc1) { @@ -122,110 +99,85 @@ class Dispatch1 extends XSModule { * acquire ROQ (all), LSQ (load/store only) and dispatch queue slots * only set valid when all of them provides enough entries */ - val redirectValid = io.redirect.valid// && !io.redirect.bits.isReplay - val allResourceReady = io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDqReady && io.toFpDqReady && io.toLsDqReady + val allResourceReady = io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept // Instructions should enter dispatch queues in order. - // When RenameWidth > DqEnqWidth, it's possible that some instructions cannot enter dispatch queue - // because previous instructions cannot enter dispatch queue. - // The reason is that although ROB and LSQ have enough empty slots, dispatch queue has limited enqueue ports. - // Thus, for i >= dpParams.DqEnqWidth, we have to check whether it's previous instructions (and the instruction itself) can enqueue. - // However, since, for instructions with indices less than dpParams.DqEnqWidth, - // they can always enter dispatch queue when ROB and LSQ are ready, we don't need to check whether they can enqueue. // thisIsBlocked: this instruction is blocked by itself (based on noSpecExec) - // thisCanOut: this instruction can enqueue (based on resource) - // nextCanOut: next instructions can out (based on blockBackward and previous instructions) + // nextCanOut: next instructions can out (based on blockBackward) // notBlockedByPrevious: previous instructions can enqueue val thisIsBlocked = VecInit((0 until RenameWidth).map(i => { // for i > 0, when Roq is empty but dispatch1 have valid instructions to enqueue, it's blocked if (i > 0) isNoSpecExec(i) && (!io.enqRoq.isEmpty || Cat(io.fromRename.take(i).map(_.valid)).orR) else isNoSpecExec(i) && !io.enqRoq.isEmpty })) - val thisCanOut = VecInit((0 until RenameWidth).map(i => { - // For i in [0, DqEnqWidth), they can always enqueue when ROB and LSQ are ready - if (i < dpParams.DqEnqWidth) true.B - else Cat(Seq(intIndex, fpIndex, lsIndex).map(_.io.reverseMapping(i).valid)).orR - })) val nextCanOut = VecInit((0 until RenameWidth).map(i => - (thisCanOut(i) && !isNoSpecExec(i) && !isBlockBackward(i)) || !io.fromRename(i).valid + (!isNoSpecExec(i) && !isBlockBackward(i)) || !io.fromRename(i).valid )) val notBlockedByPrevious = VecInit((0 until RenameWidth).map(i => if (i == 0) true.B else Cat((0 until i).map(j => nextCanOut(j))).andR )) + // for noSpecExec: (roqEmpty || !this.noSpecExec) && !previous.noSpecExec + // For blockBackward: // this instruction can actually dequeue: 3 conditions // (1) resources are ready // (2) previous instructions are ready - val thisCanActualOut = (0 until RenameWidth).map(i => allResourceReady && thisCanOut(i) && !thisIsBlocked(i) && notBlockedByPrevious(i)) + val thisCanActualOut = (0 until RenameWidth).map(i => !thisIsBlocked(i) && notBlockedByPrevious(i)) // input for ROQ and LSQ - // note that LSQ needs roqIdx + // (1) LSQ needs roqIdx; (2) DPQ needs roqIdx and lsIdx + val updateUopWithIndex = Wire(Vec(RenameWidth, new MicroOp)) for (i <- 0 until RenameWidth) { - io.enqRoq.extraWalk(i) := io.fromRename(i).valid && !thisCanActualOut(i) - io.enqRoq.req(i).valid := io.fromRename(i).valid && thisCanActualOut(i) + io.enqRoq.needAlloc(i) := io.fromRename(i).valid + io.enqRoq.req(i).valid := io.fromRename(i).valid && thisCanActualOut(i) && io.enqLsq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept io.enqRoq.req(i).bits := updatedUop(i) + XSDebug(io.enqRoq.req(i).valid, p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)} receives nroq ${io.enqRoq.resp(i)}\n") - val shouldEnqLsq = isLs(i) && io.fromRename(i).bits.ctrl.fuType =/= FuType.mou - io.enqLsq.req(i).valid := io.fromRename(i).valid && shouldEnqLsq && !redirectValid && thisCanActualOut(i) + val shouldEnqLsq = isLs(i) && !isAMO(i) + io.enqLsq.needAlloc(i) := io.fromRename(i).valid && shouldEnqLsq + io.enqLsq.req(i).valid := io.fromRename(i).valid && shouldEnqLsq && thisCanActualOut(i) && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept io.enqLsq.req(i).bits := updatedUop(i) io.enqLsq.req(i).bits.roqIdx := io.enqRoq.resp(i) - XSDebug(io.enqLsq.req(i).valid, p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)} receives lq ${io.enqLsq.resp(i).lqIdx} sq ${io.enqLsq.resp(i).sqIdx}\n") - XSDebug(io.enqRoq.req(i).valid, p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)} receives nroq ${io.enqRoq.resp(i)}\n") - } - - - /** - * Part 4: - * append ROQ and LSQ indexed to uop, and send them to dispatch queue - */ - val updateUopWithIndex = Wire(Vec(RenameWidth, new MicroOp)) - for (i <- 0 until RenameWidth) { updateUopWithIndex(i) := updatedUop(i) updateUopWithIndex(i).roqIdx := io.enqRoq.resp(i) updateUopWithIndex(i).lqIdx := io.enqLsq.resp(i).lqIdx updateUopWithIndex(i).sqIdx := io.enqLsq.resp(i).sqIdx - } - // send uops with correct indexes to dispatch queues - // Note that if one of their previous instructions cannot enqueue, they should not enter dispatch queue. - // We use notBlockedByPrevious here since mapping(i).valid implies there's a valid instruction that can enqueue, - // thus we don't need to check thisCanOut. - for (i <- 0 until dpParams.DqEnqWidth) { - io.toIntDq(i).bits := updateUopWithIndex(intIndex.io.mapping(i).bits) - io.toIntDq(i).valid := intIndex.io.mapping(i).valid && allResourceReady && - !thisIsBlocked(intIndex.io.mapping(i).bits) && notBlockedByPrevious(intIndex.io.mapping(i).bits) - - // NOTE: floating point instructions are not noSpecExec currently - // remove commit /**/ when fp instructions are possible to be noSpecExec - io.toFpDq(i).bits := updateUopWithIndex(fpIndex.io.mapping(i).bits) - io.toFpDq(i).valid := fpIndex.io.mapping(i).valid && allResourceReady && - /*!thisIsBlocked(fpIndex.io.mapping(i).bits) && */notBlockedByPrevious(fpIndex.io.mapping(i).bits) - - io.toLsDq(i).bits := updateUopWithIndex(lsIndex.io.mapping(i).bits) - io.toLsDq(i).valid := lsIndex.io.mapping(i).valid && allResourceReady && - !thisIsBlocked(lsIndex.io.mapping(i).bits) && notBlockedByPrevious(lsIndex.io.mapping(i).bits) - - XSDebug(io.toIntDq(i).valid, p"pc 0x${Hexadecimal(io.toIntDq(i).bits.cf.pc)} int index $i\n") - XSDebug(io.toFpDq(i).valid , p"pc 0x${Hexadecimal(io.toFpDq(i).bits.cf.pc )} fp index $i\n") - XSDebug(io.toLsDq(i).valid , p"pc 0x${Hexadecimal(io.toLsDq(i).bits.cf.pc )} ls index $i\n") + // send uops to dispatch queues + // Note that if one of their previous instructions cannot enqueue, they should not enter dispatch queue. + // We use notBlockedByPrevious here. + io.toIntDq.req(i).bits := updateUopWithIndex(i) + io.toIntDq.req(i).valid := io.fromRename(i).valid && isInt(i) && thisCanActualOut(i) && + io.enqLsq.canAccept && io.enqRoq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept + + io.toFpDq.req(i).bits := updateUopWithIndex(i) + io.toFpDq.req(i).valid := io.fromRename(i).valid && isFp(i) && thisCanActualOut(i) && + io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toLsDq.canAccept + + io.toLsDq.req(i).bits := updateUopWithIndex(i) + io.toLsDq.req(i).valid := io.fromRename(i).valid && isLs(i) && thisCanActualOut(i) && + io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept + + XSDebug(io.toIntDq.req(i).valid, p"pc 0x${Hexadecimal(io.toIntDq.req(i).bits.cf.pc)} int index $i\n") + XSDebug(io.toFpDq.req(i).valid , p"pc 0x${Hexadecimal(io.toFpDq.req(i).bits.cf.pc )} fp index $i\n") + XSDebug(io.toLsDq.req(i).valid , p"pc 0x${Hexadecimal(io.toLsDq.req(i).bits.cf.pc )} ls index $i\n") } /** - * Part 3: send response to rename when dispatch queue accepts the uop + * Part 4: send response to rename when dispatch queue accepts the uop */ - val readyVector = (0 until RenameWidth).map(i => !io.fromRename(i).valid || io.recv(i)) + val hasSpecialInstr = Cat((0 until RenameWidth).map(i => io.fromRename(i).valid && (isBlockBackward(i) || isNoSpecExec(i)))).orR for (i <- 0 until RenameWidth) { - io.recv(i) := thisCanActualOut(i) - io.fromRename(i).ready := Cat(readyVector).andR() + io.recv(i) := thisCanActualOut(i) && io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept + io.fromRename(i).ready := !hasSpecialInstr && io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept - XSInfo(io.recv(i), + XSInfo(io.recv(i) && io.fromRename(i).valid, p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)}, type(${isInt(i)}, ${isFp(i)}, ${isLs(i)}), " + - p"roq ${updateUopWithIndex(i).roqIdx}, lq ${updateUopWithIndex(i).lqIdx}, sq ${updateUopWithIndex(i).sqIdx}, " + - p"(${intIndex.io.reverseMapping(i).bits}, ${fpIndex.io.reverseMapping(i).bits}, ${lsIndex.io.reverseMapping(i).bits})\n" + p"roq ${updateUopWithIndex(i).roqIdx}, lq ${updateUopWithIndex(i).lqIdx}, sq ${updateUopWithIndex(i).sqIdx})\n" ) io.allocPregs(i).isInt := io.fromRename(i).valid && io.fromRename(i).bits.ctrl.rfWen && (io.fromRename(i).bits.ctrl.ldest =/= 0.U) @@ -233,6 +185,8 @@ class Dispatch1 extends XSModule { io.allocPregs(i).preg := io.fromRename(i).bits.pdest } val renameFireCnt = PopCount(io.recv) - val enqFireCnt = PopCount(io.toIntDq.map(_.valid && io.toIntDqReady)) + PopCount(io.toFpDq.map(_.valid && io.toFpDqReady)) + PopCount(io.toLsDq.map(_.valid && io.toLsDqReady)) + val enqFireCnt = PopCount(io.toIntDq.req.map(_.valid && io.toIntDq.canAccept)) + + PopCount(io.toFpDq.req.map(_.valid && io.toFpDq.canAccept)) + + PopCount(io.toLsDq.req.map(_.valid && io.toLsDq.canAccept)) XSError(enqFireCnt > renameFireCnt, "enqFireCnt should not be greater than renameFireCnt\n") } diff --git a/src/main/scala/xiangshan/backend/dispatch/DispatchQueue.scala b/src/main/scala/xiangshan/backend/dispatch/DispatchQueue.scala index 158cbbd7ddb595cd58276de68bbe354f526c57e5..acd4a9bc470be32e1f3719d0a1096ccfcfaebfe8 100644 --- a/src/main/scala/xiangshan/backend/dispatch/DispatchQueue.scala +++ b/src/main/scala/xiangshan/backend/dispatch/DispatchQueue.scala @@ -7,8 +7,10 @@ import xiangshan._ import xiangshan.backend.roq.RoqPtr class DispatchQueueIO(enqnum: Int, deqnum: Int) extends XSBundle { - val enq = Vec(enqnum, Flipped(ValidIO(new MicroOp))) - val enqReady = Output(Bool()) + val enq = new Bundle { + val canAccept = Output(Bool()) + val req = Vec(enqnum, Flipped(ValidIO(new MicroOp))) + } val deq = Vec(deqnum, DecoupledIO(new MicroOp)) val redirect = Flipped(ValidIO(new Redirect)) override def cloneType: DispatchQueueIO.this.type = @@ -27,22 +29,16 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H val stateEntries = RegInit(VecInit(Seq.fill(size)(s_invalid))) // head: first valid entry (dispatched entry) - val headPtr = RegInit(0.U.asTypeOf(new CircularQueuePtr(size))) - val headPtrMask = UIntToMask(headPtr.value, size) + val headPtr = RegInit(VecInit((0 until deqnum).map(_.U.asTypeOf(new CircularQueuePtr(size))))) + val headPtrMask = UIntToMask(headPtr(0).value, size) // tail: first invalid entry (free entry) - val tailPtr = RegInit(0.U.asTypeOf(new CircularQueuePtr(size))) - val tailPtrMask = UIntToMask(tailPtr.value, size) - - // TODO: make ptr a vector to reduce latency? - // deq: starting from head ptr - val deqIndex = (0 until deqnum).map(i => headPtr + i.U).map(_.value) - // enq: starting from tail ptr - val enqIndex = (0 until enqnum).map(i => tailPtr + i.U).map(_.value) + val tailPtr = RegInit(VecInit((0 until enqnum).map(_.U.asTypeOf(new CircularQueuePtr(size))))) + val tailPtrMask = UIntToMask(tailPtr(0).value, size) - val validEntries = distanceBetween(tailPtr, headPtr) + val validEntries = distanceBetween(tailPtr(0), headPtr(0)) val isTrueEmpty = ~Cat((0 until size).map(i => stateEntries(i) === s_valid)).orR val canEnqueue = validEntries <= (size - enqnum).U - val canActualEnqueue = canEnqueue && !(io.redirect.valid /*&& !io.redirect.bits.isReplay*/) + val canActualEnqueue = canEnqueue && !io.redirect.valid /** * Part 1: update states and uops when enqueue, dequeue, commit, redirect/replay @@ -57,25 +53,26 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H * (5) redirect (replay): from s_dispatched to s_valid (re-dispatch) */ // enqueue: from s_invalid to s_valid - io.enqReady := canEnqueue + io.enq.canAccept := canEnqueue for (i <- 0 until enqnum) { - when (io.enq(i).valid && canActualEnqueue) { - uopEntries(enqIndex(i)) := io.enq(i).bits - stateEntries(enqIndex(i)) := s_valid + when (io.enq.req(i).valid && canActualEnqueue) { + val sel = if (i == 0) 0.U else PopCount(io.enq.req.take(i).map(_.valid)) + uopEntries(tailPtr(sel).value) := io.enq.req(i).bits + stateEntries(tailPtr(sel).value) := s_valid } } // dequeue: from s_valid to s_dispatched for (i <- 0 until deqnum) { when (io.deq(i).fire() && !io.redirect.valid) { - stateEntries(deqIndex(i)) := s_invalid + stateEntries(headPtr(i).value) := s_invalid - XSError(stateEntries(deqIndex(i)) =/= s_valid, "state of the dispatch entry is not s_valid\n") + XSError(stateEntries(headPtr(i).value) =/= s_valid, "state of the dispatch entry is not s_valid\n") } } // redirect: cancel uops currently in the queue - val mispredictionValid = io.redirect.valid //&& io.redirect.bits.isMisPred + val mispredictionValid = io.redirect.valid val exceptionValid = io.redirect.valid && io.redirect.bits.isException val flushPipeValid = io.redirect.valid && io.redirect.bits.isFlushPipe val roqNeedFlush = Wire(Vec(size, Bool())) @@ -106,12 +103,15 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H // For dequeue, the first entry should never be s_invalid // Otherwise, there should be a redirect and tail walks back // in this case, we set numDeq to 0 - !deq.fire() && (if (i == 0) true.B else stateEntries(deqIndex(i)) =/= s_invalid) + !deq.fire() && (if (i == 0) true.B else stateEntries(headPtr(i).value) =/= s_invalid) } :+ true.B) val numDeq = Mux(numDeqTry > numDeqFire, numDeqFire, numDeqTry) // agreement with reservation station: don't dequeue when redirect.valid - val headPtrNext = Mux(mispredictionValid, headPtr, headPtr + numDeq) - headPtr := Mux(exceptionValid, 0.U.asTypeOf(new CircularQueuePtr(size)), headPtrNext) + for (i <- 0 until deqnum) { + headPtr(i) := Mux(exceptionValid, + i.U.asTypeOf(new CircularQueuePtr(size)), + Mux(mispredictionValid, headPtr(i), headPtr(i) + numDeq)) + } // For branch mis-prediction or memory violation replay, // we delay updating the indices for one clock cycle. @@ -124,18 +124,31 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H val flippedFlag = loValidBitVec.orR val lastOneIndex = size.U - PriorityEncoder(Mux(loValidBitVec.orR, loValidBitVec, hiValidBitVec)) val walkedTailPtr = Wire(new CircularQueuePtr(size)) - walkedTailPtr.flag := flippedFlag ^ headPtr.flag + walkedTailPtr.flag := flippedFlag ^ headPtr(0).flag walkedTailPtr.value := lastOneIndex // enqueue - val numEnq = Mux(canActualEnqueue, PriorityEncoder(io.enq.map(!_.valid) :+ true.B), 0.U) - XSError(numEnq =/= 0.U && (mispredictionValid || exceptionValid), "should not enqueue when redirect\n") - tailPtr := Mux(exceptionValid, + val numEnq = Mux(io.enq.canAccept, PopCount(io.enq.req.map(_.valid)), 0.U) + tailPtr(0) := Mux(exceptionValid, 0.U.asTypeOf(new CircularQueuePtr(size)), - Mux(lastCycleMisprediction, - Mux(isTrueEmpty, headPtr, walkedTailPtr), - tailPtr + numEnq) + Mux(io.redirect.valid, + tailPtr(0), + Mux(lastCycleMisprediction, + Mux(isTrueEmpty, headPtr(0), walkedTailPtr), + tailPtr(0) + numEnq)) ) + val lastCycleException = RegNext(exceptionValid) + val lastLastCycleMisprediction = RegNext(lastCycleMisprediction) + for (i <- 1 until enqnum) { + tailPtr(i) := Mux(exceptionValid, + i.U.asTypeOf(new CircularQueuePtr(size)), + Mux(io.redirect.valid, + tailPtr(i), + Mux(lastLastCycleMisprediction, + tailPtr(0) + i.U, + tailPtr(i) + numEnq)) + ) + } /** @@ -143,13 +156,13 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H */ // TODO: remove this when replay moves to roq for (i <- 0 until deqnum) { - io.deq(i).bits := uopEntries(deqIndex(i)) + io.deq(i).bits := uopEntries(headPtr(i).value) // do not dequeue when io.redirect valid because it may cause dispatchPtr work improperly - io.deq(i).valid := stateEntries(deqIndex(i)) === s_valid && !lastCycleMisprediction// && !io.redirect.valid + io.deq(i).valid := stateEntries(headPtr(i).value) === s_valid && !lastCycleMisprediction } // debug: dump dispatch queue states - XSDebug(p"head: $headPtr, tail: $tailPtr\n") + XSDebug(p"head: ${headPtr(0)}, tail: ${tailPtr(0)}\n") XSDebug(p"state: ") stateEntries.reverse.foreach { s => XSDebug(false, s === s_invalid, "-") @@ -158,11 +171,11 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H XSDebug(false, true.B, "\n") XSDebug(p"ptr: ") (0 until size).reverse.foreach { i => - val isPtr = i.U === headPtr.value || i.U === tailPtr.value + val isPtr = i.U === headPtr(0).value || i.U === tailPtr(0).value XSDebug(false, isPtr, "^") XSDebug(false, !isPtr, " ") } XSDebug(false, true.B, "\n") - XSError(isAfter(headPtr, tailPtr), p"assert greaterOrEqualThan(tailPtr: $tailPtr, headPtr: $headPtr) failed\n") + XSError(isAfter(headPtr(0), tailPtr(0)), p"assert greaterOrEqualThan(tailPtr: ${tailPtr(0)}, headPtr: ${headPtr(0)}) failed\n") } diff --git a/src/main/scala/xiangshan/backend/fu/CSR.scala b/src/main/scala/xiangshan/backend/fu/CSR.scala index 2f025fefd3992c5d11a17e6b686a22228c645a0c..c8fb508d68bb1e7a0423748980a010865d79f372 100644 --- a/src/main/scala/xiangshan/backend/fu/CSR.scala +++ b/src/main/scala/xiangshan/backend/fu/CSR.scala @@ -828,7 +828,6 @@ class CSR extends FunctionUnit with HasCSRConst "RoqWaitFp" -> (0xb11, "perfCntCondRoqWaitFp" ), "RoqWaitLoad" -> (0xb12, "perfCntCondRoqWaitLoad" ), "RoqWaitStore"-> (0xb13, "perfCntCondRoqWaitStore"), - "Dp1Empty" -> (0xb14, "perfCntCondDp1Empty" ), "DTlbReqCnt0" -> (0xb15, "perfCntDtlbReqCnt0" ), "DTlbReqCnt1" -> (0xb16, "perfCntDtlbReqCnt1" ), "DTlbReqCnt2" -> (0xb17, "perfCntDtlbReqCnt2" ), diff --git a/src/main/scala/xiangshan/backend/roq/Roq.scala b/src/main/scala/xiangshan/backend/roq/Roq.scala index b60fd9653563225c6ae0310c57e15f73e942e5b6..5e7fa4b44ebfa1b5b0a25b0923d3f948f0304848 100644 --- a/src/main/scala/xiangshan/backend/roq/Roq.scala +++ b/src/main/scala/xiangshan/backend/roq/Roq.scala @@ -38,17 +38,20 @@ class RoqCSRIO extends XSBundle { val dirty_fs = Output(Bool()) } +class RoqEnqIO extends XSBundle { + val canAccept = Output(Bool()) + val isEmpty = Output(Bool()) + // valid vector, for roqIdx gen and walk + val needAlloc = Vec(RenameWidth, Input(Bool())) + val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp))) + val resp = Vec(RenameWidth, Output(new RoqPtr)) +} + class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper { val io = IO(new Bundle() { val brqRedirect = Input(Valid(new Redirect)) val memRedirect = Input(Valid(new Redirect)) - val enq = new Bundle { - val canAccept = Output(Bool()) - val isEmpty = Output(Bool()) - val extraWalk = Vec(RenameWidth, Input(Bool())) - val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp))) - val resp = Vec(RenameWidth, Output(new RoqPtr)) - } + val enq = new RoqEnqIO val redirect = Output(Valid(new Redirect)) val exception = Output(new MicroOp) // exu + brq @@ -108,14 +111,7 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper { io.roqDeqPtr := deqPtrExt // common signal - val enqPtrValPlus = Wire(Vec(RenameWidth, UInt(log2Up(RoqSize).W))) - val enqPtrFlagPlus = Wire(Vec(RenameWidth, Bool())) - for (i <- 0 until RenameWidth) { - val offset = PopCount(io.enq.req.map(_.valid).take(i)) - val roqIdxExt = enqPtrExt + offset - enqPtrValPlus(i) := roqIdxExt.value - enqPtrFlagPlus(i) := roqIdxExt.flag - } + val enqPtrVec = WireInit(VecInit((0 until RenameWidth).map(i => enqPtrExt + PopCount(io.enq.needAlloc.take(i))))) val deqPtrExtPlus = Wire(Vec(RenameWidth, UInt(log2Up(RoqSize).W))) for(i <- 0 until CommitWidth){ @@ -138,14 +134,9 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper { val walkNoSpecExec = io.commits.isWalk && Cat((0 until CommitWidth).map(i => io.commits.valid(i) && io.commits.uop(i).ctrl.noSpecExec)).orR XSError(state =/= s_extrawalk && walkNoSpecExec, "noSpecExec should not walk\n") - val validDispatch = io.enq.req.map(_.valid) for (i <- 0 until RenameWidth) { - val offset = PopCount(validDispatch.take(i)) - val roqIdxExt = enqPtrExt + offset - val roqIdx = roqIdxExt.value - - when(io.enq.req(i).valid) { - microOp(roqIdx) := io.enq.req(i).bits + when(io.enq.req(i).valid && io.enq.canAccept) { + microOp(enqPtrVec(i).value) := io.enq.req(i).bits when(io.enq.req(i).bits.ctrl.blockBackward) { hasBlockBackward := true.B } @@ -153,19 +144,18 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper { hasNoSpecExec := true.B } } - io.enq.resp(i) := roqIdxExt + io.enq.resp(i) := enqPtrVec(i) } val validEntries = distanceBetween(enqPtrExt, deqPtrExt) - val firedDispatch = Cat(io.enq.req.map(_.valid)) + val firedDispatch = Mux(io.enq.canAccept, PopCount(Cat(io.enq.req.map(_.valid))), 0.U) io.enq.canAccept := (validEntries <= (RoqSize - RenameWidth).U) && !hasBlockBackward io.enq.isEmpty := isEmpty - XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(firedDispatch)}\n") + XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n") - val dispatchCnt = PopCount(firedDispatch) - enqPtrExt := enqPtrExt + dispatchCnt - when (firedDispatch.orR) { - XSInfo("dispatched %d insts\n", dispatchCnt) + enqPtrExt := enqPtrExt + firedDispatch + when (firedDispatch =/= 0.U) { + XSInfo("dispatched %d insts\n", firedDispatch) } // Writeback @@ -224,7 +214,7 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper { // extra space is used weh roq has no enough space, but mispredict recovery needs such info to walk regmap val needExtraSpaceForMPR = WireInit(VecInit( - List.tabulate(RenameWidth)(i => io.brqRedirect.valid && io.enq.extraWalk(i)) + List.tabulate(RenameWidth)(i => io.brqRedirect.valid && io.enq.needAlloc(i)) )) val extraSpaceForMPR = Reg(Vec(RenameWidth, new MicroOp)) val usedSpaceForMPR = Reg(Vec(RenameWidth, Bool())) @@ -345,12 +335,12 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper { // when redirect, walk back roq entries when(io.brqRedirect.valid){ // TODO: need check if consider exception redirect? state := s_walk - val nextEnqPtr = (enqPtrExt - 1.U) + dispatchCnt + val walkPtrStart = enqPtrExt - 1.U walkPtrExt := Mux(state === s_walk, walkPtrExt - Mux(walkFinished, walkCounter, CommitWidth.U), - Mux(state === s_extrawalk, walkPtrExt, nextEnqPtr)) + Mux(state === s_extrawalk, walkPtrExt, walkPtrStart)) // walkTgtExt := io.brqRedirect.bits.roqIdx - val currentWalkPtr = Mux(state === s_walk || state === s_extrawalk, walkPtrExt, nextEnqPtr) + val currentWalkPtr = Mux(state === s_walk || state === s_extrawalk, walkPtrExt, walkPtrStart) walkCounter := distanceBetween(currentWalkPtr, io.brqRedirect.bits.roqIdx) - Mux(state === s_walk, commitCnt, 0.U) enqPtrExt := io.brqRedirect.bits.roqIdx + 1.U } @@ -375,8 +365,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper { // write // enqueue logic writes 6 valid for (i <- 0 until RenameWidth) { - when(io.enq.req(i).fire()){ - valid(enqPtrValPlus(i)) := true.B + when(io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid){ + valid(enqPtrVec(i).value) := true.B } } // dequeue/walk logic writes 6 valid, dequeue and walk will not happen at the same time @@ -412,8 +402,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper { // write // enqueue logic set 6 writebacked to false for (i <- 0 until RenameWidth) { - when(io.enq.req(i).fire()){ - writebacked(enqPtrValPlus(i)) := false.B + when(io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid){ + writebacked(enqPtrVec(i).value) := false.B } } // writeback logic set numWbPorts writebacked to true @@ -443,8 +433,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper { // write: update when enqueue // enqueue logic set 6 flagBkup at most for (i <- 0 until RenameWidth) { - when(io.enq.req(i).fire()){ - flagBkup(enqPtrValPlus(i)) := enqPtrFlagPlus(i) + when(io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid){ + flagBkup(enqPtrVec(i).value) := enqPtrVec(i).flag } } // read: used in rollback logic diff --git a/src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala b/src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala index 1afbded5ffc540a67d6af46723528eb8b58bb3b3..782999863d46b959d83d7b21066c546bddc8042d 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala @@ -53,7 +53,7 @@ class LSQueueData(size: Int, nchannel: Int) extends XSModule with HasDCacheParam val needForward = Input(Vec(nchannel, Vec(2, UInt(size.W)))) val forward = Vec(nchannel, Flipped(new LoadForwardQueryIO)) val rdata = Output(Vec(size, new LsqEntry)) - + // val debug = new Bundle() { // val debug_data = Vec(LoadQueueSize, new LsqEntry) // } @@ -76,7 +76,7 @@ class LSQueueData(size: Int, nchannel: Int) extends XSModule with HasDCacheParam this.needForward(channel)(1) := needForward2 this.forward(channel).paddr := paddr } - + // def refillWrite(ldIdx: Int): Unit = { // } // use "this.refill.wen(ldIdx) := true.B" instead @@ -229,14 +229,17 @@ class InflightBlockInfo extends XSBundle { val valid = Bool() } +class LsqEnqIO extends XSBundle { + val canAccept = Output(Bool()) + val needAlloc = Vec(RenameWidth, Input(Bool())) + val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp))) + val resp = Vec(RenameWidth, Output(new LSIdx)) +} + // Load / Store Queue Wrapper for XiangShan Out of Order LSU class LsqWrappper extends XSModule with HasDCacheParameters { val io = IO(new Bundle() { - val enq = new Bundle() { - val canAccept = Output(Bool()) - val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp))) - val resp = Vec(RenameWidth, Output(new LSIdx)) - } + val enq = new LsqEnqIO val brqRedirect = Input(Valid(new Redirect)) val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LsPipelineBundle))) val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) @@ -261,14 +264,17 @@ class LsqWrappper extends XSModule with HasDCacheParameters { io.enq.canAccept := loadQueue.io.enq.canAccept && storeQueue.io.enq.canAccept for (i <- 0 until RenameWidth) { val isStore = CommitType.lsInstIsStore(io.enq.req(i).bits.ctrl.commitType) + + loadQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i) && !isStore loadQueue.io.enq.req(i).valid := !isStore && io.enq.req(i).valid - storeQueue.io.enq.req(i).valid := isStore && io.enq.req(i).valid loadQueue.io.enq.req(i).bits := io.enq.req(i).bits + + storeQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i) && isStore + storeQueue.io.enq.req(i).valid := isStore && io.enq.req(i).valid storeQueue.io.enq.req(i).bits := io.enq.req(i).bits + io.enq.resp(i).lqIdx := loadQueue.io.enq.resp(i) io.enq.resp(i).sqIdx := storeQueue.io.enq.resp(i) - - XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not enqueue LSQ when not") } // load queue wiring @@ -294,7 +300,7 @@ class LsqWrappper extends XSModule with HasDCacheParameters { storeQueue.io.exceptionAddr.lsIdx := io.exceptionAddr.lsIdx storeQueue.io.exceptionAddr.isStore := DontCare - loadQueue.io.forward <> io.forward + loadQueue.io.load_s1 <> io.forward storeQueue.io.forward <> io.forward // overlap forwardMask & forwardData, DO NOT CHANGE SEQUENCE io.exceptionAddr.vaddr := Mux(io.exceptionAddr.isStore, storeQueue.io.exceptionAddr.vaddr, loadQueue.io.exceptionAddr.vaddr) diff --git a/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala b/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala index 64dd6d00ae9e8286afeac8381059e3d7f2fb2d5d..6944c2ee567cce389c4110dda4a5cb6438fa6149 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala @@ -23,27 +23,28 @@ object LqPtr extends HasXSParameter { } } +class LqEnqIO extends XSBundle { + val canAccept = Output(Bool()) + val needAlloc = Vec(RenameWidth, Input(Bool())) + val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp))) + val resp = Vec(RenameWidth, Output(new LqPtr)) +} // Load Queue class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper { val io = IO(new Bundle() { - val enq = new Bundle() { - val canAccept = Output(Bool()) - val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp))) - val resp = Vec(RenameWidth, Output(new LqPtr)) - } + val enq = new LqEnqIO val brqRedirect = Input(Valid(new Redirect)) val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LsPipelineBundle))) val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // FIXME: Valid() only val ldout = Vec(2, DecoupledIO(new ExuOutput)) // writeback load - val forward = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO)) + val load_s1 = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO)) val commits = Flipped(new RoqCommitIO) val rollback = Output(Valid(new Redirect)) // replay now starts from load instead of store val dcache = new DCacheLineIO val uncache = new DCacheWordIO val roqDeqPtr = Input(new RoqPtr) val exceptionAddr = new ExceptionAddrIO - // val refill = Flipped(Valid(new DCacheLineReq )) }) val uop = Reg(Vec(LoadQueueSize, new MicroOp)) @@ -58,11 +59,11 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP val listening = Reg(Vec(LoadQueueSize, Bool())) // waiting for refill result val pending = Reg(Vec(LoadQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of roq - val enqPtrExt = RegInit(0.U.asTypeOf(new LqPtr)) + val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new LqPtr)))) val deqPtrExt = RegInit(0.U.asTypeOf(new LqPtr)) - val enqPtr = enqPtrExt.value + val enqPtr = enqPtrExt(0).value val deqPtr = deqPtrExt.value - val sameFlag = enqPtrExt.flag === deqPtrExt.flag + val sameFlag = enqPtrExt(0).flag === deqPtrExt.flag val isEmpty = enqPtr === deqPtr && sameFlag val isFull = enqPtr === deqPtr && !sameFlag val allowIn = !isFull @@ -72,19 +73,21 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP val deqMask = UIntToMask(deqPtr, LoadQueueSize) val enqMask = UIntToMask(enqPtr, LoadQueueSize) - val enqDeqMask1 = deqMask ^ enqMask - val enqDeqMask = Mux(sameFlag, enqDeqMask1, ~enqDeqMask1) - // Enqueue at dispatch - val validEntries = distanceBetween(enqPtrExt, deqPtrExt) + /** + * Enqueue at dispatch + * + * Currently, LoadQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth) + */ + val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt) val firedDispatch = io.enq.req.map(_.valid) io.enq.canAccept := validEntries <= (LoadQueueSize - RenameWidth).U XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n") for (i <- 0 until RenameWidth) { - val offset = if (i == 0) 0.U else PopCount((0 until i).map(firedDispatch(_))) - val lqIdx = enqPtrExt + offset + val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i)) + val lqIdx = enqPtrExt(offset) val index = lqIdx.value - when(io.enq.req(i).valid) { + when (io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid) { uop(index) := io.enq.req(i).bits allocated(index) := true.B datavalid(index) := false.B @@ -95,17 +98,28 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP pending(index) := false.B } io.enq.resp(i) := lqIdx - - XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not valid when not ready\n") } - when(Cat(firedDispatch).orR) { - enqPtrExt := enqPtrExt + PopCount(firedDispatch) - XSInfo("dispatched %d insts to lq\n", PopCount(firedDispatch)) + // when io.brqRedirect.valid, we don't allow eneuque even though it may fire. + when (Cat(firedDispatch).orR && io.enq.canAccept && !io.brqRedirect.valid) { + val enqNumber = PopCount(firedDispatch) + enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) + XSInfo("dispatched %d insts to lq\n", enqNumber) } - // writeback load - (0 until LoadPipelineWidth).map(i => { + /** + * Writeback load from load units + * + * Most load instructions writeback to regfile at the same time. + * However, + * (1) For an mmio instruction with exceptions, it writes back to ROB immediately. + * (2) For an mmio instruction without exceptions, it does not write back. + * The mmio instruction will be sent to lower level when it reaches ROB's head. + * After uncache response, it will write back through arbiter with loadUnit. + * (3) For cache misses, it is marked miss and sent to dcache later. + * After cache refills, it will write back through arbiter with loadUnit. + */ + for (i <- 0 until LoadPipelineWidth) { dataModule.io.wb(i).wen := false.B when(io.loadIn(i).fire()) { when(io.loadIn(i).bits.miss) { @@ -140,7 +154,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP val loadWbIndex = io.loadIn(i).bits.uop.lqIdx.value datavalid(loadWbIndex) := !io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio writebacked(loadWbIndex) := !io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio - allocated(loadWbIndex) := !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR val loadWbData = Wire(new LsqEntry) loadWbData.paddr := io.loadIn(i).bits.paddr @@ -155,13 +168,20 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP dataModule.io.wb(i).wen := true.B val dcacheMissed = io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio - miss(loadWbIndex) := dcacheMissed + miss(loadWbIndex) := dcacheMissed && !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR listening(loadWbIndex) := dcacheMissed - pending(loadWbIndex) := io.loadIn(i).bits.mmio + pending(loadWbIndex) := io.loadIn(i).bits.mmio && !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR } - }) + } - // cache miss request + /** + * Cache miss request + * + * (1) writeback: miss + * (2) send to dcache: listing + * (3) dcache response: datavalid + * (4) writeback to ROB: writeback + */ val inflightReqs = RegInit(VecInit(Seq.fill(cfg.nLoadMissEntries)(0.U.asTypeOf(new InflightBlockInfo)))) val inflightReqFull = inflightReqs.map(req => req.valid).reduce(_&&_) val reqBlockIndex = PriorityEncoder(~VecInit(inflightReqs.map(req => req.valid)).asUInt) @@ -303,21 +323,18 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP } }) - // move tailPtr - // allocatedMask: dequeuePtr can go to the next 1-bit - val allocatedMask = VecInit((0 until LoadQueueSize).map(i => allocated(i) || !enqDeqMask(i))) - // find the first one from deqPtr (deqPtr) - val nextTail1 = getFirstOneWithFlag(allocatedMask, deqMask, deqPtrExt.flag) - val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, enqPtrExt) - deqPtrExt := nextTail - - // When load commited, mark it as !allocated, this entry will be recycled later + /** + * Load commits + * + * When load commited, mark it as !allocated and move deqPtrExt forward. + */ (0 until CommitWidth).map(i => { when(loadCommit(i)) { allocated(mcommitIdx(i)) := false.B XSDebug("load commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc) } }) + deqPtrExt := deqPtrExt + PopCount(loadCommit) def getFirstOne(mask: Vec[Bool], startMask: UInt) = { val length = mask.length @@ -326,15 +343,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP PriorityEncoder(Mux(highBitsUint.orR(), highBitsUint, mask.asUInt)) } - def getFirstOneWithFlag(mask: Vec[Bool], startMask: UInt, startFlag: Bool) = { - val length = mask.length - val highBits = (0 until length).map(i => mask(i) & ~startMask(i)) - val highBitsUint = Cat(highBits.reverse) - val changeDirection = !highBitsUint.orR() - val index = PriorityEncoder(Mux(!changeDirection, highBitsUint, mask.asUInt)) - LqPtr(startFlag ^ changeDirection, index) - } - def getOldestInTwo(valid: Seq[Bool], uop: Seq[MicroOp]) = { assert(valid.length == uop.length) assert(valid.length == 2) @@ -355,25 +363,25 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP }) } - def rangeMask(start: LqPtr, end: LqPtr): UInt = { - val startMask = (1.U((LoadQueueSize + 1).W) << start.value).asUInt - 1.U - val endMask = (1.U((LoadQueueSize + 1).W) << end.value).asUInt - 1.U - val xorMask = startMask(LoadQueueSize - 1, 0) ^ endMask(LoadQueueSize - 1, 0) - Mux(start.flag === end.flag, xorMask, ~xorMask) - } - - // ignore data forward - (0 until LoadPipelineWidth).foreach(i => { - io.forward(i).forwardMask := DontCare - io.forward(i).forwardData := DontCare - }) - - // store backward query and rollback + /** + * Memory violation detection + * + * When store writes back, it searches LoadQueue for younger load instructions + * with the same load physical address. They loaded wrong data and need re-execution. + * + * Cycle 0: Store Writeback + * Generate match vector for store address with rangeMask(stPtr, enqPtr). + * Besides, load instructions in LoadUnit_S1 and S2 are also checked. + * Cycle 1: Redirect Generation + * There're three possible types of violations. Choose the oldest load. + * Set io.redirect according to the detected violation. + */ + io.load_s1 := DontCare def detectRollback(i: Int) = { val startIndex = io.storeIn(i).bits.uop.lqIdx.value val lqIdxMask = UIntToMask(startIndex, LoadQueueSize) val xorMask = lqIdxMask ^ enqMask - val sameFlag = io.storeIn(i).bits.uop.lqIdx.flag === enqPtrExt.flag + val sameFlag = io.storeIn(i).bits.uop.lqIdx.flag === enqPtrExt(0).flag val toEnqPtrMask = Mux(sameFlag, xorMask, ~xorMask) // check if load already in lq needs to be rolledback @@ -405,13 +413,13 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP // check if rollback is needed for load in l1 val l1ViolationVec = RegNext(VecInit((0 until LoadPipelineWidth).map(j => { - io.forward(j).valid && // L1 valid - isAfter(io.forward(j).uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) && - io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.forward(j).paddr(PAddrBits - 1, 3) && - (io.storeIn(i).bits.mask & io.forward(j).mask).orR + io.load_s1(j).valid && // L1 valid + isAfter(io.load_s1(j).uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) && + io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.load_s1(j).paddr(PAddrBits - 1, 3) && + (io.storeIn(i).bits.mask & io.load_s1(j).mask).orR }))) val l1Violation = l1ViolationVec.asUInt().orR() - val l1ViolationUop = getOldestInTwo(l1ViolationVec, RegNext(VecInit(io.forward.map(_.uop)))) + val l1ViolationUop = getOldestInTwo(l1ViolationVec, RegNext(VecInit(io.load_s1.map(_.uop)))) XSDebug(l1Violation, p"${Binary(Cat(l1ViolationVec))}, $l1ViolationUop\n") val rollbackValidVec = Seq(lqViolation, wbViolation, l1Violation) @@ -465,10 +473,10 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP val rollbackSelected = ParallelOperation(rollback, rollbackSel) val lastCycleRedirect = RegNext(io.brqRedirect) - io.rollback := DontCare // Note that we use roqIdx - 1.U to flush the load instruction itself. // Thus, here if last cycle's roqIdx equals to this cycle's roqIdx, it still triggers the redirect. - io.rollback.valid := rollbackSelected.valid && (!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx)) && + io.rollback.valid := rollbackSelected.valid && + (!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx)) && !(lastCycleRedirect.valid && (lastCycleRedirect.bits.isFlushPipe || lastCycleRedirect.bits.isException)) io.rollback.bits.roqIdx := rollbackSelected.bits.roqIdx - 1.U @@ -476,13 +484,18 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP io.rollback.bits.isMisPred := false.B io.rollback.bits.isException := false.B io.rollback.bits.isFlushPipe := false.B + io.rollback.bits.pc := DontCare io.rollback.bits.target := rollbackSelected.bits.cf.pc io.rollback.bits.brTag := rollbackSelected.bits.brTag - // Memory mapped IO / other uncached operations + when(io.rollback.valid) { + XSDebug("Mem rollback: pc %x roqidx %d\n", io.rollback.bits.pc, io.rollback.bits.roqIdx.asUInt) + } - // setup misc mem access req - // mask / paddr / data can be get from lq.data + /** + * Memory mapped IO / other uncached operations + * + */ val commitType = io.commits.uop(0).ctrl.commitType io.uncache.req.valid := pending(deqPtr) && allocated(deqPtr) && commitType === CommitType.LOAD && @@ -494,11 +507,11 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP io.uncache.req.bits.data := dataModule.io.rdata(deqPtr).data io.uncache.req.bits.mask := dataModule.io.rdata(deqPtr).mask - io.uncache.req.bits.meta.id := DontCare // TODO: // FIXME + io.uncache.req.bits.meta.id := DontCare io.uncache.req.bits.meta.vaddr := DontCare io.uncache.req.bits.meta.paddr := dataModule.io.rdata(deqPtr).paddr io.uncache.req.bits.meta.uop := uop(deqPtr) - io.uncache.req.bits.meta.mmio := true.B // dataModule.io.rdata(deqPtr).mmio + io.uncache.req.bits.meta.mmio := true.B io.uncache.req.bits.meta.tlb_miss := false.B io.uncache.req.bits.meta.mask := dataModule.io.rdata(deqPtr).mask io.uncache.req.bits.meta.replay := false.B @@ -507,17 +520,7 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP when (io.uncache.req.fire()) { pending(deqPtr) := false.B - } - dataModule.io.uncache.wen := false.B - when(io.uncache.resp.fire()){ - datavalid(deqPtr) := true.B - dataModule.io.uncacheWrite(deqPtr, io.uncache.resp.bits.data(XLEN-1, 0)) - dataModule.io.uncache.wen := true.B - // TODO: write back exception info - } - - when(io.uncache.req.fire()){ XSDebug("uncache req: pc %x addr %x data %x op %x mask %x\n", uop(deqPtr).cf.pc, io.uncache.req.bits.addr, @@ -527,7 +530,12 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP ) } + dataModule.io.uncache.wen := false.B when(io.uncache.resp.fire()){ + datavalid(deqPtr) := true.B + dataModule.io.uncacheWrite(deqPtr, io.uncache.resp.bits.data(XLEN-1, 0)) + dataModule.io.uncache.wen := true.B + XSDebug("uncache resp: data %x\n", io.dcache.resp.bits.data) } @@ -539,29 +547,19 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP val needCancel = Wire(Vec(LoadQueueSize, Bool())) for (i <- 0 until LoadQueueSize) { needCancel(i) := uop(i).roqIdx.needFlush(io.brqRedirect) && allocated(i) && !commited(i) - when(needCancel(i)) { - // when(io.brqRedirect.bits.isReplay){ - // valid(i) := false.B - // writebacked(i) := false.B - // listening(i) := false.B - // miss(i) := false.B - // pending(i) := false.B - // }.otherwise{ + when (needCancel(i)) { allocated(i) := false.B - // } } } - when (io.brqRedirect.valid && io.brqRedirect.bits.isMisPred) { - enqPtrExt := enqPtrExt - PopCount(needCancel) - } - - // assert(!io.rollback.valid) - when(io.rollback.valid) { - XSDebug("Mem rollback: pc %x roqidx %d\n", io.rollback.bits.pc, io.rollback.bits.roqIdx.asUInt) + // we recover the pointers in the next cycle after redirect + val needCancelReg = RegNext(needCancel) + when (lastCycleRedirect.valid) { + val cancelCount = PopCount(needCancelReg) + enqPtrExt := VecInit(enqPtrExt.map(_ - cancelCount)) } // debug info - XSDebug("head %d:%d tail %d:%d\n", enqPtrExt.flag, enqPtr, deqPtrExt.flag, deqPtr) + XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt.flag, deqPtr) def PrintFlag(flag: Bool, name: String): Unit = { when(flag) { diff --git a/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala b/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala index 6dca78ebe8c15de737e90b3246a01d05d14111c6..39a3ded04b265a407355e5caf0f266f563d4f437 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala @@ -21,14 +21,17 @@ object SqPtr extends HasXSParameter { } } +class SqEnqIO extends XSBundle { + val canAccept = Output(Bool()) + val needAlloc = Vec(RenameWidth, Input(Bool())) + val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp))) + val resp = Vec(RenameWidth, Output(new SqPtr)) +} + // Store Queue class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper { val io = IO(new Bundle() { - val enq = new Bundle() { - val canAccept = Output(Bool()) - val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp))) - val resp = Vec(RenameWidth, Output(new SqPtr)) - } + val enq = new SqEnqIO val brqRedirect = Input(Valid(new Redirect)) val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) val sbuffer = Vec(StorePipelineWidth, Decoupled(new DCacheWordReq)) @@ -51,33 +54,29 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue val commited = Reg(Vec(StoreQueueSize, Bool())) // inst has been commited by roq val pending = Reg(Vec(StoreQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of roq - val enqPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) - val deqPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) - val enqPtr = enqPtrExt.value - val deqPtr = deqPtrExt.value - val sameFlag = enqPtrExt.flag === deqPtrExt.flag - val isEmpty = enqPtr === deqPtr && sameFlag - val isFull = enqPtr === deqPtr && !sameFlag - val allowIn = !isFull - - val storeCommit = (0 until CommitWidth).map(i => io.commits.valid(i) && !io.commits.isWalk && io.commits.uop(i).ctrl.commitType === CommitType.STORE) - val mcommitIdx = (0 until CommitWidth).map(i => io.commits.uop(i).sqIdx.value) + require(StoreQueueSize > RenameWidth) + val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new SqPtr)))) + val deqPtrExt = RegInit(VecInit((0 until StorePipelineWidth).map(_.U.asTypeOf(new SqPtr)))) + val enqPtr = enqPtrExt(0).value + val deqPtr = deqPtrExt(0).value val tailMask = UIntToMask(deqPtr, StoreQueueSize) val headMask = UIntToMask(enqPtr, StoreQueueSize) - val enqDeqMask1 = tailMask ^ headMask - val enqDeqMask = Mux(sameFlag, enqDeqMask1, ~enqDeqMask1) - // Enqueue at dispatch - val validEntries = distanceBetween(enqPtrExt, deqPtrExt) + /** + * Enqueue at dispatch + * + * Currently, StoreQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth) + */ + val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt(0)) val firedDispatch = io.enq.req.map(_.valid) io.enq.canAccept := validEntries <= (StoreQueueSize - RenameWidth).U XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n") for (i <- 0 until RenameWidth) { - val offset = if (i == 0) 0.U else PopCount((0 until i).map(firedDispatch(_))) - val sqIdx = enqPtrExt + offset + val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i)) + val sqIdx = enqPtrExt(offset) val index = sqIdx.value - when(io.enq.req(i).valid) { + when (io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid) { uop(index) := io.enq.req(i).bits allocated(index) := true.B datavalid(index) := false.B @@ -86,17 +85,28 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue pending(index) := false.B } io.enq.resp(i) := sqIdx - - XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not valid when not ready\n") } - when(Cat(firedDispatch).orR) { - enqPtrExt := enqPtrExt + PopCount(firedDispatch) - XSInfo("dispatched %d insts to sq\n", PopCount(firedDispatch)) + when (Cat(firedDispatch).orR && io.enq.canAccept && !io.brqRedirect.valid) { + val enqNumber = PopCount(firedDispatch) + enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) + XSInfo("dispatched %d insts to sq\n", enqNumber) } - // writeback store - (0 until StorePipelineWidth).map(i => { + /** + * Writeback store from store units + * + * Most store instructions writeback to regfile in the previous cycle. + * However, + * (1) For an mmio instruction with exceptions, we need to mark it as datavalid + * (in this way it will trigger an exception when it reaches ROB's head) + * instead of pending to avoid sending them to lower level. + * (2) For an mmio instruction without exceptions, we mark it as pending. + * When the instruction reaches ROB's head, StoreQueue sends it to uncache channel. + * Upon receiving the response, StoreQueue writes back the instruction + * through arbiter with store units. It will later commit as normal. + */ + for (i <- 0 until StorePipelineWidth) { dataModule.io.wb(i).wen := false.B when(io.storeIn(i).fire()) { val stWbIndex = io.storeIn(i).bits.uop.sqIdx.value @@ -129,94 +139,16 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue io.storeIn(i).bits.uop.cf.exceptionVec.asUInt ) } - }) - - def getFirstOne(mask: Vec[Bool], startMask: UInt) = { - val length = mask.length - val highBits = (0 until length).map(i => mask(i) & ~startMask(i)) - val highBitsUint = Cat(highBits.reverse) - PriorityEncoder(Mux(highBitsUint.orR(), highBitsUint, mask.asUInt)) - } - - def getFirstOneWithFlag(mask: Vec[Bool], startMask: UInt, startFlag: Bool) = { - val length = mask.length - val highBits = (0 until length).map(i => mask(i) & ~startMask(i)) - val highBitsUint = Cat(highBits.reverse) - val changeDirection = !highBitsUint.orR() - val index = PriorityEncoder(Mux(!changeDirection, highBitsUint, mask.asUInt)) - SqPtr(startFlag ^ changeDirection, index) - } - - def selectFirstTwo(valid: Vec[Bool], startMask: UInt) = { - val selVec = Wire(Vec(2, UInt(log2Up(StoreQueueSize).W))) - val selValid = Wire(Vec(2, Bool())) - selVec(0) := getFirstOne(valid, startMask) - val firstSelMask = UIntToOH(selVec(0)) - val secondSelVec = VecInit((0 until valid.length).map(i => valid(i) && !firstSelMask(i))) - selVec(1) := getFirstOne(secondSelVec, startMask) - selValid(0) := Cat(valid).orR - selValid(1) := Cat(secondSelVec).orR - (selValid, selVec) - } - - def selectFirstTwoRoughly(valid: Vec[Bool]) = { - // TODO: do not select according to seq, just select 2 valid bit randomly - val firstSelVec = valid - val notFirstVec = Wire(Vec(valid.length, Bool())) - (0 until valid.length).map(i => - notFirstVec(i) := (if(i != 0) { valid(i) || !notFirstVec(i) } else { false.B }) - ) - val secondSelVec = VecInit((0 until valid.length).map(i => valid(i) && !notFirstVec(i))) - - val selVec = Wire(Vec(2, UInt(log2Up(valid.length).W))) - val selValid = Wire(Vec(2, Bool())) - selVec(0) := PriorityEncoder(firstSelVec) - selVec(1) := PriorityEncoder(secondSelVec) - selValid(0) := Cat(firstSelVec).orR - selValid(1) := Cat(secondSelVec).orR - (selValid, selVec) } - // writeback finished mmio store - io.mmioStout.bits.uop := uop(deqPtr) - io.mmioStout.bits.uop.sqIdx := deqPtrExt - io.mmioStout.bits.uop.cf.exceptionVec := dataModule.io.rdata(deqPtr).exception.asBools - io.mmioStout.bits.data := dataModule.io.rdata(deqPtr).data - io.mmioStout.bits.redirectValid := false.B - io.mmioStout.bits.redirect := DontCare - io.mmioStout.bits.brUpdate := DontCare - io.mmioStout.bits.debug.isMMIO := true.B - io.mmioStout.bits.fflags := DontCare - io.mmioStout.valid := allocated(deqPtr) && datavalid(deqPtr) && !writebacked(deqPtr) // finished mmio store - when(io.mmioStout.fire()) { - writebacked(deqPtr) := true.B - allocated(deqPtr) := false.B // potential opt: move deqPtr immediately - } - - // remove retired insts from sq, add retired store to sbuffer - - // move tailPtr - // TailPtr slow recovery: recycle bubbles in store queue - // allocatedMask: dequeuePtr can go to the next 1-bit - val allocatedMask = VecInit((0 until StoreQueueSize).map(i => allocated(i) || !enqDeqMask(i))) - // find the first one from deqPtr (deqPtr) - val nextTail1 = getFirstOneWithFlag(allocatedMask, tailMask, deqPtrExt.flag) - val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, enqPtrExt) - deqPtrExt := nextTail - - // TailPtr fast recovery - // val tailRecycle = VecInit(List( - // io.uncache.resp.fire() || io.sbuffer(0).fire(), - // io.sbuffer(1).fire() - // )) - - when(io.sbuffer(0).fire()){ - deqPtrExt := deqPtrExt + Mux(io.sbuffer(1).fire(), 2.U, 1.U) - } - - // load forward query + /** + * load forward query + * + * Check store queue for instructions that is older than the load. + * The response will be valid at the next cycle after req. + */ // check over all lq entries and forward data from the first matched store - (0 until LoadPipelineWidth).map(i => { + for (i <- 0 until LoadPipelineWidth) { io.forward(i).forwardMask := 0.U(8.W).asBools io.forward(i).forwardData := DontCare @@ -226,8 +158,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue // Forward1: Mux(same_flag, range(tail, sqIdx), range(tail, LoadQueueSize)) // Forward2: Mux(same_flag, 0.U, range(0, sqIdx) ) // i.e. forward1 is the target entries with the same flag bits and forward2 otherwise - - val differentFlag = deqPtrExt.flag =/= io.forward(i).sqIdx.flag + val differentFlag = deqPtrExt(0).flag =/= io.forward(i).sqIdx.flag val forwardMask = UIntToMask(io.forward(i).sqIdx.value, StoreQueueSize) val storeWritebackedVec = WireInit(VecInit(Seq.fill(StoreQueueSize)(false.B))) for (j <- 0 until StoreQueueSize) { @@ -236,7 +167,9 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue val needForward1 = Mux(differentFlag, ~tailMask, tailMask ^ forwardMask) & storeWritebackedVec.asUInt val needForward2 = Mux(differentFlag, forwardMask, 0.U(StoreQueueSize.W)) & storeWritebackedVec.asUInt - XSDebug("" + i + " f1 %b f2 %b sqIdx %d pa %x\n", needForward1, needForward2, io.forward(i).sqIdx.asUInt, io.forward(i).paddr) + XSDebug(p"$i f1 ${Binary(needForward1)} f2 ${Binary(needForward2)} " + + p"sqIdx ${io.forward(i).sqIdx} pa ${Hexadecimal(io.forward(i).paddr)}\n" + ) // do real fwd query dataModule.io.forwardQuery( @@ -248,40 +181,19 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue io.forward(i).forwardMask := dataModule.io.forward(i).forwardMask io.forward(i).forwardData := dataModule.io.forward(i).forwardData - }) - - // When store commited, mark it as commited (will not be influenced by redirect), - (0 until CommitWidth).map(i => { - when(storeCommit(i)) { - commited(mcommitIdx(i)) := true.B - XSDebug("store commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc) - } - }) - - (0 until 2).map(i => { - val ptr = (deqPtrExt + i.U).value - val mmio = dataModule.io.rdata(ptr).mmio - io.sbuffer(i).valid := allocated(ptr) && commited(ptr) && !mmio - io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR - io.sbuffer(i).bits.addr := dataModule.io.rdata(ptr).paddr - io.sbuffer(i).bits.data := dataModule.io.rdata(ptr).data - io.sbuffer(i).bits.mask := dataModule.io.rdata(ptr).mask - io.sbuffer(i).bits.meta := DontCare - io.sbuffer(i).bits.meta.tlb_miss := false.B - io.sbuffer(i).bits.meta.uop := DontCare - io.sbuffer(i).bits.meta.mmio := mmio - io.sbuffer(i).bits.meta.mask := dataModule.io.rdata(ptr).mask - - when(io.sbuffer(i).fire()) { - allocated(ptr) := false.B - XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr) - } - }) - - // Memory mapped IO / other uncached operations + } - // setup misc mem access req - // mask / paddr / data can be get from sq.data + /** + * Memory mapped IO / other uncached operations + * + * States: + * (1) writeback from store units: mark as pending + * (2) when they reach ROB's head, they can be sent to uncache channel + * (3) response from uncache channel: mark as datavalid + * (4) writeback to ROB (and other units): mark as writebacked + * (5) ROB commits the instruction: same as normal instructions + */ + //(2) when they reach ROB's head, they can be sent to uncache channel val commitType = io.commits.uop(0).ctrl.commitType io.uncache.req.valid := pending(deqPtr) && allocated(deqPtr) && commitType === CommitType.STORE && @@ -297,30 +209,88 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue io.uncache.req.bits.meta.vaddr := DontCare io.uncache.req.bits.meta.paddr := dataModule.io.rdata(deqPtr).paddr io.uncache.req.bits.meta.uop := uop(deqPtr) - io.uncache.req.bits.meta.mmio := true.B // dataModule.io.rdata(deqPtr).mmio + io.uncache.req.bits.meta.mmio := true.B io.uncache.req.bits.meta.tlb_miss := false.B io.uncache.req.bits.meta.mask := dataModule.io.rdata(deqPtr).mask io.uncache.req.bits.meta.replay := false.B - io.uncache.resp.ready := true.B - when(io.uncache.req.fire()){ pending(deqPtr) := false.B + + XSDebug( + p"uncache req: pc ${Hexadecimal(uop(deqPtr).cf.pc)} " + + p"addr ${Hexadecimal(io.uncache.req.bits.addr)} " + + p"data ${Hexadecimal(io.uncache.req.bits.data)} " + + p"op ${Hexadecimal(io.uncache.req.bits.cmd)} " + + p"mask ${Hexadecimal(io.uncache.req.bits.mask)}\n" + ) } - when(io.uncache.resp.fire()){ - datavalid(deqPtr) := true.B // will be writeback to CDB in the next cycle - // TODO: write back exception info + // (3) response from uncache channel: mark as datavalid + io.uncache.resp.ready := true.B + when (io.uncache.resp.fire()) { + datavalid(deqPtr) := true.B } - when(io.uncache.req.fire()){ - XSDebug("uncache req: pc %x addr %x data %x op %x mask %x\n", - uop(deqPtr).cf.pc, - io.uncache.req.bits.addr, - io.uncache.req.bits.data, - io.uncache.req.bits.cmd, - io.uncache.req.bits.mask - ) + // (4) writeback to ROB (and other units): mark as writebacked + io.mmioStout.valid := allocated(deqPtr) && datavalid(deqPtr) && !writebacked(deqPtr) + io.mmioStout.bits.uop := uop(deqPtr) + io.mmioStout.bits.uop.sqIdx := deqPtrExt(0) + io.mmioStout.bits.uop.cf.exceptionVec := dataModule.io.rdata(deqPtr).exception.asBools + io.mmioStout.bits.data := dataModule.io.rdata(deqPtr).data + io.mmioStout.bits.redirectValid := false.B + io.mmioStout.bits.redirect := DontCare + io.mmioStout.bits.brUpdate := DontCare + io.mmioStout.bits.debug.isMMIO := true.B + io.mmioStout.bits.fflags := DontCare + when (io.mmioStout.fire()) { + writebacked(deqPtr) := true.B + allocated(deqPtr) := false.B + deqPtrExt := VecInit(deqPtrExt.map(_ + 1.U)) + } + + /** + * ROB commits store instructions (mark them as commited) + * + * (1) When store commits, mark it as commited. + * (2) They will not be cancelled and can be sent to lower level. + */ + for (i <- 0 until CommitWidth) { + val storeCommit = !io.commits.isWalk && io.commits.valid(i) && io.commits.uop(i).ctrl.commitType === CommitType.STORE + when (storeCommit) { + commited(io.commits.uop(i).sqIdx.value) := true.B + XSDebug("store commit %d: idx %d %x\n", i.U, io.commits.uop(i).sqIdx.value, io.commits.uop(i).cf.pc) + } + } + + // Commited stores will not be cancelled and can be sent to lower level. + // remove retired insts from sq, add retired store to sbuffer + for (i <- 0 until StorePipelineWidth) { + val ptr = deqPtrExt(i).value + val mmio = dataModule.io.rdata(ptr).mmio + io.sbuffer(i).valid := allocated(ptr) && commited(ptr) && !mmio + io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR + io.sbuffer(i).bits.addr := dataModule.io.rdata(ptr).paddr + io.sbuffer(i).bits.data := dataModule.io.rdata(ptr).data + io.sbuffer(i).bits.mask := dataModule.io.rdata(ptr).mask + io.sbuffer(i).bits.meta := DontCare + io.sbuffer(i).bits.meta.tlb_miss := false.B + io.sbuffer(i).bits.meta.uop := DontCare + io.sbuffer(i).bits.meta.mmio := mmio + io.sbuffer(i).bits.meta.mask := dataModule.io.rdata(ptr).mask + + when (io.sbuffer(i).fire()) { + allocated(ptr) := false.B + XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr) + } + } + // note that sbuffer will not accept req(1) if req(0) is not accepted. + when (Cat(io.sbuffer.map(_.fire())).orR) { + val stepForward = Mux(io.sbuffer(1).fire(), 2.U, 1.U) + deqPtrExt := VecInit(deqPtrExt.map(_ + stepForward)) + when (io.sbuffer(1).fire()) { + assert(io.sbuffer(0).fire()) + } } // Read vaddr for mem exception @@ -331,22 +301,19 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue val needCancel = Wire(Vec(StoreQueueSize, Bool())) for (i <- 0 until StoreQueueSize) { needCancel(i) := uop(i).roqIdx.needFlush(io.brqRedirect) && allocated(i) && !commited(i) - when(needCancel(i)) { - // when(io.brqRedirect.bits.isReplay){ - // datavalid(i) := false.B - // writebacked(i) := false.B - // pending(i) := false.B - // }.otherwise{ + when (needCancel(i)) { allocated(i) := false.B - // } } } - when (io.brqRedirect.valid && io.brqRedirect.bits.isMisPred) { - enqPtrExt := enqPtrExt - PopCount(needCancel) + // we recover the pointers in the next cycle after redirect + val lastCycleRedirectValid = RegNext(io.brqRedirect.valid) + val needCancelCount = PopCount(RegNext(needCancel)) + when (lastCycleRedirectValid) { + enqPtrExt := VecInit(enqPtrExt.map(_ - needCancelCount)) } // debug info - XSDebug("head %d:%d tail %d:%d\n", enqPtrExt.flag, enqPtr, deqPtrExt.flag, deqPtr) + XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt(0).flag, deqPtr) def PrintFlag(flag: Bool, name: String): Unit = { when(flag) { diff --git a/src/main/scala/xiangshan/mem/sbuffer/NewSbuffer.scala b/src/main/scala/xiangshan/mem/sbuffer/NewSbuffer.scala index 372482c8dddb96a21d131e7dacba3f5f8d00f720..82c1b54f692c85d2f8e43b5745914da771859300 100644 --- a/src/main/scala/xiangshan/mem/sbuffer/NewSbuffer.scala +++ b/src/main/scala/xiangshan/mem/sbuffer/NewSbuffer.scala @@ -189,6 +189,9 @@ class NewSbuffer extends XSModule with HasSbufferCst { val updatedSbuffer = io.in.zipWithIndex.foldLeft[Seq[SbufferEntry]](initialSbuffer)(enqSbuffer) val updatedState = updatedSbuffer.map(_._1) val updatedSbufferLine = VecInit(updatedSbuffer.map(_._2)) + when (!io.in(0).ready) { + io.in(1).ready := false.B + } for(i <- 0 until StoreBufferSize){ buffer.write(i.U, updatedSbufferLine(i))