Skip to content

X
XiangShan

OpenXiangShan / XiangShan
11 个月 前同步成功

通知 1183
Star 3914
Fork 526
X
XiangShan

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 5d88c099 编写于 作者: Y Yinan Xu 提交者： GitHub
浏览文件
操作

Merge pull request #333 from RISCVERS/opt-dpq 

dispatch: timing optimizations
上级 fd7be884 4fa135e8
隐藏空白更改
内联 并排
Showing with 407 addition and 492 deletion
src/main/scala/utils/CircularQueuePtr.scala
浏览文件 @ 5d88c099
......@@ -45,6 +45,8 @@ trait HasCircularQueuePtrHelper {
}
final def === (that_ptr: T): Bool = ptr.asUInt()===that_ptr.asUInt()
final def =/= (that_ptr: T): Bool = ptr.asUInt()=/=that_ptr.asUInt()
}
......
src/main/scala/xiangshan/XSCore.scala
浏览文件 @ 5d88c099
......@@ -62,7 +62,6 @@ case class XSCoreParameters
StoreQueueSize: Int = 48,
RoqSize: Int = 192,
dpParams: DispatchParameters = DispatchParameters(
DqEnqWidth = 4,
IntDqSize = 24,
FpDqSize = 24,
LsDqSize = 24,
......
src/main/scala/xiangshan/backend/CtrlBlock.scala
浏览文件 @ 5d88c099
......@@ -12,6 +12,7 @@ import xiangshan.backend.exu._
import xiangshan.backend.exu.Exu.exuConfigs
import xiangshan.backend.regfile.RfReadPort
import xiangshan.backend.roq.{Roq, RoqPtr, RoqCSRIO}
import xiangshan.mem.LsqEnqIO
class CtrlToIntBlockIO extends XSBundle {
val enqIqCtrl = Vec(exuParameters.IntExuCnt, DecoupledIO(new MicroOp))
......@@ -30,11 +31,7 @@ class CtrlToFpBlockIO extends XSBundle {
class CtrlToLsBlockIO extends XSBundle {
val enqIqCtrl = Vec(exuParameters.LsExuCnt, DecoupledIO(new MicroOp))
val enqIqData = Vec(exuParameters.LsExuCnt, Output(new ExuInput))
val enqLsq = new Bundle() {
val canAccept = Input(Bool())
val req = Vec(RenameWidth, ValidIO(new MicroOp))
val resp = Vec(RenameWidth, Input(new LSIdx))
}
val enqLsq = Flipped(new LsqEnqIO)
val redirect = ValidIO(new Redirect)
}
......
src/main/scala/xiangshan/backend/dispatch/Dispatch.scala
浏览文件 @ 5d88c099
......@@ -6,12 +6,12 @@ import xiangshan._
import utils._
import xiangshan.backend.regfile.RfReadPort
import chisel3.ExcitingUtils._
import xiangshan.backend.roq.RoqPtr
import xiangshan.backend.roq.{RoqPtr, RoqEnqIO}
import xiangshan.backend.rename.RenameBypassInfo
import xiangshan.mem.LsqEnqIO
case class DispatchParameters
(
DqEnqWidth: Int,
IntDqSize: Int,
FpDqSize: Int,
LsDqSize: Int,
......@@ -30,19 +30,9 @@ class Dispatch extends XSModule {
// to busytable: set pdest to busy (not ready) when they are dispatched
val allocPregs = Vec(RenameWidth, Output(new ReplayPregReq))
// enq Roq
val enqRoq = new Bundle {
val canAccept = Input(Bool())
val isEmpty = Input(Bool())
val extraWalk = Vec(RenameWidth, Output(Bool()))
val req = Vec(RenameWidth, ValidIO(new MicroOp))
val resp = Vec(RenameWidth, Input(new RoqPtr))
}
val enqRoq = Flipped(new RoqEnqIO)
// enq Lsq
val enqLsq = new Bundle() {
val canAccept = Input(Bool())
val req = Vec(RenameWidth, ValidIO(new MicroOp))
val resp = Vec(RenameWidth, Input(new LSIdx))
}
val enqLsq = Flipped(new LsqEnqIO)
// read regfile
val readIntRf = Vec(NRIntReadPorts, Flipped(new RfReadPort))
val readFpRf = Vec(NRFpReadPorts, Flipped(new RfReadPort))
......@@ -56,9 +46,9 @@ class Dispatch extends XSModule {
})
val dispatch1 = Module(new Dispatch1)
val intDq = Module(new DispatchQueue(dpParams.IntDqSize, dpParams.DqEnqWidth, dpParams.IntDqDeqWidth))
val fpDq = Module(new DispatchQueue(dpParams.FpDqSize, dpParams.DqEnqWidth, dpParams.FpDqDeqWidth))
val lsDq = Module(new DispatchQueue(dpParams.LsDqSize, dpParams.DqEnqWidth, dpParams.LsDqDeqWidth))
val intDq = Module(new DispatchQueue(dpParams.IntDqSize, RenameWidth, dpParams.IntDqDeqWidth))
val fpDq = Module(new DispatchQueue(dpParams.FpDqSize, RenameWidth, dpParams.FpDqDeqWidth))
val lsDq = Module(new DispatchQueue(dpParams.LsDqSize, RenameWidth, dpParams.LsDqDeqWidth))
// pipeline between rename and dispatch
// accepts all at once
......@@ -68,15 +58,12 @@ class Dispatch extends XSModule {
}
// dispatch 1: accept uops from rename and dispatch them to the three dispatch queues
dispatch1.io.redirect <> io.redirect
// dispatch1.io.redirect <> io.redirect
dispatch1.io.renameBypass := RegEnable(io.renameBypass, io.fromRename(0).valid && dispatch1.io.fromRename(0).ready)
dispatch1.io.enqRoq <> io.enqRoq
dispatch1.io.enqLsq <> io.enqLsq
dispatch1.io.toIntDqReady <> intDq.io.enqReady
dispatch1.io.toIntDq <> intDq.io.enq
dispatch1.io.toFpDqReady <> fpDq.io.enqReady
dispatch1.io.toFpDq <> fpDq.io.enq
dispatch1.io.toLsDqReady <> lsDq.io.enqReady
dispatch1.io.toLsDq <> lsDq.io.enq
dispatch1.io.allocPregs <> io.allocPregs
......
src/main/scala/xiangshan/backend/dispatch/Dispatch1.scala
浏览文件 @ 5d88c099
......@@ -5,40 +5,35 @@ import chisel3.util._
import chisel3.ExcitingUtils._
import xiangshan._
import utils.{XSDebug, XSError, XSInfo}
import xiangshan.backend.roq.RoqPtr
import xiangshan.backend.roq.{RoqPtr, RoqEnqIO}
import xiangshan.backend.rename.RenameBypassInfo
import xiangshan.mem.LsqEnqIO
// read rob and enqueue
class Dispatch1 extends XSModule {
val io = IO(new Bundle() {
val redirect = Flipped(ValidIO(new Redirect))
// from rename
val fromRename = Vec(RenameWidth, Flipped(DecoupledIO(new MicroOp)))
val renameBypass = Input(new RenameBypassInfo)
val recv = Output(Vec(RenameWidth, Bool()))
// enq Roq
val enqRoq = new Bundle {
val enqRoq = Flipped(new RoqEnqIO)
// enq Lsq
val enqLsq = Flipped(new LsqEnqIO)
val allocPregs = Vec(RenameWidth, Output(new ReplayPregReq))
// to dispatch queue
val toIntDq = new Bundle {
val canAccept = Input(Bool())
val isEmpty = Input(Bool())
// if set, Roq needs extra walk
val extraWalk = Vec(RenameWidth, Output(Bool()))
val req = Vec(RenameWidth, ValidIO(new MicroOp))
val resp = Vec(RenameWidth, Input(new RoqPtr))
}
// enq Lsq
val enqLsq = new Bundle() {
val toFpDq = new Bundle {
val canAccept = Input(Bool())
val req = Vec(RenameWidth, ValidIO(new MicroOp))
}
val toLsDq = new Bundle {
val canAccept = Input(Bool())
val req = Vec(RenameWidth, ValidIO(new MicroOp))
val resp = Vec(RenameWidth, Input(new LSIdx))
}
val allocPregs = Vec(RenameWidth, Output(new ReplayPregReq))
// to dispatch queue
val toIntDqReady = Input(Bool())
val toIntDq = Vec(dpParams.DqEnqWidth, ValidIO(new MicroOp))
val toFpDqReady = Input(Bool())
val toFpDq = Vec(dpParams.DqEnqWidth, ValidIO(new MicroOp))
val toLsDqReady = Input(Bool())
val toLsDq = Vec(dpParams.DqEnqWidth, ValidIO(new MicroOp))
})
......@@ -50,28 +45,10 @@ class Dispatch1 extends XSModule {
val isFp = VecInit(io.fromRename.map(req => FuType.isFpExu (req.bits.ctrl.fuType)))
val isLs = VecInit(io.fromRename.map(req => FuType.isMemExu(req.bits.ctrl.fuType)))
val isStore = VecInit(io.fromRename.map(req => FuType.isStoreExu(req.bits.ctrl.fuType)))
val isAMO = VecInit(io.fromRename.map(req => req.bits.ctrl.fuType === FuType.mou))
val isBlockBackward = VecInit(io.fromRename.map(_.bits.ctrl.blockBackward))
val isNoSpecExec = VecInit(io.fromRename.map(_.bits.ctrl.noSpecExec))
// generate index mapping
val intIndex = Module(new IndexMapping(RenameWidth, dpParams.DqEnqWidth, false))
val fpIndex = Module(new IndexMapping(RenameWidth, dpParams.DqEnqWidth, false))
val lsIndex = Module(new IndexMapping(RenameWidth, dpParams.DqEnqWidth, false))
for (i <- 0 until RenameWidth) {
intIndex.io.validBits(i) := isInt(i) && io.fromRename(i).valid
fpIndex.io.validBits(i) := isFp(i) && io.fromRename(i).valid
lsIndex.io.validBits(i) := isLs(i) && io.fromRename(i).valid
}
intIndex.io.priority := DontCare
fpIndex.io.priority := DontCare
lsIndex.io.priority := DontCare
if (!env.FPGAPlatform) {
val dispatchNotEmpty = Cat(io.fromRename.map(_.valid)).orR
ExcitingUtils.addSource(!dispatchNotEmpty, "perfCntCondDp1Empty", Perf)
}
/**
* Part 2:
* Update commitType, psrc1, psrc2, psrc3, old_pdest for the uops
......@@ -84,7 +61,7 @@ class Dispatch1 extends XSModule {
val updatedOldPdest = Wire(Vec(RenameWidth, UInt(PhyRegIdxWidth.W)))
for (i <- 0 until RenameWidth) {
updatedCommitType(i) := Cat(isLs(i), isStore(i) | isFp(i))
updatedCommitType(i) := Cat(isLs(i) && !isAMO(i), isStore(i) | isFp(i))
updatedPsrc1(i) := io.fromRename.take(i).map(_.bits.pdest)
.zip(if (i == 0) Seq() else io.renameBypass.lsrc1_bypass(i-1).asBools)
.foldLeft(io.fromRename(i).bits.psrc1) {
......@@ -122,110 +99,85 @@ class Dispatch1 extends XSModule {
* acquire ROQ (all), LSQ (load/store only) and dispatch queue slots
* only set valid when all of them provides enough entries
*/
val redirectValid = io.redirect.valid// && !io.redirect.bits.isReplay
val allResourceReady = io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDqReady && io.toFpDqReady && io.toLsDqReady
val allResourceReady = io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
// Instructions should enter dispatch queues in order.
// When RenameWidth > DqEnqWidth, it's possible that some instructions cannot enter dispatch queue
// because previous instructions cannot enter dispatch queue.
// The reason is that although ROB and LSQ have enough empty slots, dispatch queue has limited enqueue ports.
// Thus, for i >= dpParams.DqEnqWidth, we have to check whether it's previous instructions (and the instruction itself) can enqueue.
// However, since, for instructions with indices less than dpParams.DqEnqWidth,
// they can always enter dispatch queue when ROB and LSQ are ready, we don't need to check whether they can enqueue.
// thisIsBlocked: this instruction is blocked by itself (based on noSpecExec)
// thisCanOut: this instruction can enqueue (based on resource)
// nextCanOut: next instructions can out (based on blockBackward and previous instructions)
// nextCanOut: next instructions can out (based on blockBackward)
// notBlockedByPrevious: previous instructions can enqueue
val thisIsBlocked = VecInit((0 until RenameWidth).map(i => {
// for i > 0, when Roq is empty but dispatch1 have valid instructions to enqueue, it's blocked
if (i > 0) isNoSpecExec(i) && (!io.enqRoq.isEmpty || Cat(io.fromRename.take(i).map(_.valid)).orR)
else isNoSpecExec(i) && !io.enqRoq.isEmpty
}))
val thisCanOut = VecInit((0 until RenameWidth).map(i => {
// For i in [0, DqEnqWidth), they can always enqueue when ROB and LSQ are ready
if (i < dpParams.DqEnqWidth) true.B
else Cat(Seq(intIndex, fpIndex, lsIndex).map(_.io.reverseMapping(i).valid)).orR
}))
val nextCanOut = VecInit((0 until RenameWidth).map(i =>
(thisCanOut(i) && !isNoSpecExec(i) && !isBlockBackward(i)) || !io.fromRename(i).valid
(!isNoSpecExec(i) && !isBlockBackward(i)) || !io.fromRename(i).valid
))
val notBlockedByPrevious = VecInit((0 until RenameWidth).map(i =>
if (i == 0) true.B
else Cat((0 until i).map(j => nextCanOut(j))).andR
))
// for noSpecExec: (roqEmpty || !this.noSpecExec) && !previous.noSpecExec
// For blockBackward:
// this instruction can actually dequeue: 3 conditions
// (1) resources are ready
// (2) previous instructions are ready
val thisCanActualOut = (0 until RenameWidth).map(i => allResourceReady && thisCanOut(i) && !thisIsBlocked(i) && notBlockedByPrevious(i))
val thisCanActualOut = (0 until RenameWidth).map(i => !thisIsBlocked(i) && notBlockedByPrevious(i))
// input for ROQ and LSQ
// note that LSQ needs roqIdx
// (1) LSQ needs roqIdx; (2) DPQ needs roqIdx and lsIdx
val updateUopWithIndex = Wire(Vec(RenameWidth, new MicroOp))
for (i <- 0 until RenameWidth) {
io.enqRoq.extraWalk(i) := io.fromRename(i).valid && !thisCanActualOut(i)
io.enqRoq.req(i).valid := io.fromRename(i).valid && thisCanActualOut(i)
io.enqRoq.needAlloc(i) := io.fromRename(i).valid
io.enqRoq.req(i).valid := io.fromRename(i).valid && thisCanActualOut(i) && io.enqLsq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
io.enqRoq.req(i).bits := updatedUop(i)
XSDebug(io.enqRoq.req(i).valid, p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)} receives nroq ${io.enqRoq.resp(i)}\n")
val shouldEnqLsq = isLs(i) && io.fromRename(i).bits.ctrl.fuType =/= FuType.mou
io.enqLsq.req(i).valid := io.fromRename(i).valid && shouldEnqLsq && !redirectValid && thisCanActualOut(i)
val shouldEnqLsq = isLs(i) && !isAMO(i)
io.enqLsq.needAlloc(i) := io.fromRename(i).valid && shouldEnqLsq
io.enqLsq.req(i).valid := io.fromRename(i).valid && shouldEnqLsq && thisCanActualOut(i) && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
io.enqLsq.req(i).bits := updatedUop(i)
io.enqLsq.req(i).bits.roqIdx := io.enqRoq.resp(i)
XSDebug(io.enqLsq.req(i).valid,
p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)} receives lq ${io.enqLsq.resp(i).lqIdx} sq ${io.enqLsq.resp(i).sqIdx}\n")
XSDebug(io.enqRoq.req(i).valid, p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)} receives nroq ${io.enqRoq.resp(i)}\n")
}
/**
* Part 4:
* append ROQ and LSQ indexed to uop, and send them to dispatch queue
*/
val updateUopWithIndex = Wire(Vec(RenameWidth, new MicroOp))
for (i <- 0 until RenameWidth) {
updateUopWithIndex(i) := updatedUop(i)
updateUopWithIndex(i).roqIdx := io.enqRoq.resp(i)
updateUopWithIndex(i).lqIdx := io.enqLsq.resp(i).lqIdx
updateUopWithIndex(i).sqIdx := io.enqLsq.resp(i).sqIdx
}
// send uops with correct indexes to dispatch queues
// Note that if one of their previous instructions cannot enqueue, they should not enter dispatch queue.
// We use notBlockedByPrevious here since mapping(i).valid implies there's a valid instruction that can enqueue,
// thus we don't need to check thisCanOut.
for (i <- 0 until dpParams.DqEnqWidth) {
io.toIntDq(i).bits := updateUopWithIndex(intIndex.io.mapping(i).bits)
io.toIntDq(i).valid := intIndex.io.mapping(i).valid && allResourceReady &&
!thisIsBlocked(intIndex.io.mapping(i).bits) && notBlockedByPrevious(intIndex.io.mapping(i).bits)
// NOTE: floating point instructions are not noSpecExec currently
// remove commit /**/ when fp instructions are possible to be noSpecExec
io.toFpDq(i).bits := updateUopWithIndex(fpIndex.io.mapping(i).bits)
io.toFpDq(i).valid := fpIndex.io.mapping(i).valid && allResourceReady &&
/*!thisIsBlocked(fpIndex.io.mapping(i).bits) && */notBlockedByPrevious(fpIndex.io.mapping(i).bits)
io.toLsDq(i).bits := updateUopWithIndex(lsIndex.io.mapping(i).bits)
io.toLsDq(i).valid := lsIndex.io.mapping(i).valid && allResourceReady &&
!thisIsBlocked(lsIndex.io.mapping(i).bits) && notBlockedByPrevious(lsIndex.io.mapping(i).bits)
XSDebug(io.toIntDq(i).valid, p"pc 0x${Hexadecimal(io.toIntDq(i).bits.cf.pc)} int index $i\n")
XSDebug(io.toFpDq(i).valid , p"pc 0x${Hexadecimal(io.toFpDq(i).bits.cf.pc )} fp index $i\n")
XSDebug(io.toLsDq(i).valid , p"pc 0x${Hexadecimal(io.toLsDq(i).bits.cf.pc )} ls index $i\n")
// send uops to dispatch queues
// Note that if one of their previous instructions cannot enqueue, they should not enter dispatch queue.
// We use notBlockedByPrevious here.
io.toIntDq.req(i).bits := updateUopWithIndex(i)
io.toIntDq.req(i).valid := io.fromRename(i).valid && isInt(i) && thisCanActualOut(i) &&
io.enqLsq.canAccept && io.enqRoq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
io.toFpDq.req(i).bits := updateUopWithIndex(i)
io.toFpDq.req(i).valid := io.fromRename(i).valid && isFp(i) && thisCanActualOut(i) &&
io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toLsDq.canAccept
io.toLsDq.req(i).bits := updateUopWithIndex(i)
io.toLsDq.req(i).valid := io.fromRename(i).valid && isLs(i) && thisCanActualOut(i) &&
io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept
XSDebug(io.toIntDq.req(i).valid, p"pc 0x${Hexadecimal(io.toIntDq.req(i).bits.cf.pc)} int index $i\n")
XSDebug(io.toFpDq.req(i).valid , p"pc 0x${Hexadecimal(io.toFpDq.req(i).bits.cf.pc )} fp index $i\n")
XSDebug(io.toLsDq.req(i).valid , p"pc 0x${Hexadecimal(io.toLsDq.req(i).bits.cf.pc )} ls index $i\n")
}
/**
* Part 3: send response to rename when dispatch queue accepts the uop
* Part 4: send response to rename when dispatch queue accepts the uop
*/
val readyVector = (0 until RenameWidth).map(i => !io.fromRename(i).valid || io.recv(i))
val hasSpecialInstr = Cat((0 until RenameWidth).map(i => io.fromRename(i).valid && (isBlockBackward(i) || isNoSpecExec(i)))).orR
for (i <- 0 until RenameWidth) {
io.recv(i) := thisCanActualOut(i)
io.fromRename(i).ready := Cat(readyVector).andR()
io.recv(i) := thisCanActualOut(i) && io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
io.fromRename(i).ready := !hasSpecialInstr && io.enqLsq.canAccept && io.enqRoq.canAccept && io.toIntDq.canAccept && io.toFpDq.canAccept && io.toLsDq.canAccept
XSInfo(io.recv(i),
XSInfo(io.recv(i) && io.fromRename(i).valid,
p"pc 0x${Hexadecimal(io.fromRename(i).bits.cf.pc)}, type(${isInt(i)}, ${isFp(i)}, ${isLs(i)}), " +
p"roq ${updateUopWithIndex(i).roqIdx}, lq ${updateUopWithIndex(i).lqIdx}, sq ${updateUopWithIndex(i).sqIdx}, " +
p"(${intIndex.io.reverseMapping(i).bits}, ${fpIndex.io.reverseMapping(i).bits}, ${lsIndex.io.reverseMapping(i).bits})\n"
p"roq ${updateUopWithIndex(i).roqIdx}, lq ${updateUopWithIndex(i).lqIdx}, sq ${updateUopWithIndex(i).sqIdx})\n"
)
io.allocPregs(i).isInt := io.fromRename(i).valid && io.fromRename(i).bits.ctrl.rfWen && (io.fromRename(i).bits.ctrl.ldest =/= 0.U)
......@@ -233,6 +185,8 @@ class Dispatch1 extends XSModule {
io.allocPregs(i).preg := io.fromRename(i).bits.pdest
}
val renameFireCnt = PopCount(io.recv)
val enqFireCnt = PopCount(io.toIntDq.map(_.valid && io.toIntDqReady)) + PopCount(io.toFpDq.map(_.valid && io.toFpDqReady)) + PopCount(io.toLsDq.map(_.valid && io.toLsDqReady))
val enqFireCnt = PopCount(io.toIntDq.req.map(_.valid && io.toIntDq.canAccept)) +
PopCount(io.toFpDq.req.map(_.valid && io.toFpDq.canAccept)) +
PopCount(io.toLsDq.req.map(_.valid && io.toLsDq.canAccept))
XSError(enqFireCnt > renameFireCnt, "enqFireCnt should not be greater than renameFireCnt\n")
}
src/main/scala/xiangshan/backend/dispatch/DispatchQueue.scala
浏览文件 @ 5d88c099
......@@ -7,8 +7,10 @@ import xiangshan._
import xiangshan.backend.roq.RoqPtr
class DispatchQueueIO(enqnum: Int, deqnum: Int) extends XSBundle {
val enq = Vec(enqnum, Flipped(ValidIO(new MicroOp)))
val enqReady = Output(Bool())
val enq = new Bundle {
val canAccept = Output(Bool())
val req = Vec(enqnum, Flipped(ValidIO(new MicroOp)))
}
val deq = Vec(deqnum, DecoupledIO(new MicroOp))
val redirect = Flipped(ValidIO(new Redirect))
override def cloneType: DispatchQueueIO.this.type =
......@@ -27,22 +29,16 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
val stateEntries = RegInit(VecInit(Seq.fill(size)(s_invalid)))
// head: first valid entry (dispatched entry)
val headPtr = RegInit(0.U.asTypeOf(new CircularQueuePtr(size)))
val headPtrMask = UIntToMask(headPtr.value, size)
val headPtr = RegInit(VecInit((0 until deqnum).map(_.U.asTypeOf(new CircularQueuePtr(size)))))
val headPtrMask = UIntToMask(headPtr(0).value, size)
// tail: first invalid entry (free entry)
val tailPtr = RegInit(0.U.asTypeOf(new CircularQueuePtr(size)))
val tailPtrMask = UIntToMask(tailPtr.value, size)
// TODO: make ptr a vector to reduce latency?
// deq: starting from head ptr
val deqIndex = (0 until deqnum).map(i => headPtr + i.U).map(_.value)
// enq: starting from tail ptr
val enqIndex = (0 until enqnum).map(i => tailPtr + i.U).map(_.value)
val tailPtr = RegInit(VecInit((0 until enqnum).map(_.U.asTypeOf(new CircularQueuePtr(size)))))
val tailPtrMask = UIntToMask(tailPtr(0).value, size)
val validEntries = distanceBetween(tailPtr, headPtr)
val validEntries = distanceBetween(tailPtr(0), headPtr(0))
val isTrueEmpty = ~Cat((0 until size).map(i => stateEntries(i) === s_valid)).orR
val canEnqueue = validEntries <= (size - enqnum).U
val canActualEnqueue = canEnqueue && !(io.redirect.valid /*&& !io.redirect.bits.isReplay*/)
val canActualEnqueue = canEnqueue && !io.redirect.valid
/**
* Part 1: update states and uops when enqueue, dequeue, commit, redirect/replay
......@@ -57,25 +53,26 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
* (5) redirect (replay): from s_dispatched to s_valid (re-dispatch)
*/
// enqueue: from s_invalid to s_valid
io.enqReady := canEnqueue
io.enq.canAccept := canEnqueue
for (i <- 0 until enqnum) {
when (io.enq(i).valid && canActualEnqueue) {
uopEntries(enqIndex(i)) := io.enq(i).bits
stateEntries(enqIndex(i)) := s_valid
when (io.enq.req(i).valid && canActualEnqueue) {
val sel = if (i == 0) 0.U else PopCount(io.enq.req.take(i).map(_.valid))
uopEntries(tailPtr(sel).value) := io.enq.req(i).bits
stateEntries(tailPtr(sel).value) := s_valid
}
}
// dequeue: from s_valid to s_dispatched
for (i <- 0 until deqnum) {
when (io.deq(i).fire() && !io.redirect.valid) {
stateEntries(deqIndex(i)) := s_invalid
stateEntries(headPtr(i).value) := s_invalid
XSError(stateEntries(deqIndex(i)) =/= s_valid, "state of the dispatch entry is not s_valid\n")
XSError(stateEntries(headPtr(i).value) =/= s_valid, "state of the dispatch entry is not s_valid\n")
}
}
// redirect: cancel uops currently in the queue
val mispredictionValid = io.redirect.valid //&& io.redirect.bits.isMisPred
val mispredictionValid = io.redirect.valid
val exceptionValid = io.redirect.valid && io.redirect.bits.isException
val flushPipeValid = io.redirect.valid && io.redirect.bits.isFlushPipe
val roqNeedFlush = Wire(Vec(size, Bool()))
......@@ -106,12 +103,15 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
// For dequeue, the first entry should never be s_invalid
// Otherwise, there should be a redirect and tail walks back
// in this case, we set numDeq to 0
!deq.fire() && (if (i == 0) true.B else stateEntries(deqIndex(i)) =/= s_invalid)
!deq.fire() && (if (i == 0) true.B else stateEntries(headPtr(i).value) =/= s_invalid)
} :+ true.B)
val numDeq = Mux(numDeqTry > numDeqFire, numDeqFire, numDeqTry)
// agreement with reservation station: don't dequeue when redirect.valid
val headPtrNext = Mux(mispredictionValid, headPtr, headPtr + numDeq)
headPtr := Mux(exceptionValid, 0.U.asTypeOf(new CircularQueuePtr(size)), headPtrNext)
for (i <- 0 until deqnum) {
headPtr(i) := Mux(exceptionValid,
i.U.asTypeOf(new CircularQueuePtr(size)),
Mux(mispredictionValid, headPtr(i), headPtr(i) + numDeq))
}
// For branch mis-prediction or memory violation replay,
// we delay updating the indices for one clock cycle.
......@@ -124,18 +124,31 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
val flippedFlag = loValidBitVec.orR
val lastOneIndex = size.U - PriorityEncoder(Mux(loValidBitVec.orR, loValidBitVec, hiValidBitVec))
val walkedTailPtr = Wire(new CircularQueuePtr(size))
walkedTailPtr.flag := flippedFlag ^ headPtr.flag
walkedTailPtr.flag := flippedFlag ^ headPtr(0).flag
walkedTailPtr.value := lastOneIndex
// enqueue
val numEnq = Mux(canActualEnqueue, PriorityEncoder(io.enq.map(!_.valid) :+ true.B), 0.U)
XSError(numEnq =/= 0.U && (mispredictionValid || exceptionValid), "should not enqueue when redirect\n")
tailPtr := Mux(exceptionValid,
val numEnq = Mux(io.enq.canAccept, PopCount(io.enq.req.map(_.valid)), 0.U)
tailPtr(0) := Mux(exceptionValid,
0.U.asTypeOf(new CircularQueuePtr(size)),
Mux(lastCycleMisprediction,
Mux(isTrueEmpty, headPtr, walkedTailPtr),
tailPtr + numEnq)
Mux(io.redirect.valid,
tailPtr(0),
Mux(lastCycleMisprediction,
Mux(isTrueEmpty, headPtr(0), walkedTailPtr),
tailPtr(0) + numEnq))
)
val lastCycleException = RegNext(exceptionValid)
val lastLastCycleMisprediction = RegNext(lastCycleMisprediction)
for (i <- 1 until enqnum) {
tailPtr(i) := Mux(exceptionValid,
i.U.asTypeOf(new CircularQueuePtr(size)),
Mux(io.redirect.valid,
tailPtr(i),
Mux(lastLastCycleMisprediction,
tailPtr(0) + i.U,
tailPtr(i) + numEnq))
)
}
/**
......@@ -143,13 +156,13 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
*/
// TODO: remove this when replay moves to roq
for (i <- 0 until deqnum) {
io.deq(i).bits := uopEntries(deqIndex(i))
io.deq(i).bits := uopEntries(headPtr(i).value)
// do not dequeue when io.redirect valid because it may cause dispatchPtr work improperly
io.deq(i).valid := stateEntries(deqIndex(i)) === s_valid && !lastCycleMisprediction// && !io.redirect.valid
io.deq(i).valid := stateEntries(headPtr(i).value) === s_valid && !lastCycleMisprediction
}
// debug: dump dispatch queue states
XSDebug(p"head: $headPtr, tail: $tailPtr\n")
XSDebug(p"head: ${headPtr(0)}, tail: ${tailPtr(0)}\n")
XSDebug(p"state: ")
stateEntries.reverse.foreach { s =>
XSDebug(false, s === s_invalid, "-")
......@@ -158,11 +171,11 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
XSDebug(false, true.B, "\n")
XSDebug(p"ptr: ")
(0 until size).reverse.foreach { i =>
val isPtr = i.U === headPtr.value || i.U === tailPtr.value
val isPtr = i.U === headPtr(0).value || i.U === tailPtr(0).value
XSDebug(false, isPtr, "^")
XSDebug(false, !isPtr, " ")
}
XSDebug(false, true.B, "\n")
XSError(isAfter(headPtr, tailPtr), p"assert greaterOrEqualThan(tailPtr: $tailPtr, headPtr: $headPtr) failed\n")
XSError(isAfter(headPtr(0), tailPtr(0)), p"assert greaterOrEqualThan(tailPtr: ${tailPtr(0)}, headPtr: ${headPtr(0)}) failed\n")
}
src/main/scala/xiangshan/backend/fu/CSR.scala
浏览文件 @ 5d88c099
......@@ -828,7 +828,6 @@ class CSR extends FunctionUnit with HasCSRConst
"RoqWaitFp" -> (0xb11, "perfCntCondRoqWaitFp" ),
"RoqWaitLoad" -> (0xb12, "perfCntCondRoqWaitLoad" ),
"RoqWaitStore"-> (0xb13, "perfCntCondRoqWaitStore"),
"Dp1Empty" -> (0xb14, "perfCntCondDp1Empty" ),
"DTlbReqCnt0" -> (0xb15, "perfCntDtlbReqCnt0" ),
"DTlbReqCnt1" -> (0xb16, "perfCntDtlbReqCnt1" ),
"DTlbReqCnt2" -> (0xb17, "perfCntDtlbReqCnt2" ),
......
src/main/scala/xiangshan/backend/roq/Roq.scala
浏览文件 @ 5d88c099
......@@ -38,17 +38,20 @@ class RoqCSRIO extends XSBundle {
val dirty_fs = Output(Bool())
}
class RoqEnqIO extends XSBundle {
val canAccept = Output(Bool())
val isEmpty = Output(Bool())
// valid vector, for roqIdx gen and walk
val needAlloc = Vec(RenameWidth, Input(Bool()))
val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
val resp = Vec(RenameWidth, Output(new RoqPtr))
}
class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
val io = IO(new Bundle() {
val brqRedirect = Input(Valid(new Redirect))
val memRedirect = Input(Valid(new Redirect))
val enq = new Bundle {
val canAccept = Output(Bool())
val isEmpty = Output(Bool())
val extraWalk = Vec(RenameWidth, Input(Bool()))
val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
val resp = Vec(RenameWidth, Output(new RoqPtr))
}
val enq = new RoqEnqIO
val redirect = Output(Valid(new Redirect))
val exception = Output(new MicroOp)
// exu + brq
......@@ -108,14 +111,7 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
io.roqDeqPtr := deqPtrExt
// common signal
val enqPtrValPlus = Wire(Vec(RenameWidth, UInt(log2Up(RoqSize).W)))
val enqPtrFlagPlus = Wire(Vec(RenameWidth, Bool()))
for (i <- 0 until RenameWidth) {
val offset = PopCount(io.enq.req.map(_.valid).take(i))
val roqIdxExt = enqPtrExt + offset
enqPtrValPlus(i) := roqIdxExt.value
enqPtrFlagPlus(i) := roqIdxExt.flag
}
val enqPtrVec = WireInit(VecInit((0 until RenameWidth).map(i => enqPtrExt + PopCount(io.enq.needAlloc.take(i)))))
val deqPtrExtPlus = Wire(Vec(RenameWidth, UInt(log2Up(RoqSize).W)))
for(i <- 0 until CommitWidth){
......@@ -138,14 +134,9 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
val walkNoSpecExec = io.commits.isWalk && Cat((0 until CommitWidth).map(i => io.commits.valid(i) && io.commits.uop(i).ctrl.noSpecExec)).orR
XSError(state =/= s_extrawalk && walkNoSpecExec, "noSpecExec should not walk\n")
val validDispatch = io.enq.req.map(_.valid)
for (i <- 0 until RenameWidth) {
val offset = PopCount(validDispatch.take(i))
val roqIdxExt = enqPtrExt + offset
val roqIdx = roqIdxExt.value
when(io.enq.req(i).valid) {
microOp(roqIdx) := io.enq.req(i).bits
when(io.enq.req(i).valid && io.enq.canAccept) {
microOp(enqPtrVec(i).value) := io.enq.req(i).bits
when(io.enq.req(i).bits.ctrl.blockBackward) {
hasBlockBackward := true.B
}
......@@ -153,19 +144,18 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
hasNoSpecExec := true.B
}
}
io.enq.resp(i) := roqIdxExt
io.enq.resp(i) := enqPtrVec(i)
}
val validEntries = distanceBetween(enqPtrExt, deqPtrExt)
val firedDispatch = Cat(io.enq.req.map(_.valid))
val firedDispatch = Mux(io.enq.canAccept, PopCount(Cat(io.enq.req.map(_.valid))), 0.U)
io.enq.canAccept := (validEntries <= (RoqSize - RenameWidth).U) && !hasBlockBackward
io.enq.isEmpty := isEmpty
XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(firedDispatch)}\n")
XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n")
val dispatchCnt = PopCount(firedDispatch)
enqPtrExt := enqPtrExt + dispatchCnt
when (firedDispatch.orR) {
XSInfo("dispatched %d insts\n", dispatchCnt)
enqPtrExt := enqPtrExt + firedDispatch
when (firedDispatch =/= 0.U) {
XSInfo("dispatched %d insts\n", firedDispatch)
}
// Writeback
......@@ -224,7 +214,7 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
// extra space is used weh roq has no enough space, but mispredict recovery needs such info to walk regmap
val needExtraSpaceForMPR = WireInit(VecInit(
List.tabulate(RenameWidth)(i => io.brqRedirect.valid && io.enq.extraWalk(i))
List.tabulate(RenameWidth)(i => io.brqRedirect.valid && io.enq.needAlloc(i))
))
val extraSpaceForMPR = Reg(Vec(RenameWidth, new MicroOp))
val usedSpaceForMPR = Reg(Vec(RenameWidth, Bool()))
......@@ -345,12 +335,12 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
// when redirect, walk back roq entries
when(io.brqRedirect.valid){ // TODO: need check if consider exception redirect?
state := s_walk
val nextEnqPtr = (enqPtrExt - 1.U) + dispatchCnt
val walkPtrStart = enqPtrExt - 1.U
walkPtrExt := Mux(state === s_walk,
walkPtrExt - Mux(walkFinished, walkCounter, CommitWidth.U),
Mux(state === s_extrawalk, walkPtrExt, nextEnqPtr))
Mux(state === s_extrawalk, walkPtrExt, walkPtrStart))
// walkTgtExt := io.brqRedirect.bits.roqIdx
val currentWalkPtr = Mux(state === s_walk || state === s_extrawalk, walkPtrExt, nextEnqPtr)
val currentWalkPtr = Mux(state === s_walk || state === s_extrawalk, walkPtrExt, walkPtrStart)
walkCounter := distanceBetween(currentWalkPtr, io.brqRedirect.bits.roqIdx) - Mux(state === s_walk, commitCnt, 0.U)
enqPtrExt := io.brqRedirect.bits.roqIdx + 1.U
}
......@@ -375,8 +365,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
// write
// enqueue logic writes 6 valid
for (i <- 0 until RenameWidth) {
when(io.enq.req(i).fire()){
valid(enqPtrValPlus(i)) := true.B
when(io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid){
valid(enqPtrVec(i).value) := true.B
}
}
// dequeue/walk logic writes 6 valid, dequeue and walk will not happen at the same time
......@@ -412,8 +402,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
// write
// enqueue logic set 6 writebacked to false
for (i <- 0 until RenameWidth) {
when(io.enq.req(i).fire()){
writebacked(enqPtrValPlus(i)) := false.B
when(io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid){
writebacked(enqPtrVec(i).value) := false.B
}
}
// writeback logic set numWbPorts writebacked to true
......@@ -443,8 +433,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
// write: update when enqueue
// enqueue logic set 6 flagBkup at most
for (i <- 0 until RenameWidth) {
when(io.enq.req(i).fire()){
flagBkup(enqPtrValPlus(i)) := enqPtrFlagPlus(i)
when(io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid){
flagBkup(enqPtrVec(i).value) := enqPtrVec(i).flag
}
}
// read: used in rollback logic
......
src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala
浏览文件 @ 5d88c099
......@@ -53,7 +53,7 @@ class LSQueueData(size: Int, nchannel: Int) extends XSModule with HasDCacheParam
val needForward = Input(Vec(nchannel, Vec(2, UInt(size.W))))
val forward = Vec(nchannel, Flipped(new LoadForwardQueryIO))
val rdata = Output(Vec(size, new LsqEntry))
// val debug = new Bundle() {
// val debug_data = Vec(LoadQueueSize, new LsqEntry)
// }
......@@ -76,7 +76,7 @@ class LSQueueData(size: Int, nchannel: Int) extends XSModule with HasDCacheParam
this.needForward(channel)(1) := needForward2
this.forward(channel).paddr := paddr
}
// def refillWrite(ldIdx: Int): Unit = {
// }
// use "this.refill.wen(ldIdx) := true.B" instead
......@@ -229,14 +229,17 @@ class InflightBlockInfo extends XSBundle {
val valid = Bool()
}
class LsqEnqIO extends XSBundle {
val canAccept = Output(Bool())
val needAlloc = Vec(RenameWidth, Input(Bool()))
val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
val resp = Vec(RenameWidth, Output(new LSIdx))
}
// Load / Store Queue Wrapper for XiangShan Out of Order LSU
class LsqWrappper extends XSModule with HasDCacheParameters {
val io = IO(new Bundle() {
val enq = new Bundle() {
val canAccept = Output(Bool())
val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
val resp = Vec(RenameWidth, Output(new LSIdx))
}
val enq = new LsqEnqIO
val brqRedirect = Input(Valid(new Redirect))
val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LsPipelineBundle)))
val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle)))
......@@ -261,14 +264,17 @@ class LsqWrappper extends XSModule with HasDCacheParameters {
io.enq.canAccept := loadQueue.io.enq.canAccept && storeQueue.io.enq.canAccept
for (i <- 0 until RenameWidth) {
val isStore = CommitType.lsInstIsStore(io.enq.req(i).bits.ctrl.commitType)
loadQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i) && !isStore
loadQueue.io.enq.req(i).valid := !isStore && io.enq.req(i).valid
storeQueue.io.enq.req(i).valid := isStore && io.enq.req(i).valid
loadQueue.io.enq.req(i).bits := io.enq.req(i).bits
storeQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i) && isStore
storeQueue.io.enq.req(i).valid := isStore && io.enq.req(i).valid
storeQueue.io.enq.req(i).bits := io.enq.req(i).bits
io.enq.resp(i).lqIdx := loadQueue.io.enq.resp(i)
io.enq.resp(i).sqIdx := storeQueue.io.enq.resp(i)
XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not enqueue LSQ when not")
}
// load queue wiring
......@@ -294,7 +300,7 @@ class LsqWrappper extends XSModule with HasDCacheParameters {
storeQueue.io.exceptionAddr.lsIdx := io.exceptionAddr.lsIdx
storeQueue.io.exceptionAddr.isStore := DontCare
loadQueue.io.forward <> io.forward
loadQueue.io.load_s1 <> io.forward
storeQueue.io.forward <> io.forward // overlap forwardMask & forwardData, DO NOT CHANGE SEQUENCE
io.exceptionAddr.vaddr := Mux(io.exceptionAddr.isStore, storeQueue.io.exceptionAddr.vaddr, loadQueue.io.exceptionAddr.vaddr)
......
src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala
浏览文件 @ 5d88c099
......@@ -23,27 +23,28 @@ object LqPtr extends HasXSParameter {
}
}
class LqEnqIO extends XSBundle {
val canAccept = Output(Bool())
val needAlloc = Vec(RenameWidth, Input(Bool()))
val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
val resp = Vec(RenameWidth, Output(new LqPtr))
}
// Load Queue
class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper {
val io = IO(new Bundle() {
val enq = new Bundle() {
val canAccept = Output(Bool())
val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
val resp = Vec(RenameWidth, Output(new LqPtr))
}
val enq = new LqEnqIO
val brqRedirect = Input(Valid(new Redirect))
val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LsPipelineBundle)))
val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // FIXME: Valid() only
val ldout = Vec(2, DecoupledIO(new ExuOutput)) // writeback load
val forward = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO))
val load_s1 = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO))
val commits = Flipped(new RoqCommitIO)
val rollback = Output(Valid(new Redirect)) // replay now starts from load instead of store
val dcache = new DCacheLineIO
val uncache = new DCacheWordIO
val roqDeqPtr = Input(new RoqPtr)
val exceptionAddr = new ExceptionAddrIO
// val refill = Flipped(Valid(new DCacheLineReq ))
})
val uop = Reg(Vec(LoadQueueSize, new MicroOp))
......@@ -58,11 +59,11 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val listening = Reg(Vec(LoadQueueSize, Bool())) // waiting for refill result
val pending = Reg(Vec(LoadQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of roq
val enqPtrExt = RegInit(0.U.asTypeOf(new LqPtr))
val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new LqPtr))))
val deqPtrExt = RegInit(0.U.asTypeOf(new LqPtr))
val enqPtr = enqPtrExt.value
val enqPtr = enqPtrExt(0).value
val deqPtr = deqPtrExt.value
val sameFlag = enqPtrExt.flag === deqPtrExt.flag
val sameFlag = enqPtrExt(0).flag === deqPtrExt.flag
val isEmpty = enqPtr === deqPtr && sameFlag
val isFull = enqPtr === deqPtr && !sameFlag
val allowIn = !isFull
......@@ -72,19 +73,21 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val deqMask = UIntToMask(deqPtr, LoadQueueSize)
val enqMask = UIntToMask(enqPtr, LoadQueueSize)
val enqDeqMask1 = deqMask ^ enqMask
val enqDeqMask = Mux(sameFlag, enqDeqMask1, ~enqDeqMask1)
// Enqueue at dispatch
val validEntries = distanceBetween(enqPtrExt, deqPtrExt)
/**
* Enqueue at dispatch
*
* Currently, LoadQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth)
*/
val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt)
val firedDispatch = io.enq.req.map(_.valid)
io.enq.canAccept := validEntries <= (LoadQueueSize - RenameWidth).U
XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n")
for (i <- 0 until RenameWidth) {
val offset = if (i == 0) 0.U else PopCount((0 until i).map(firedDispatch(_)))
val lqIdx = enqPtrExt + offset
val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i))
val lqIdx = enqPtrExt(offset)
val index = lqIdx.value
when(io.enq.req(i).valid) {
when (io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid) {
uop(index) := io.enq.req(i).bits
allocated(index) := true.B
datavalid(index) := false.B
......@@ -95,17 +98,28 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
pending(index) := false.B
}
io.enq.resp(i) := lqIdx
XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not valid when not ready\n")
}
when(Cat(firedDispatch).orR) {
enqPtrExt := enqPtrExt + PopCount(firedDispatch)
XSInfo("dispatched %d insts to lq\n", PopCount(firedDispatch))
// when io.brqRedirect.valid, we don't allow eneuque even though it may fire.
when (Cat(firedDispatch).orR && io.enq.canAccept && !io.brqRedirect.valid) {
val enqNumber = PopCount(firedDispatch)
enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber))
XSInfo("dispatched %d insts to lq\n", enqNumber)
}
// writeback load
(0 until LoadPipelineWidth).map(i => {
/**
* Writeback load from load units
*
* Most load instructions writeback to regfile at the same time.
* However,
* (1) For an mmio instruction with exceptions, it writes back to ROB immediately.
* (2) For an mmio instruction without exceptions, it does not write back.
* The mmio instruction will be sent to lower level when it reaches ROB's head.
* After uncache response, it will write back through arbiter with loadUnit.
* (3) For cache misses, it is marked miss and sent to dcache later.
* After cache refills, it will write back through arbiter with loadUnit.
*/
for (i <- 0 until LoadPipelineWidth) {
dataModule.io.wb(i).wen := false.B
when(io.loadIn(i).fire()) {
when(io.loadIn(i).bits.miss) {
......@@ -140,7 +154,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val loadWbIndex = io.loadIn(i).bits.uop.lqIdx.value
datavalid(loadWbIndex) := !io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio
writebacked(loadWbIndex) := !io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio
allocated(loadWbIndex) := !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR
val loadWbData = Wire(new LsqEntry)
loadWbData.paddr := io.loadIn(i).bits.paddr
......@@ -155,13 +168,20 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
dataModule.io.wb(i).wen := true.B
val dcacheMissed = io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio
miss(loadWbIndex) := dcacheMissed
miss(loadWbIndex) := dcacheMissed && !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR
listening(loadWbIndex) := dcacheMissed
pending(loadWbIndex) := io.loadIn(i).bits.mmio
pending(loadWbIndex) := io.loadIn(i).bits.mmio && !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR
}
})
}
// cache miss request
/**
* Cache miss request
*
* (1) writeback: miss
* (2) send to dcache: listing
* (3) dcache response: datavalid
* (4) writeback to ROB: writeback
*/
val inflightReqs = RegInit(VecInit(Seq.fill(cfg.nLoadMissEntries)(0.U.asTypeOf(new InflightBlockInfo))))
val inflightReqFull = inflightReqs.map(req => req.valid).reduce(_&&_)
val reqBlockIndex = PriorityEncoder(~VecInit(inflightReqs.map(req => req.valid)).asUInt)
......@@ -303,21 +323,18 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
}
})
// move tailPtr
// allocatedMask: dequeuePtr can go to the next 1-bit
val allocatedMask = VecInit((0 until LoadQueueSize).map(i => allocated(i) || !enqDeqMask(i)))
// find the first one from deqPtr (deqPtr)
val nextTail1 = getFirstOneWithFlag(allocatedMask, deqMask, deqPtrExt.flag)
val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, enqPtrExt)
deqPtrExt := nextTail
// When load commited, mark it as !allocated, this entry will be recycled later
/**
* Load commits
*
* When load commited, mark it as !allocated and move deqPtrExt forward.
*/
(0 until CommitWidth).map(i => {
when(loadCommit(i)) {
allocated(mcommitIdx(i)) := false.B
XSDebug("load commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc)
}
})
deqPtrExt := deqPtrExt + PopCount(loadCommit)
def getFirstOne(mask: Vec[Bool], startMask: UInt) = {
val length = mask.length
......@@ -326,15 +343,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
PriorityEncoder(Mux(highBitsUint.orR(), highBitsUint, mask.asUInt))
}
def getFirstOneWithFlag(mask: Vec[Bool], startMask: UInt, startFlag: Bool) = {
val length = mask.length
val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
val highBitsUint = Cat(highBits.reverse)
val changeDirection = !highBitsUint.orR()
val index = PriorityEncoder(Mux(!changeDirection, highBitsUint, mask.asUInt))
LqPtr(startFlag ^ changeDirection, index)
}
def getOldestInTwo(valid: Seq[Bool], uop: Seq[MicroOp]) = {
assert(valid.length == uop.length)
assert(valid.length == 2)
......@@ -355,25 +363,25 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
})
}
def rangeMask(start: LqPtr, end: LqPtr): UInt = {
val startMask = (1.U((LoadQueueSize + 1).W) << start.value).asUInt - 1.U
val endMask = (1.U((LoadQueueSize + 1).W) << end.value).asUInt - 1.U
val xorMask = startMask(LoadQueueSize - 1, 0) ^ endMask(LoadQueueSize - 1, 0)
Mux(start.flag === end.flag, xorMask, ~xorMask)
}
// ignore data forward
(0 until LoadPipelineWidth).foreach(i => {
io.forward(i).forwardMask := DontCare
io.forward(i).forwardData := DontCare
})
// store backward query and rollback
/**
* Memory violation detection
*
* When store writes back, it searches LoadQueue for younger load instructions
* with the same load physical address. They loaded wrong data and need re-execution.
*
* Cycle 0: Store Writeback
* Generate match vector for store address with rangeMask(stPtr, enqPtr).
* Besides, load instructions in LoadUnit_S1 and S2 are also checked.
* Cycle 1: Redirect Generation
* There're three possible types of violations. Choose the oldest load.
* Set io.redirect according to the detected violation.
*/
io.load_s1 := DontCare
def detectRollback(i: Int) = {
val startIndex = io.storeIn(i).bits.uop.lqIdx.value
val lqIdxMask = UIntToMask(startIndex, LoadQueueSize)
val xorMask = lqIdxMask ^ enqMask
val sameFlag = io.storeIn(i).bits.uop.lqIdx.flag === enqPtrExt.flag
val sameFlag = io.storeIn(i).bits.uop.lqIdx.flag === enqPtrExt(0).flag
val toEnqPtrMask = Mux(sameFlag, xorMask, ~xorMask)
// check if load already in lq needs to be rolledback
......@@ -405,13 +413,13 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
// check if rollback is needed for load in l1
val l1ViolationVec = RegNext(VecInit((0 until LoadPipelineWidth).map(j => {
io.forward(j).valid && // L1 valid
isAfter(io.forward(j).uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) &&
io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.forward(j).paddr(PAddrBits - 1, 3) &&
(io.storeIn(i).bits.mask & io.forward(j).mask).orR
io.load_s1(j).valid && // L1 valid
isAfter(io.load_s1(j).uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) &&
io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.load_s1(j).paddr(PAddrBits - 1, 3) &&
(io.storeIn(i).bits.mask & io.load_s1(j).mask).orR
})))
val l1Violation = l1ViolationVec.asUInt().orR()
val l1ViolationUop = getOldestInTwo(l1ViolationVec, RegNext(VecInit(io.forward.map(_.uop))))
val l1ViolationUop = getOldestInTwo(l1ViolationVec, RegNext(VecInit(io.load_s1.map(_.uop))))
XSDebug(l1Violation, p"${Binary(Cat(l1ViolationVec))}, $l1ViolationUop\n")
val rollbackValidVec = Seq(lqViolation, wbViolation, l1Violation)
......@@ -465,10 +473,10 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val rollbackSelected = ParallelOperation(rollback, rollbackSel)
val lastCycleRedirect = RegNext(io.brqRedirect)
io.rollback := DontCare
// Note that we use roqIdx - 1.U to flush the load instruction itself.
// Thus, here if last cycle's roqIdx equals to this cycle's roqIdx, it still triggers the redirect.
io.rollback.valid := rollbackSelected.valid && (!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx)) &&
io.rollback.valid := rollbackSelected.valid &&
(!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx)) &&
!(lastCycleRedirect.valid && (lastCycleRedirect.bits.isFlushPipe || lastCycleRedirect.bits.isException))
io.rollback.bits.roqIdx := rollbackSelected.bits.roqIdx - 1.U
......@@ -476,13 +484,18 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
io.rollback.bits.isMisPred := false.B
io.rollback.bits.isException := false.B
io.rollback.bits.isFlushPipe := false.B
io.rollback.bits.pc := DontCare
io.rollback.bits.target := rollbackSelected.bits.cf.pc
io.rollback.bits.brTag := rollbackSelected.bits.brTag
// Memory mapped IO / other uncached operations
when(io.rollback.valid) {
XSDebug("Mem rollback: pc %x roqidx %d\n", io.rollback.bits.pc, io.rollback.bits.roqIdx.asUInt)
}
// setup misc mem access req
// mask / paddr / data can be get from lq.data
/**
* Memory mapped IO / other uncached operations
*
*/
val commitType = io.commits.uop(0).ctrl.commitType
io.uncache.req.valid := pending(deqPtr) && allocated(deqPtr) &&
commitType === CommitType.LOAD &&
......@@ -494,11 +507,11 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
io.uncache.req.bits.data := dataModule.io.rdata(deqPtr).data
io.uncache.req.bits.mask := dataModule.io.rdata(deqPtr).mask
io.uncache.req.bits.meta.id := DontCare // TODO: // FIXME
io.uncache.req.bits.meta.id := DontCare
io.uncache.req.bits.meta.vaddr := DontCare
io.uncache.req.bits.meta.paddr := dataModule.io.rdata(deqPtr).paddr
io.uncache.req.bits.meta.uop := uop(deqPtr)
io.uncache.req.bits.meta.mmio := true.B // dataModule.io.rdata(deqPtr).mmio
io.uncache.req.bits.meta.mmio := true.B
io.uncache.req.bits.meta.tlb_miss := false.B
io.uncache.req.bits.meta.mask := dataModule.io.rdata(deqPtr).mask
io.uncache.req.bits.meta.replay := false.B
......@@ -507,17 +520,7 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
when (io.uncache.req.fire()) {
pending(deqPtr) := false.B
}
dataModule.io.uncache.wen := false.B
when(io.uncache.resp.fire()){
datavalid(deqPtr) := true.B
dataModule.io.uncacheWrite(deqPtr, io.uncache.resp.bits.data(XLEN-1, 0))
dataModule.io.uncache.wen := true.B
// TODO: write back exception info
}
when(io.uncache.req.fire()){
XSDebug("uncache req: pc %x addr %x data %x op %x mask %x\n",
uop(deqPtr).cf.pc,
io.uncache.req.bits.addr,
......@@ -527,7 +530,12 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
)
}
dataModule.io.uncache.wen := false.B
when(io.uncache.resp.fire()){
datavalid(deqPtr) := true.B
dataModule.io.uncacheWrite(deqPtr, io.uncache.resp.bits.data(XLEN-1, 0))
dataModule.io.uncache.wen := true.B
XSDebug("uncache resp: data %x\n", io.dcache.resp.bits.data)
}
......@@ -539,29 +547,19 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val needCancel = Wire(Vec(LoadQueueSize, Bool()))
for (i <- 0 until LoadQueueSize) {
needCancel(i) := uop(i).roqIdx.needFlush(io.brqRedirect) && allocated(i) && !commited(i)
when(needCancel(i)) {
// when(io.brqRedirect.bits.isReplay){
// valid(i) := false.B
// writebacked(i) := false.B
// listening(i) := false.B
// miss(i) := false.B
// pending(i) := false.B
// }.otherwise{
when (needCancel(i)) {
allocated(i) := false.B
// }
}
}
when (io.brqRedirect.valid && io.brqRedirect.bits.isMisPred) {
enqPtrExt := enqPtrExt - PopCount(needCancel)
}
// assert(!io.rollback.valid)
when(io.rollback.valid) {
XSDebug("Mem rollback: pc %x roqidx %d\n", io.rollback.bits.pc, io.rollback.bits.roqIdx.asUInt)
// we recover the pointers in the next cycle after redirect
val needCancelReg = RegNext(needCancel)
when (lastCycleRedirect.valid) {
val cancelCount = PopCount(needCancelReg)
enqPtrExt := VecInit(enqPtrExt.map(_ - cancelCount))
}
// debug info
XSDebug("head %d:%d tail %d:%d\n", enqPtrExt.flag, enqPtr, deqPtrExt.flag, deqPtr)
XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt.flag, deqPtr)
def PrintFlag(flag: Bool, name: String): Unit = {
when(flag) {
......
src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala
浏览文件 @ 5d88c099
......@@ -21,14 +21,17 @@ object SqPtr extends HasXSParameter {
}
}
class SqEnqIO extends XSBundle {
val canAccept = Output(Bool())
val needAlloc = Vec(RenameWidth, Input(Bool()))
val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
val resp = Vec(RenameWidth, Output(new SqPtr))
}
// Store Queue
class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper {
val io = IO(new Bundle() {
val enq = new Bundle() {
val canAccept = Output(Bool())
val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp)))
val resp = Vec(RenameWidth, Output(new SqPtr))
}
val enq = new SqEnqIO
val brqRedirect = Input(Valid(new Redirect))
val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle)))
val sbuffer = Vec(StorePipelineWidth, Decoupled(new DCacheWordReq))
......@@ -51,33 +54,29 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
val commited = Reg(Vec(StoreQueueSize, Bool())) // inst has been commited by roq
val pending = Reg(Vec(StoreQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of roq
val enqPtrExt = RegInit(0.U.asTypeOf(new SqPtr))
val deqPtrExt = RegInit(0.U.asTypeOf(new SqPtr))
val enqPtr = enqPtrExt.value
val deqPtr = deqPtrExt.value
val sameFlag = enqPtrExt.flag === deqPtrExt.flag
val isEmpty = enqPtr === deqPtr && sameFlag
val isFull = enqPtr === deqPtr && !sameFlag
val allowIn = !isFull
val storeCommit = (0 until CommitWidth).map(i => io.commits.valid(i) && !io.commits.isWalk && io.commits.uop(i).ctrl.commitType === CommitType.STORE)
val mcommitIdx = (0 until CommitWidth).map(i => io.commits.uop(i).sqIdx.value)
require(StoreQueueSize > RenameWidth)
val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new SqPtr))))
val deqPtrExt = RegInit(VecInit((0 until StorePipelineWidth).map(_.U.asTypeOf(new SqPtr))))
val enqPtr = enqPtrExt(0).value
val deqPtr = deqPtrExt(0).value
val tailMask = UIntToMask(deqPtr, StoreQueueSize)
val headMask = UIntToMask(enqPtr, StoreQueueSize)
val enqDeqMask1 = tailMask ^ headMask
val enqDeqMask = Mux(sameFlag, enqDeqMask1, ~enqDeqMask1)
// Enqueue at dispatch
val validEntries = distanceBetween(enqPtrExt, deqPtrExt)
/**
* Enqueue at dispatch
*
* Currently, StoreQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth)
*/
val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt(0))
val firedDispatch = io.enq.req.map(_.valid)
io.enq.canAccept := validEntries <= (StoreQueueSize - RenameWidth).U
XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n")
for (i <- 0 until RenameWidth) {
val offset = if (i == 0) 0.U else PopCount((0 until i).map(firedDispatch(_)))
val sqIdx = enqPtrExt + offset
val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i))
val sqIdx = enqPtrExt(offset)
val index = sqIdx.value
when(io.enq.req(i).valid) {
when (io.enq.req(i).valid && io.enq.canAccept && !io.brqRedirect.valid) {
uop(index) := io.enq.req(i).bits
allocated(index) := true.B
datavalid(index) := false.B
......@@ -86,17 +85,28 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
pending(index) := false.B
}
io.enq.resp(i) := sqIdx
XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not valid when not ready\n")
}
when(Cat(firedDispatch).orR) {
enqPtrExt := enqPtrExt + PopCount(firedDispatch)
XSInfo("dispatched %d insts to sq\n", PopCount(firedDispatch))
when (Cat(firedDispatch).orR && io.enq.canAccept && !io.brqRedirect.valid) {
val enqNumber = PopCount(firedDispatch)
enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber))
XSInfo("dispatched %d insts to sq\n", enqNumber)
}
// writeback store
(0 until StorePipelineWidth).map(i => {
/**
* Writeback store from store units
*
* Most store instructions writeback to regfile in the previous cycle.
* However,
* (1) For an mmio instruction with exceptions, we need to mark it as datavalid
* (in this way it will trigger an exception when it reaches ROB's head)
* instead of pending to avoid sending them to lower level.
* (2) For an mmio instruction without exceptions, we mark it as pending.
* When the instruction reaches ROB's head, StoreQueue sends it to uncache channel.
* Upon receiving the response, StoreQueue writes back the instruction
* through arbiter with store units. It will later commit as normal.
*/
for (i <- 0 until StorePipelineWidth) {
dataModule.io.wb(i).wen := false.B
when(io.storeIn(i).fire()) {
val stWbIndex = io.storeIn(i).bits.uop.sqIdx.value
......@@ -129,94 +139,16 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
io.storeIn(i).bits.uop.cf.exceptionVec.asUInt
)
}
})
def getFirstOne(mask: Vec[Bool], startMask: UInt) = {
val length = mask.length
val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
val highBitsUint = Cat(highBits.reverse)
PriorityEncoder(Mux(highBitsUint.orR(), highBitsUint, mask.asUInt))
}
def getFirstOneWithFlag(mask: Vec[Bool], startMask: UInt, startFlag: Bool) = {
val length = mask.length
val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
val highBitsUint = Cat(highBits.reverse)
val changeDirection = !highBitsUint.orR()
val index = PriorityEncoder(Mux(!changeDirection, highBitsUint, mask.asUInt))
SqPtr(startFlag ^ changeDirection, index)
}
def selectFirstTwo(valid: Vec[Bool], startMask: UInt) = {
val selVec = Wire(Vec(2, UInt(log2Up(StoreQueueSize).W)))
val selValid = Wire(Vec(2, Bool()))
selVec(0) := getFirstOne(valid, startMask)
val firstSelMask = UIntToOH(selVec(0))
val secondSelVec = VecInit((0 until valid.length).map(i => valid(i) && !firstSelMask(i)))
selVec(1) := getFirstOne(secondSelVec, startMask)
selValid(0) := Cat(valid).orR
selValid(1) := Cat(secondSelVec).orR
(selValid, selVec)
}
def selectFirstTwoRoughly(valid: Vec[Bool]) = {
// TODO: do not select according to seq, just select 2 valid bit randomly
val firstSelVec = valid
val notFirstVec = Wire(Vec(valid.length, Bool()))
(0 until valid.length).map(i =>
notFirstVec(i) := (if(i != 0) { valid(i) || !notFirstVec(i) } else { false.B })
)
val secondSelVec = VecInit((0 until valid.length).map(i => valid(i) && !notFirstVec(i)))
val selVec = Wire(Vec(2, UInt(log2Up(valid.length).W)))
val selValid = Wire(Vec(2, Bool()))
selVec(0) := PriorityEncoder(firstSelVec)
selVec(1) := PriorityEncoder(secondSelVec)
selValid(0) := Cat(firstSelVec).orR
selValid(1) := Cat(secondSelVec).orR
(selValid, selVec)
}
// writeback finished mmio store
io.mmioStout.bits.uop := uop(deqPtr)
io.mmioStout.bits.uop.sqIdx := deqPtrExt
io.mmioStout.bits.uop.cf.exceptionVec := dataModule.io.rdata(deqPtr).exception.asBools
io.mmioStout.bits.data := dataModule.io.rdata(deqPtr).data
io.mmioStout.bits.redirectValid := false.B
io.mmioStout.bits.redirect := DontCare
io.mmioStout.bits.brUpdate := DontCare
io.mmioStout.bits.debug.isMMIO := true.B
io.mmioStout.bits.fflags := DontCare
io.mmioStout.valid := allocated(deqPtr) && datavalid(deqPtr) && !writebacked(deqPtr) // finished mmio store
when(io.mmioStout.fire()) {
writebacked(deqPtr) := true.B
allocated(deqPtr) := false.B // potential opt: move deqPtr immediately
}
// remove retired insts from sq, add retired store to sbuffer
// move tailPtr
// TailPtr slow recovery: recycle bubbles in store queue
// allocatedMask: dequeuePtr can go to the next 1-bit
val allocatedMask = VecInit((0 until StoreQueueSize).map(i => allocated(i) || !enqDeqMask(i)))
// find the first one from deqPtr (deqPtr)
val nextTail1 = getFirstOneWithFlag(allocatedMask, tailMask, deqPtrExt.flag)
val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, enqPtrExt)
deqPtrExt := nextTail
// TailPtr fast recovery
// val tailRecycle = VecInit(List(
// io.uncache.resp.fire() || io.sbuffer(0).fire(),
// io.sbuffer(1).fire()
// ))
when(io.sbuffer(0).fire()){
deqPtrExt := deqPtrExt + Mux(io.sbuffer(1).fire(), 2.U, 1.U)
}
// load forward query
/**
* load forward query
*
* Check store queue for instructions that is older than the load.
* The response will be valid at the next cycle after req.
*/
// check over all lq entries and forward data from the first matched store
(0 until LoadPipelineWidth).map(i => {
for (i <- 0 until LoadPipelineWidth) {
io.forward(i).forwardMask := 0.U(8.W).asBools
io.forward(i).forwardData := DontCare
......@@ -226,8 +158,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
// Forward1: Mux(same_flag, range(tail, sqIdx), range(tail, LoadQueueSize))
// Forward2: Mux(same_flag, 0.U, range(0, sqIdx) )
// i.e. forward1 is the target entries with the same flag bits and forward2 otherwise
val differentFlag = deqPtrExt.flag =/= io.forward(i).sqIdx.flag
val differentFlag = deqPtrExt(0).flag =/= io.forward(i).sqIdx.flag
val forwardMask = UIntToMask(io.forward(i).sqIdx.value, StoreQueueSize)
val storeWritebackedVec = WireInit(VecInit(Seq.fill(StoreQueueSize)(false.B)))
for (j <- 0 until StoreQueueSize) {
......@@ -236,7 +167,9 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
val needForward1 = Mux(differentFlag, ~tailMask, tailMask ^ forwardMask) & storeWritebackedVec.asUInt
val needForward2 = Mux(differentFlag, forwardMask, 0.U(StoreQueueSize.W)) & storeWritebackedVec.asUInt
XSDebug("" + i + " f1 %b f2 %b sqIdx %d pa %x\n", needForward1, needForward2, io.forward(i).sqIdx.asUInt, io.forward(i).paddr)
XSDebug(p"$i f1 ${Binary(needForward1)} f2 ${Binary(needForward2)} " +
p"sqIdx ${io.forward(i).sqIdx} pa ${Hexadecimal(io.forward(i).paddr)}\n"
)
// do real fwd query
dataModule.io.forwardQuery(
......@@ -248,40 +181,19 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
io.forward(i).forwardMask := dataModule.io.forward(i).forwardMask
io.forward(i).forwardData := dataModule.io.forward(i).forwardData
})
// When store commited, mark it as commited (will not be influenced by redirect),
(0 until CommitWidth).map(i => {
when(storeCommit(i)) {
commited(mcommitIdx(i)) := true.B
XSDebug("store commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc)
}
})
(0 until 2).map(i => {
val ptr = (deqPtrExt + i.U).value
val mmio = dataModule.io.rdata(ptr).mmio
io.sbuffer(i).valid := allocated(ptr) && commited(ptr) && !mmio
io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR
io.sbuffer(i).bits.addr := dataModule.io.rdata(ptr).paddr
io.sbuffer(i).bits.data := dataModule.io.rdata(ptr).data
io.sbuffer(i).bits.mask := dataModule.io.rdata(ptr).mask
io.sbuffer(i).bits.meta := DontCare
io.sbuffer(i).bits.meta.tlb_miss := false.B
io.sbuffer(i).bits.meta.uop := DontCare
io.sbuffer(i).bits.meta.mmio := mmio
io.sbuffer(i).bits.meta.mask := dataModule.io.rdata(ptr).mask
when(io.sbuffer(i).fire()) {
allocated(ptr) := false.B
XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr)
}
})
// Memory mapped IO / other uncached operations
}
// setup misc mem access req
// mask / paddr / data can be get from sq.data
/**
* Memory mapped IO / other uncached operations
*
* States:
* (1) writeback from store units: mark as pending
* (2) when they reach ROB's head, they can be sent to uncache channel
* (3) response from uncache channel: mark as datavalid
* (4) writeback to ROB (and other units): mark as writebacked
* (5) ROB commits the instruction: same as normal instructions
*/
//(2) when they reach ROB's head, they can be sent to uncache channel
val commitType = io.commits.uop(0).ctrl.commitType
io.uncache.req.valid := pending(deqPtr) && allocated(deqPtr) &&
commitType === CommitType.STORE &&
......@@ -297,30 +209,88 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
io.uncache.req.bits.meta.vaddr := DontCare
io.uncache.req.bits.meta.paddr := dataModule.io.rdata(deqPtr).paddr
io.uncache.req.bits.meta.uop := uop(deqPtr)
io.uncache.req.bits.meta.mmio := true.B // dataModule.io.rdata(deqPtr).mmio
io.uncache.req.bits.meta.mmio := true.B
io.uncache.req.bits.meta.tlb_miss := false.B
io.uncache.req.bits.meta.mask := dataModule.io.rdata(deqPtr).mask
io.uncache.req.bits.meta.replay := false.B
io.uncache.resp.ready := true.B
when(io.uncache.req.fire()){
pending(deqPtr) := false.B
XSDebug(
p"uncache req: pc ${Hexadecimal(uop(deqPtr).cf.pc)} " +
p"addr ${Hexadecimal(io.uncache.req.bits.addr)} " +
p"data ${Hexadecimal(io.uncache.req.bits.data)} " +
p"op ${Hexadecimal(io.uncache.req.bits.cmd)} " +
p"mask ${Hexadecimal(io.uncache.req.bits.mask)}\n"
)
}
when(io.uncache.resp.fire()){
datavalid(deqPtr) := true.B // will be writeback to CDB in the next cycle
// TODO: write back exception info
// (3) response from uncache channel: mark as datavalid
io.uncache.resp.ready := true.B
when (io.uncache.resp.fire()) {
datavalid(deqPtr) := true.B
}
when(io.uncache.req.fire()){
XSDebug("uncache req: pc %x addr %x data %x op %x mask %x\n",
uop(deqPtr).cf.pc,
io.uncache.req.bits.addr,
io.uncache.req.bits.data,
io.uncache.req.bits.cmd,
io.uncache.req.bits.mask
)
// (4) writeback to ROB (and other units): mark as writebacked
io.mmioStout.valid := allocated(deqPtr) && datavalid(deqPtr) && !writebacked(deqPtr)
io.mmioStout.bits.uop := uop(deqPtr)
io.mmioStout.bits.uop.sqIdx := deqPtrExt(0)
io.mmioStout.bits.uop.cf.exceptionVec := dataModule.io.rdata(deqPtr).exception.asBools
io.mmioStout.bits.data := dataModule.io.rdata(deqPtr).data
io.mmioStout.bits.redirectValid := false.B
io.mmioStout.bits.redirect := DontCare
io.mmioStout.bits.brUpdate := DontCare
io.mmioStout.bits.debug.isMMIO := true.B
io.mmioStout.bits.fflags := DontCare
when (io.mmioStout.fire()) {
writebacked(deqPtr) := true.B
allocated(deqPtr) := false.B
deqPtrExt := VecInit(deqPtrExt.map(_ + 1.U))
}
/**
* ROB commits store instructions (mark them as commited)
*
* (1) When store commits, mark it as commited.
* (2) They will not be cancelled and can be sent to lower level.
*/
for (i <- 0 until CommitWidth) {
val storeCommit = !io.commits.isWalk && io.commits.valid(i) && io.commits.uop(i).ctrl.commitType === CommitType.STORE
when (storeCommit) {
commited(io.commits.uop(i).sqIdx.value) := true.B
XSDebug("store commit %d: idx %d %x\n", i.U, io.commits.uop(i).sqIdx.value, io.commits.uop(i).cf.pc)
}
}
// Commited stores will not be cancelled and can be sent to lower level.
// remove retired insts from sq, add retired store to sbuffer
for (i <- 0 until StorePipelineWidth) {
val ptr = deqPtrExt(i).value
val mmio = dataModule.io.rdata(ptr).mmio
io.sbuffer(i).valid := allocated(ptr) && commited(ptr) && !mmio
io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR
io.sbuffer(i).bits.addr := dataModule.io.rdata(ptr).paddr
io.sbuffer(i).bits.data := dataModule.io.rdata(ptr).data
io.sbuffer(i).bits.mask := dataModule.io.rdata(ptr).mask
io.sbuffer(i).bits.meta := DontCare
io.sbuffer(i).bits.meta.tlb_miss := false.B
io.sbuffer(i).bits.meta.uop := DontCare
io.sbuffer(i).bits.meta.mmio := mmio
io.sbuffer(i).bits.meta.mask := dataModule.io.rdata(ptr).mask
when (io.sbuffer(i).fire()) {
allocated(ptr) := false.B
XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr)
}
}
// note that sbuffer will not accept req(1) if req(0) is not accepted.
when (Cat(io.sbuffer.map(_.fire())).orR) {
val stepForward = Mux(io.sbuffer(1).fire(), 2.U, 1.U)
deqPtrExt := VecInit(deqPtrExt.map(_ + stepForward))
when (io.sbuffer(1).fire()) {
assert(io.sbuffer(0).fire())
}
}
// Read vaddr for mem exception
......@@ -331,22 +301,19 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
val needCancel = Wire(Vec(StoreQueueSize, Bool()))
for (i <- 0 until StoreQueueSize) {
needCancel(i) := uop(i).roqIdx.needFlush(io.brqRedirect) && allocated(i) && !commited(i)
when(needCancel(i)) {
// when(io.brqRedirect.bits.isReplay){
// datavalid(i) := false.B
// writebacked(i) := false.B
// pending(i) := false.B
// }.otherwise{
when (needCancel(i)) {
allocated(i) := false.B
// }
}
}
when (io.brqRedirect.valid && io.brqRedirect.bits.isMisPred) {
enqPtrExt := enqPtrExt - PopCount(needCancel)
// we recover the pointers in the next cycle after redirect
val lastCycleRedirectValid = RegNext(io.brqRedirect.valid)
val needCancelCount = PopCount(RegNext(needCancel))
when (lastCycleRedirectValid) {
enqPtrExt := VecInit(enqPtrExt.map(_ - needCancelCount))
}
// debug info
XSDebug("head %d:%d tail %d:%d\n", enqPtrExt.flag, enqPtr, deqPtrExt.flag, deqPtr)
XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt(0).flag, deqPtr)
def PrintFlag(flag: Bool, name: String): Unit = {
when(flag) {
......
src/main/scala/xiangshan/mem/sbuffer/NewSbuffer.scala
浏览文件 @ 5d88c099
......@@ -189,6 +189,9 @@ class NewSbuffer extends XSModule with HasSbufferCst {
val updatedSbuffer = io.in.zipWithIndex.foldLeft[Seq[SbufferEntry]](initialSbuffer)(enqSbuffer)
val updatedState = updatedSbuffer.map(_._1)
val updatedSbufferLine = VecInit(updatedSbuffer.map(_._2))
when (!io.in(0).ready) {
io.in(1).ready := false.B
}
for(i <- 0 until StoreBufferSize){
buffer.write(i.U, updatedSbufferLine(i))
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或