Skip to content

X
XiangShan

OpenXiangShan / XiangShan
12 个月 前同步成功

通知 1183
Star 3914
Fork 526
X
XiangShan

体验新版 GitCode，发现更多精彩内容 >>

提交 51556e23 编写于 作者: Y Yinan Xu
浏览文件
操作

Merge remote-tracking branch 'origin/opt-lsq' into opt-dpq

上级 cc449d02 be69d1ab
隐藏空白更改
内联 并排
Showing with 228 addition and 260 deletion
src/main/scala/utils/CircularQueuePtr.scala
浏览文件 @ 51556e23
......@@ -45,6 +45,8 @@ trait HasCircularQueuePtrHelper {
}
final def === (that_ptr: T): Bool = ptr.asUInt()===that_ptr.asUInt()
final def =/= (that_ptr: T): Bool = ptr.asUInt()=/=that_ptr.asUInt()
}
......
src/main/scala/xiangshan/mem/lsqueue/LSQWrapper.scala
浏览文件 @ 51556e23
......@@ -294,7 +294,7 @@ class LsqWrappper extends XSModule with HasDCacheParameters {
storeQueue.io.exceptionAddr.lsIdx := io.exceptionAddr.lsIdx
storeQueue.io.exceptionAddr.isStore := DontCare
loadQueue.io.forward <> io.forward
loadQueue.io.load_s1 <> io.forward
storeQueue.io.forward <> io.forward // overlap forwardMask & forwardData, DO NOT CHANGE SEQUENCE
io.exceptionAddr.vaddr := Mux(io.exceptionAddr.isStore, storeQueue.io.exceptionAddr.vaddr, loadQueue.io.exceptionAddr.vaddr)
......
src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala
浏览文件 @ 51556e23
......@@ -36,14 +36,13 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LsPipelineBundle)))
val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // FIXME: Valid() only
val ldout = Vec(2, DecoupledIO(new ExuOutput)) // writeback load
val forward = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO))
val load_s1 = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO))
val commits = Flipped(new RoqCommitIO)
val rollback = Output(Valid(new Redirect)) // replay now starts from load instead of store
val dcache = new DCacheLineIO
val uncache = new DCacheWordIO
val roqDeqPtr = Input(new RoqPtr)
val exceptionAddr = new ExceptionAddrIO
// val refill = Flipped(Valid(new DCacheLineReq ))
})
val uop = Reg(Vec(LoadQueueSize, new MicroOp))
......@@ -58,11 +57,11 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val listening = Reg(Vec(LoadQueueSize, Bool())) // waiting for refill result
val pending = Reg(Vec(LoadQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of roq
val enqPtrExt = RegInit(0.U.asTypeOf(new LqPtr))
val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new LqPtr))))
val deqPtrExt = RegInit(0.U.asTypeOf(new LqPtr))
val enqPtr = enqPtrExt.value
val enqPtr = enqPtrExt(0).value
val deqPtr = deqPtrExt.value
val sameFlag = enqPtrExt.flag === deqPtrExt.flag
val sameFlag = enqPtrExt(0).flag === deqPtrExt.flag
val isEmpty = enqPtr === deqPtr && sameFlag
val isFull = enqPtr === deqPtr && !sameFlag
val allowIn = !isFull
......@@ -72,17 +71,19 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val deqMask = UIntToMask(deqPtr, LoadQueueSize)
val enqMask = UIntToMask(enqPtr, LoadQueueSize)
val enqDeqMask1 = deqMask ^ enqMask
val enqDeqMask = Mux(sameFlag, enqDeqMask1, ~enqDeqMask1)
// Enqueue at dispatch
val validEntries = distanceBetween(enqPtrExt, deqPtrExt)
/**
* Enqueue at dispatch
*
* Currently, LoadQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth)
*/
val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt)
val firedDispatch = io.enq.req.map(_.valid)
io.enq.canAccept := validEntries <= (LoadQueueSize - RenameWidth).U
XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n")
for (i <- 0 until RenameWidth) {
val offset = if (i == 0) 0.U else PopCount((0 until i).map(firedDispatch(_)))
val lqIdx = enqPtrExt + offset
val lqIdx = enqPtrExt(offset)
val index = lqIdx.value
when(io.enq.req(i).valid) {
uop(index) := io.enq.req(i).bits
......@@ -99,13 +100,26 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not valid when not ready\n")
}
when(Cat(firedDispatch).orR) {
enqPtrExt := enqPtrExt + PopCount(firedDispatch)
XSInfo("dispatched %d insts to lq\n", PopCount(firedDispatch))
// when io.brqRedirect.valid, we don't allow eneuque even though it may fire.
when (Cat(firedDispatch).orR && !io.brqRedirect.valid) {
val enqNumber = PopCount(firedDispatch)
enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber))
XSInfo("dispatched %d insts to lq\n", enqNumber)
}
// writeback load
(0 until LoadPipelineWidth).map(i => {
/**
* Writeback load from load units
*
* Most load instructions writeback to regfile at the same time.
* However,
* (1) For an mmio instruction with exceptions, it writes back to ROB immediately.
* (2) For an mmio instruction without exceptions, it does not write back.
* The mmio instruction will be sent to lower level when it reaches ROB's head.
* After uncache response, it will write back through arbiter with loadUnit.
* (3) For cache misses, it is marked miss and sent to dcache later.
* After cache refills, it will write back through arbiter with loadUnit.
*/
for (i <- 0 until LoadPipelineWidth) {
dataModule.io.wb(i).wen := false.B
when(io.loadIn(i).fire()) {
when(io.loadIn(i).bits.miss) {
......@@ -140,7 +154,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val loadWbIndex = io.loadIn(i).bits.uop.lqIdx.value
datavalid(loadWbIndex) := !io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio
writebacked(loadWbIndex) := !io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio
allocated(loadWbIndex) := !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR
val loadWbData = Wire(new LsqEntry)
loadWbData.paddr := io.loadIn(i).bits.paddr
......@@ -155,13 +168,20 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
dataModule.io.wb(i).wen := true.B
val dcacheMissed = io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio
miss(loadWbIndex) := dcacheMissed
miss(loadWbIndex) := dcacheMissed && !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR
listening(loadWbIndex) := dcacheMissed
pending(loadWbIndex) := io.loadIn(i).bits.mmio
pending(loadWbIndex) := io.loadIn(i).bits.mmio && !io.loadIn(i).bits.uop.cf.exceptionVec.asUInt.orR
}
})
}
// cache miss request
/**
* Cache miss request
*
* (1) writeback: miss
* (2) send to dcache: listing
* (3) dcache response: datavalid
* (4) writeback to ROB: writeback
*/
val inflightReqs = RegInit(VecInit(Seq.fill(cfg.nLoadMissEntries)(0.U.asTypeOf(new InflightBlockInfo))))
val inflightReqFull = inflightReqs.map(req => req.valid).reduce(_&&_)
val reqBlockIndex = PriorityEncoder(~VecInit(inflightReqs.map(req => req.valid)).asUInt)
......@@ -303,21 +323,18 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
}
})
// move tailPtr
// allocatedMask: dequeuePtr can go to the next 1-bit
val allocatedMask = VecInit((0 until LoadQueueSize).map(i => allocated(i) || !enqDeqMask(i)))
// find the first one from deqPtr (deqPtr)
val nextTail1 = getFirstOneWithFlag(allocatedMask, deqMask, deqPtrExt.flag)
val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, enqPtrExt)
deqPtrExt := nextTail
// When load commited, mark it as !allocated, this entry will be recycled later
/**
* Load commits
*
* When load commited, mark it as !allocated and move deqPtrExt forward.
*/
(0 until CommitWidth).map(i => {
when(loadCommit(i)) {
allocated(mcommitIdx(i)) := false.B
XSDebug("load commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc)
}
})
deqPtrExt := deqPtrExt + PopCount(loadCommit)
def getFirstOne(mask: Vec[Bool], startMask: UInt) = {
val length = mask.length
......@@ -326,15 +343,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
PriorityEncoder(Mux(highBitsUint.orR(), highBitsUint, mask.asUInt))
}
def getFirstOneWithFlag(mask: Vec[Bool], startMask: UInt, startFlag: Bool) = {
val length = mask.length
val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
val highBitsUint = Cat(highBits.reverse)
val changeDirection = !highBitsUint.orR()
val index = PriorityEncoder(Mux(!changeDirection, highBitsUint, mask.asUInt))
LqPtr(startFlag ^ changeDirection, index)
}
def getOldestInTwo(valid: Seq[Bool], uop: Seq[MicroOp]) = {
assert(valid.length == uop.length)
assert(valid.length == 2)
......@@ -355,25 +363,25 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
})
}
def rangeMask(start: LqPtr, end: LqPtr): UInt = {
val startMask = (1.U((LoadQueueSize + 1).W) << start.value).asUInt - 1.U
val endMask = (1.U((LoadQueueSize + 1).W) << end.value).asUInt - 1.U
val xorMask = startMask(LoadQueueSize - 1, 0) ^ endMask(LoadQueueSize - 1, 0)
Mux(start.flag === end.flag, xorMask, ~xorMask)
}
// ignore data forward
(0 until LoadPipelineWidth).foreach(i => {
io.forward(i).forwardMask := DontCare
io.forward(i).forwardData := DontCare
})
// store backward query and rollback
/**
* Memory violation detection
*
* When store writes back, it searches LoadQueue for younger load instructions
* with the same load physical address. They loaded wrong data and need re-execution.
*
* Cycle 0: Store Writeback
* Generate match vector for store address with rangeMask(stPtr, enqPtr).
* Besides, load instructions in LoadUnit_S1 and S2 are also checked.
* Cycle 1: Redirect Generation
* There're three possible types of violations. Choose the oldest load.
* Set io.redirect according to the detected violation.
*/
io.load_s1 := DontCare
def detectRollback(i: Int) = {
val startIndex = io.storeIn(i).bits.uop.lqIdx.value
val lqIdxMask = UIntToMask(startIndex, LoadQueueSize)
val xorMask = lqIdxMask ^ enqMask
val sameFlag = io.storeIn(i).bits.uop.lqIdx.flag === enqPtrExt.flag
val sameFlag = io.storeIn(i).bits.uop.lqIdx.flag === enqPtrExt(0).flag
val toEnqPtrMask = Mux(sameFlag, xorMask, ~xorMask)
// check if load already in lq needs to be rolledback
......@@ -405,13 +413,13 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
// check if rollback is needed for load in l1
val l1ViolationVec = RegNext(VecInit((0 until LoadPipelineWidth).map(j => {
io.forward(j).valid && // L1 valid
isAfter(io.forward(j).uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) &&
io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.forward(j).paddr(PAddrBits - 1, 3) &&
(io.storeIn(i).bits.mask & io.forward(j).mask).orR
io.load_s1(j).valid && // L1 valid
isAfter(io.load_s1(j).uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) &&
io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.load_s1(j).paddr(PAddrBits - 1, 3) &&
(io.storeIn(i).bits.mask & io.load_s1(j).mask).orR
})))
val l1Violation = l1ViolationVec.asUInt().orR()
val l1ViolationUop = getOldestInTwo(l1ViolationVec, RegNext(VecInit(io.forward.map(_.uop))))
val l1ViolationUop = getOldestInTwo(l1ViolationVec, RegNext(VecInit(io.load_s1.map(_.uop))))
XSDebug(l1Violation, p"${Binary(Cat(l1ViolationVec))}, $l1ViolationUop\n")
val rollbackValidVec = Seq(lqViolation, wbViolation, l1Violation)
......@@ -465,23 +473,27 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val rollbackSelected = ParallelOperation(rollback, rollbackSel)
val lastCycleRedirect = RegNext(io.brqRedirect)
io.rollback := DontCare
// Note that we use roqIdx - 1.U to flush the load instruction itself.
// Thus, here if last cycle's roqIdx equals to this cycle's roqIdx, it still triggers the redirect.
io.rollback.valid := rollbackSelected.valid && (!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx))
io.rollback.valid := rollbackSelected.valid &&
(!lastCycleRedirect.valid || !isAfter(rollbackSelected.bits.roqIdx, lastCycleRedirect.bits.roqIdx))
io.rollback.bits.roqIdx := rollbackSelected.bits.roqIdx - 1.U
io.rollback.bits.isReplay := true.B
io.rollback.bits.isMisPred := false.B
io.rollback.bits.isException := false.B
io.rollback.bits.isFlushPipe := false.B
io.rollback.bits.pc := DontCare
io.rollback.bits.target := rollbackSelected.bits.cf.pc
io.rollback.bits.brTag := rollbackSelected.bits.brTag
// Memory mapped IO / other uncached operations
when(io.rollback.valid) {
XSDebug("Mem rollback: pc %x roqidx %d\n", io.rollback.bits.pc, io.rollback.bits.roqIdx.asUInt)
}
// setup misc mem access req
// mask / paddr / data can be get from lq.data
/**
* Memory mapped IO / other uncached operations
*
*/
val commitType = io.commits.uop(0).ctrl.commitType
io.uncache.req.valid := pending(deqPtr) && allocated(deqPtr) &&
commitType === CommitType.LOAD &&
......@@ -493,11 +505,11 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
io.uncache.req.bits.data := dataModule.io.rdata(deqPtr).data
io.uncache.req.bits.mask := dataModule.io.rdata(deqPtr).mask
io.uncache.req.bits.meta.id := DontCare // TODO: // FIXME
io.uncache.req.bits.meta.id := DontCare
io.uncache.req.bits.meta.vaddr := DontCare
io.uncache.req.bits.meta.paddr := dataModule.io.rdata(deqPtr).paddr
io.uncache.req.bits.meta.uop := uop(deqPtr)
io.uncache.req.bits.meta.mmio := true.B // dataModule.io.rdata(deqPtr).mmio
io.uncache.req.bits.meta.mmio := true.B
io.uncache.req.bits.meta.tlb_miss := false.B
io.uncache.req.bits.meta.mask := dataModule.io.rdata(deqPtr).mask
io.uncache.req.bits.meta.replay := false.B
......@@ -506,17 +518,7 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
when (io.uncache.req.fire()) {
pending(deqPtr) := false.B
}
dataModule.io.uncache.wen := false.B
when(io.uncache.resp.fire()){
datavalid(deqPtr) := true.B
dataModule.io.uncacheWrite(deqPtr, io.uncache.resp.bits.data(XLEN-1, 0))
dataModule.io.uncache.wen := true.B
// TODO: write back exception info
}
when(io.uncache.req.fire()){
XSDebug("uncache req: pc %x addr %x data %x op %x mask %x\n",
uop(deqPtr).cf.pc,
io.uncache.req.bits.addr,
......@@ -526,7 +528,12 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
)
}
dataModule.io.uncache.wen := false.B
when(io.uncache.resp.fire()){
datavalid(deqPtr) := true.B
dataModule.io.uncacheWrite(deqPtr, io.uncache.resp.bits.data(XLEN-1, 0))
dataModule.io.uncache.wen := true.B
XSDebug("uncache resp: data %x\n", io.dcache.resp.bits.data)
}
......@@ -538,29 +545,19 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
val needCancel = Wire(Vec(LoadQueueSize, Bool()))
for (i <- 0 until LoadQueueSize) {
needCancel(i) := uop(i).roqIdx.needFlush(io.brqRedirect) && allocated(i) && !commited(i)
when(needCancel(i)) {
// when(io.brqRedirect.bits.isReplay){
// valid(i) := false.B
// writebacked(i) := false.B
// listening(i) := false.B
// miss(i) := false.B
// pending(i) := false.B
// }.otherwise{
when (needCancel(i)) {
allocated(i) := false.B
// }
}
}
when (io.brqRedirect.valid && io.brqRedirect.bits.isMisPred) {
enqPtrExt := enqPtrExt - PopCount(needCancel)
}
// assert(!io.rollback.valid)
when(io.rollback.valid) {
XSDebug("Mem rollback: pc %x roqidx %d\n", io.rollback.bits.pc, io.rollback.bits.roqIdx.asUInt)
// we recover the pointers in the next cycle after redirect
val needCancelReg = RegNext(needCancel)
when (lastCycleRedirect.valid) {
val cancelCount = PopCount(needCancelReg)
enqPtrExt := VecInit(enqPtrExt.map(_ - cancelCount))
}
// debug info
XSDebug("head %d:%d tail %d:%d\n", enqPtrExt.flag, enqPtr, deqPtrExt.flag, deqPtr)
XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt.flag, deqPtr)
def PrintFlag(flag: Bool, name: String): Unit = {
when(flag) {
......
src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala
浏览文件 @ 51556e23
......@@ -51,31 +51,27 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
val commited = Reg(Vec(StoreQueueSize, Bool())) // inst has been commited by roq
val pending = Reg(Vec(StoreQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of roq
val enqPtrExt = RegInit(0.U.asTypeOf(new SqPtr))
val deqPtrExt = RegInit(0.U.asTypeOf(new SqPtr))
val enqPtr = enqPtrExt.value
val deqPtr = deqPtrExt.value
val sameFlag = enqPtrExt.flag === deqPtrExt.flag
val isEmpty = enqPtr === deqPtr && sameFlag
val isFull = enqPtr === deqPtr && !sameFlag
val allowIn = !isFull
val storeCommit = (0 until CommitWidth).map(i => io.commits.valid(i) && !io.commits.isWalk && io.commits.uop(i).ctrl.commitType === CommitType.STORE)
val mcommitIdx = (0 until CommitWidth).map(i => io.commits.uop(i).sqIdx.value)
require(StoreQueueSize > RenameWidth)
val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new SqPtr))))
val deqPtrExt = RegInit(VecInit((0 until StorePipelineWidth).map(_.U.asTypeOf(new SqPtr))))
val enqPtr = enqPtrExt(0).value
val deqPtr = deqPtrExt(0).value
val tailMask = UIntToMask(deqPtr, StoreQueueSize)
val headMask = UIntToMask(enqPtr, StoreQueueSize)
val enqDeqMask1 = tailMask ^ headMask
val enqDeqMask = Mux(sameFlag, enqDeqMask1, ~enqDeqMask1)
// Enqueue at dispatch
val validEntries = distanceBetween(enqPtrExt, deqPtrExt)
/**
* Enqueue at dispatch
*
* Currently, StoreQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth)
*/
val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt(0))
val firedDispatch = io.enq.req.map(_.valid)
io.enq.canAccept := validEntries <= (StoreQueueSize - RenameWidth).U
XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n")
for (i <- 0 until RenameWidth) {
val offset = if (i == 0) 0.U else PopCount((0 until i).map(firedDispatch(_)))
val sqIdx = enqPtrExt + offset
val sqIdx = enqPtrExt(offset)
val index = sqIdx.value
when(io.enq.req(i).valid) {
uop(index) := io.enq.req(i).bits
......@@ -90,13 +86,26 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
XSError(!io.enq.canAccept && io.enq.req(i).valid, "should not valid when not ready\n")
}
when(Cat(firedDispatch).orR) {
enqPtrExt := enqPtrExt + PopCount(firedDispatch)
XSInfo("dispatched %d insts to sq\n", PopCount(firedDispatch))
when(Cat(firedDispatch).orR && !io.brqRedirect.valid) {
val enqNumber = PopCount(firedDispatch)
enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber))
XSInfo("dispatched %d insts to sq\n", enqNumber)
}
// writeback store
(0 until StorePipelineWidth).map(i => {
/**
* Writeback store from store units
*
* Most store instructions writeback to regfile in the previous cycle.
* However,
* (1) For an mmio instruction with exceptions, we need to mark it as datavalid
* (in this way it will trigger an exception when it reaches ROB's head)
* instead of pending to avoid sending them to lower level.
* (2) For an mmio instruction without exceptions, we mark it as pending.
* When the instruction reaches ROB's head, StoreQueue sends it to uncache channel.
* Upon receiving the response, StoreQueue writes back the instruction
* through arbiter with store units. It will later commit as normal.
*/
for (i <- 0 until StorePipelineWidth) {
dataModule.io.wb(i).wen := false.B
when(io.storeIn(i).fire()) {
val stWbIndex = io.storeIn(i).bits.uop.sqIdx.value
......@@ -129,94 +138,16 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
io.storeIn(i).bits.uop.cf.exceptionVec.asUInt
)
}
})
def getFirstOne(mask: Vec[Bool], startMask: UInt) = {
val length = mask.length
val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
val highBitsUint = Cat(highBits.reverse)
PriorityEncoder(Mux(highBitsUint.orR(), highBitsUint, mask.asUInt))
}
def getFirstOneWithFlag(mask: Vec[Bool], startMask: UInt, startFlag: Bool) = {
val length = mask.length
val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
val highBitsUint = Cat(highBits.reverse)
val changeDirection = !highBitsUint.orR()
val index = PriorityEncoder(Mux(!changeDirection, highBitsUint, mask.asUInt))
SqPtr(startFlag ^ changeDirection, index)
}
def selectFirstTwo(valid: Vec[Bool], startMask: UInt) = {
val selVec = Wire(Vec(2, UInt(log2Up(StoreQueueSize).W)))
val selValid = Wire(Vec(2, Bool()))
selVec(0) := getFirstOne(valid, startMask)
val firstSelMask = UIntToOH(selVec(0))
val secondSelVec = VecInit((0 until valid.length).map(i => valid(i) && !firstSelMask(i)))
selVec(1) := getFirstOne(secondSelVec, startMask)
selValid(0) := Cat(valid).orR
selValid(1) := Cat(secondSelVec).orR
(selValid, selVec)
}
def selectFirstTwoRoughly(valid: Vec[Bool]) = {
// TODO: do not select according to seq, just select 2 valid bit randomly
val firstSelVec = valid
val notFirstVec = Wire(Vec(valid.length, Bool()))
(0 until valid.length).map(i =>
notFirstVec(i) := (if(i != 0) { valid(i) || !notFirstVec(i) } else { false.B })
)
val secondSelVec = VecInit((0 until valid.length).map(i => valid(i) && !notFirstVec(i)))
val selVec = Wire(Vec(2, UInt(log2Up(valid.length).W)))
val selValid = Wire(Vec(2, Bool()))
selVec(0) := PriorityEncoder(firstSelVec)
selVec(1) := PriorityEncoder(secondSelVec)
selValid(0) := Cat(firstSelVec).orR
selValid(1) := Cat(secondSelVec).orR
(selValid, selVec)
}
// writeback finished mmio store
io.mmioStout.bits.uop := uop(deqPtr)
io.mmioStout.bits.uop.sqIdx := deqPtrExt
io.mmioStout.bits.uop.cf.exceptionVec := dataModule.io.rdata(deqPtr).exception.asBools
io.mmioStout.bits.data := dataModule.io.rdata(deqPtr).data
io.mmioStout.bits.redirectValid := false.B
io.mmioStout.bits.redirect := DontCare
io.mmioStout.bits.brUpdate := DontCare
io.mmioStout.bits.debug.isMMIO := true.B
io.mmioStout.bits.fflags := DontCare
io.mmioStout.valid := allocated(deqPtr) && datavalid(deqPtr) && !writebacked(deqPtr) // finished mmio store
when(io.mmioStout.fire()) {
writebacked(deqPtr) := true.B
allocated(deqPtr) := false.B // potential opt: move deqPtr immediately
}
// remove retired insts from sq, add retired store to sbuffer
// move tailPtr
// TailPtr slow recovery: recycle bubbles in store queue
// allocatedMask: dequeuePtr can go to the next 1-bit
val allocatedMask = VecInit((0 until StoreQueueSize).map(i => allocated(i) || !enqDeqMask(i)))
// find the first one from deqPtr (deqPtr)
val nextTail1 = getFirstOneWithFlag(allocatedMask, tailMask, deqPtrExt.flag)
val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, enqPtrExt)
deqPtrExt := nextTail
// TailPtr fast recovery
// val tailRecycle = VecInit(List(
// io.uncache.resp.fire() || io.sbuffer(0).fire(),
// io.sbuffer(1).fire()
// ))
when(io.sbuffer(0).fire()){
deqPtrExt := deqPtrExt + Mux(io.sbuffer(1).fire(), 2.U, 1.U)
}
// load forward query
/**
* load forward query
*
* Check store queue for instructions that is older than the load.
* The response will be valid at the next cycle after req.
*/
// check over all lq entries and forward data from the first matched store
(0 until LoadPipelineWidth).map(i => {
for (i <- 0 until LoadPipelineWidth) {
io.forward(i).forwardMask := 0.U(8.W).asBools
io.forward(i).forwardData := DontCare
......@@ -226,8 +157,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
// Forward1: Mux(same_flag, range(tail, sqIdx), range(tail, LoadQueueSize))
// Forward2: Mux(same_flag, 0.U, range(0, sqIdx) )
// i.e. forward1 is the target entries with the same flag bits and forward2 otherwise
val differentFlag = deqPtrExt.flag =/= io.forward(i).sqIdx.flag
val differentFlag = deqPtrExt(0).flag =/= io.forward(i).sqIdx.flag
val forwardMask = UIntToMask(io.forward(i).sqIdx.value, StoreQueueSize)
val storeWritebackedVec = WireInit(VecInit(Seq.fill(StoreQueueSize)(false.B)))
for (j <- 0 until StoreQueueSize) {
......@@ -236,7 +166,9 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
val needForward1 = Mux(differentFlag, ~tailMask, tailMask ^ forwardMask) & storeWritebackedVec.asUInt
val needForward2 = Mux(differentFlag, forwardMask, 0.U(StoreQueueSize.W)) & storeWritebackedVec.asUInt
XSDebug("" + i + " f1 %b f2 %b sqIdx %d pa %x\n", needForward1, needForward2, io.forward(i).sqIdx.asUInt, io.forward(i).paddr)
XSDebug(p"$i f1 ${Binary(needForward1)} f2 ${Binary(needForward2)} " +
p"sqIdx ${io.forward(i).sqIdx} pa ${Hexadecimal(io.forward(i).paddr)}\n"
)
// do real fwd query
dataModule.io.forwardQuery(
......@@ -248,40 +180,19 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
io.forward(i).forwardMask := dataModule.io.forward(i).forwardMask
io.forward(i).forwardData := dataModule.io.forward(i).forwardData
})
// When store commited, mark it as commited (will not be influenced by redirect),
(0 until CommitWidth).map(i => {
when(storeCommit(i)) {
commited(mcommitIdx(i)) := true.B
XSDebug("store commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc)
}
})
(0 until 2).map(i => {
val ptr = (deqPtrExt + i.U).value
val mmio = dataModule.io.rdata(ptr).mmio
io.sbuffer(i).valid := allocated(ptr) && commited(ptr) && !mmio
io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR
io.sbuffer(i).bits.addr := dataModule.io.rdata(ptr).paddr
io.sbuffer(i).bits.data := dataModule.io.rdata(ptr).data
io.sbuffer(i).bits.mask := dataModule.io.rdata(ptr).mask
io.sbuffer(i).bits.meta := DontCare
io.sbuffer(i).bits.meta.tlb_miss := false.B
io.sbuffer(i).bits.meta.uop := DontCare
io.sbuffer(i).bits.meta.mmio := mmio
io.sbuffer(i).bits.meta.mask := dataModule.io.rdata(ptr).mask
when(io.sbuffer(i).fire()) {
allocated(ptr) := false.B
XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr)
}
})
// Memory mapped IO / other uncached operations
}
// setup misc mem access req
// mask / paddr / data can be get from sq.data
/**
* Memory mapped IO / other uncached operations
*
* States:
* (1) writeback from store units: mark as pending
* (2) when they reach ROB's head, they can be sent to uncache channel
* (3) response from uncache channel: mark as datavalid
* (4) writeback to ROB (and other units): mark as writebacked
* (5) ROB commits the instruction: same as normal instructions
*/
//(2) when they reach ROB's head, they can be sent to uncache channel
val commitType = io.commits.uop(0).ctrl.commitType
io.uncache.req.valid := pending(deqPtr) && allocated(deqPtr) &&
commitType === CommitType.STORE &&
......@@ -297,30 +208,88 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
io.uncache.req.bits.meta.vaddr := DontCare
io.uncache.req.bits.meta.paddr := dataModule.io.rdata(deqPtr).paddr
io.uncache.req.bits.meta.uop := uop(deqPtr)
io.uncache.req.bits.meta.mmio := true.B // dataModule.io.rdata(deqPtr).mmio
io.uncache.req.bits.meta.mmio := true.B
io.uncache.req.bits.meta.tlb_miss := false.B
io.uncache.req.bits.meta.mask := dataModule.io.rdata(deqPtr).mask
io.uncache.req.bits.meta.replay := false.B
io.uncache.resp.ready := true.B
when(io.uncache.req.fire()){
pending(deqPtr) := false.B
XSDebug(
p"uncache req: pc ${Hexadecimal(uop(deqPtr).cf.pc)} " +
p"addr ${Hexadecimal(io.uncache.req.bits.addr)} " +
p"data ${Hexadecimal(io.uncache.req.bits.data)} " +
p"op ${Hexadecimal(io.uncache.req.bits.cmd)} " +
p"mask ${Hexadecimal(io.uncache.req.bits.mask)}\n"
)
}
when(io.uncache.resp.fire()){
datavalid(deqPtr) := true.B // will be writeback to CDB in the next cycle
// TODO: write back exception info
// (3) response from uncache channel: mark as datavalid
io.uncache.resp.ready := true.B
when (io.uncache.resp.fire()) {
datavalid(deqPtr) := true.B
}
when(io.uncache.req.fire()){
XSDebug("uncache req: pc %x addr %x data %x op %x mask %x\n",
uop(deqPtr).cf.pc,
io.uncache.req.bits.addr,
io.uncache.req.bits.data,
io.uncache.req.bits.cmd,
io.uncache.req.bits.mask
)
// (4) writeback to ROB (and other units): mark as writebacked
io.mmioStout.valid := allocated(deqPtr) && datavalid(deqPtr) && !writebacked(deqPtr)
io.mmioStout.bits.uop := uop(deqPtr)
io.mmioStout.bits.uop.sqIdx := deqPtrExt(0)
io.mmioStout.bits.uop.cf.exceptionVec := dataModule.io.rdata(deqPtr).exception.asBools
io.mmioStout.bits.data := dataModule.io.rdata(deqPtr).data
io.mmioStout.bits.redirectValid := false.B
io.mmioStout.bits.redirect := DontCare
io.mmioStout.bits.brUpdate := DontCare
io.mmioStout.bits.debug.isMMIO := true.B
io.mmioStout.bits.fflags := DontCare
when (io.mmioStout.fire()) {
writebacked(deqPtr) := true.B
allocated(deqPtr) := false.B
deqPtrExt := VecInit(deqPtrExt.map(_ + 1.U))
}
/**
* ROB commits store instructions (mark them as commited)
*
* (1) When store commits, mark it as commited.
* (2) They will not be cancelled and can be sent to lower level.
*/
for (i <- 0 until CommitWidth) {
val storeCommit = !io.commits.isWalk && io.commits.valid(i) && io.commits.uop(i).ctrl.commitType === CommitType.STORE
when (storeCommit) {
commited(io.commits.uop(i).sqIdx.value) := true.B
XSDebug("store commit %d: idx %d %x\n", i.U, io.commits.uop(i).sqIdx.value, io.commits.uop(i).cf.pc)
}
}
// Commited stores will not be cancelled and can be sent to lower level.
// remove retired insts from sq, add retired store to sbuffer
for (i <- 0 until StorePipelineWidth) {
val ptr = deqPtrExt(i).value
val mmio = dataModule.io.rdata(ptr).mmio
io.sbuffer(i).valid := allocated(ptr) && commited(ptr) && !mmio
io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR
io.sbuffer(i).bits.addr := dataModule.io.rdata(ptr).paddr
io.sbuffer(i).bits.data := dataModule.io.rdata(ptr).data
io.sbuffer(i).bits.mask := dataModule.io.rdata(ptr).mask
io.sbuffer(i).bits.meta := DontCare
io.sbuffer(i).bits.meta.tlb_miss := false.B
io.sbuffer(i).bits.meta.uop := DontCare
io.sbuffer(i).bits.meta.mmio := mmio
io.sbuffer(i).bits.meta.mask := dataModule.io.rdata(ptr).mask
when (io.sbuffer(i).fire()) {
allocated(ptr) := false.B
XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr)
}
}
// note that sbuffer will not accept req(1) if req(0) is not accepted.
when (Cat(io.sbuffer.map(_.fire())).orR) {
val stepForward = Mux(io.sbuffer(1).fire(), 2.U, 1.U)
deqPtrExt := VecInit(deqPtrExt.map(_ + stepForward))
when (io.sbuffer(1).fire()) {
assert(io.sbuffer(0).fire())
}
}
// Read vaddr for mem exception
......@@ -331,22 +300,19 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
val needCancel = Wire(Vec(StoreQueueSize, Bool()))
for (i <- 0 until StoreQueueSize) {
needCancel(i) := uop(i).roqIdx.needFlush(io.brqRedirect) && allocated(i) && !commited(i)
when(needCancel(i)) {
// when(io.brqRedirect.bits.isReplay){
// datavalid(i) := false.B
// writebacked(i) := false.B
// pending(i) := false.B
// }.otherwise{
when (needCancel(i)) {
allocated(i) := false.B
// }
}
}
when (io.brqRedirect.valid && io.brqRedirect.bits.isMisPred) {
enqPtrExt := enqPtrExt - PopCount(needCancel)
// we recover the pointers in the next cycle after redirect
val lastCycleRedirectValid = RegNext(io.brqRedirect.valid)
val needCancelCount = PopCount(RegNext(needCancel))
when (lastCycleRedirectValid) {
enqPtrExt := VecInit(enqPtrExt.map(_ - needCancelCount))
}
// debug info
XSDebug("head %d:%d tail %d:%d\n", enqPtrExt.flag, enqPtr, deqPtrExt.flag, deqPtr)
XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt(0).flag, deqPtr)
def PrintFlag(flag: Bool, name: String): Unit = {
when(flag) {
......
src/main/scala/xiangshan/mem/sbuffer/NewSbuffer.scala
浏览文件 @ 51556e23
......@@ -189,6 +189,9 @@ class NewSbuffer extends XSModule with HasSbufferCst {
val updatedSbuffer = io.in.zipWithIndex.foldLeft[Seq[SbufferEntry]](initialSbuffer)(enqSbuffer)
val updatedState = updatedSbuffer.map(_._1)
val updatedSbufferLine = VecInit(updatedSbuffer.map(_._2))
when (!io.in(0).ready) {
io.in(1).ready := false.B
}
for(i <- 0 until StoreBufferSize){
buffer.write(i.U, updatedSbufferLine(i))
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或