Merge remote-tracking branch 'origin/master' into perf-debug

.gitmodules

@@ -4,5 +4,5 @@
 	branch = 2bdb03dbca3f77ad4c378cc1b95ab4961bc1448a
 [submodule "block-inclusivecache-sifive"]
 	path = block-inclusivecache-sifive
-	url = https://github.com/sifive/block-inclusivecache-sifive.git
-	branch = d4db623ff534f775ffc49f59c4a9ef24d5d759d0
+	url = https://github.com/RISCVERS/block-inclusivecache-sifive.git
+	branch = 5491dcc937ed3c6f7722bef9db448653daab75e8

block-inclusivecache-sifive @ 5ca43398

-Subproject commit d4db623ff534f775ffc49f59c4a9ef24d5d759d0
+Subproject commit 5ca43398ac8b1b293291bd4e6e8c233be6c66968

src/main/scala/xiangshan/backend/Backend.scala

@@ -213,8 +213,11 @@ class Backend extends XSModule
   roq.io.dp1Req.foreach(_.bits.debugInfo.dispatchTime := timer)
   dispatch.io.roqIdxs <> roq.io.roqIdxs
   io.mem.dp1Req <> dispatch.io.toLsroq
-  dispatch.io.commits <> roq.io.commits
   dispatch.io.lsIdxs <> io.mem.lsIdxs
+  dispatch.io.dequeueRoqIndex.valid := roq.io.commitRoqIndex.valid || io.mem.oldestStore.valid
+  // store writeback must be after commit roqIdx
+  dispatch.io.dequeueRoqIndex.bits := Mux(io.mem.oldestStore.valid, io.mem.oldestStore.bits, roq.io.commitRoqIndex.bits)
+
 
   intRf.io.readPorts <> dispatch.io.readIntRf
   fpRf.io.readPorts <> dispatch.io.readFpRf ++ issueQueues.flatMap(_.io.readFpRf)

src/main/scala/xiangshan/backend/dispatch/Dispatch.scala

@@ -36,7 +36,7 @@ class Dispatch extends XSModule {
     val toLsroq =  Vec(RenameWidth, DecoupledIO(new MicroOp))
     // get LsIdx
     val lsIdxs = Input(Vec(RenameWidth, new LSIdx))
-    val commits = Input(Vec(CommitWidth, Valid(new RoqCommit)))
+    val dequeueRoqIndex = Input(Valid(new RoqPtr))
     // read regfile
     val readIntRf = Vec(NRIntReadPorts, Flipped(new RfReadPort))
     val readFpRf = Vec(NRFpReadPorts - exuParameters.StuCnt, Flipped(new RfReadPort))
@@ -80,30 +80,21 @@ class Dispatch extends XSModule {
   // dispatch queue: queue uops and dispatch them to different reservation stations or issue queues
   // it may cancel the uops
   intDq.io.redirect <> io.redirect
-  intDq.io.commits <> io.commits
-  intDq.io.commits.zip(io.commits).map { case (dqCommit, commit) =>
-    dqCommit.valid := commit.valid && dqCommit.bits.uop.ctrl.commitType === CommitType.INT
-  }
+  intDq.io.dequeueRoqIndex <> io.dequeueRoqIndex
   intDq.io.replayPregReq.zipWithIndex.map { case(replay, i) =>
     io.replayPregReq(i) <> replay
   }
   intDq.io.otherWalkDone := !fpDq.io.inReplayWalk && !lsDq.io.inReplayWalk
 
   fpDq.io.redirect <> io.redirect
-  fpDq.io.commits <> io.commits
-  fpDq.io.commits.zip(io.commits).map { case (dqCommit, commit) =>
-    dqCommit.valid := commit.valid && dqCommit.bits.uop.ctrl.commitType === CommitType.FP
-  }
+  fpDq.io.dequeueRoqIndex <> io.dequeueRoqIndex
   fpDq.io.replayPregReq.zipWithIndex.map { case(replay, i) =>
     io.replayPregReq(i + dpParams.IntDqReplayWidth) <> replay
   }
   fpDq.io.otherWalkDone := !intDq.io.inReplayWalk && !lsDq.io.inReplayWalk
 
   lsDq.io.redirect <> io.redirect
-  lsDq.io.commits <> io.commits
-  lsDq.io.commits.zip(io.commits).map { case (dqCommit, commit) =>
-    dqCommit.valid := commit.valid && CommitType.isLoadStore(dqCommit.bits.uop.ctrl.commitType)
-  }
+  lsDq.io.dequeueRoqIndex <> io.dequeueRoqIndex
   lsDq.io.replayPregReq.zipWithIndex.map { case(replay, i) =>
     io.replayPregReq(i + dpParams.IntDqReplayWidth + dpParams.FpDqReplayWidth) <> replay
   }

src/main/scala/xiangshan/backend/dispatch/DispatchQueue.scala

@@ -4,13 +4,13 @@ import chisel3._
 import chisel3.util._
 import utils._
 import xiangshan.backend.decode.SrcType
-import xiangshan.{MicroOp, Redirect, ReplayPregReq, RoqCommit, XSBundle, XSModule}
-
+import xiangshan._
+import xiangshan.backend.roq.RoqPtr
 
 class DispatchQueueIO(enqnum: Int, deqnum: Int, replayWidth: Int) extends XSBundle {
   val enq = Vec(enqnum, Flipped(DecoupledIO(new MicroOp)))
   val deq = Vec(deqnum, DecoupledIO(new MicroOp))
-  val commits = Input(Vec(CommitWidth, Valid(new RoqCommit)))
+  val dequeueRoqIndex = Input(Valid(new RoqPtr))
   val redirect = Flipped(ValidIO(new Redirect))
   val replayPregReq = Output(Vec(replayWidth, new ReplayPregReq))
   val inReplayWalk = Output(Bool())
@@ -89,14 +89,17 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int, replayWidth: Int) exten
   }
 
   // commit: from s_dispatched to s_invalid
-  val numCommit = PopCount(io.commits.map(commit => !commit.bits.isWalk && commit.valid))
-  val commitBits = (1.U((CommitWidth+1).W) << numCommit).asUInt() - 1.U
-  for (i <- 0 until CommitWidth) {
-    when (commitBits(i)) {
-      stateEntries(commitIndex(i)) := s_invalid
-
-      XSError(stateEntries(commitIndex(i)) =/= s_dispatched, "state of the commit entry is not s_dispatched\n")
+  val needDequeue = Wire(Vec(size, Bool()))
+  val deqRoqIdx = io.dequeueRoqIndex.bits
+  for (i <- 0 until size) {
+    needDequeue(i) := stateEntries(i)  === s_dispatched && io.dequeueRoqIndex.valid && !isAfter(uopEntries(i).roqIdx, deqRoqIdx) && dispatchedMask(i)
+    when (needDequeue(i)) {
+      stateEntries(i) := s_invalid
     }
+
+    XSInfo(needDequeue(i), p"dispatched entry($i)(pc = ${Hexadecimal(uopEntries(i).cf.pc)}) " +
+      p"roqIndex 0x${Hexadecimal(uopEntries(i).roqIdx.asUInt)} " +
+      p"left dispatch queue with deqRoqIndex 0x${Hexadecimal(io.dequeueRoqIndex.bits.asUInt)}\n")
   }
 
   // redirect: cancel uops currently in the queue
@@ -107,14 +110,15 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int, replayWidth: Int) exten
   val needCancel = Wire(Vec(size, Bool()))
   for (i <- 0 until size) {
     roqNeedFlush(i) := uopEntries(i.U).roqIdx.needFlush(io.redirect)
-    needCancel(i) := stateEntries(i) =/= s_invalid && ((roqNeedFlush(i) && mispredictionValid) || exceptionValid || flushPipeValid)
+    needCancel(i) := stateEntries(i) =/= s_invalid && ((roqNeedFlush(i) && mispredictionValid) || exceptionValid || flushPipeValid) && !needDequeue(i)
+
     when (needCancel(i)) {
       stateEntries(i) := s_invalid
     }
 
-    XSInfo(needCancel(i), p"valid entry($i)(pc = ${Hexadecimal(uopEntries(i.U).cf.pc)}) " +
-      p"roqIndex ${uopEntries(i.U).roqIdx} " +
-      p"cancelled with redirect roqIndex ${io.redirect.bits.roqIdx}\n")
+    XSInfo(needCancel(i), p"valid entry($i)(pc = ${Hexadecimal(uopEntries(i).cf.pc)}) " +
+      p"roqIndex 0x${Hexadecimal(uopEntries(i).roqIdx.asUInt)} " +
+      p"cancelled with redirect roqIndex 0x${Hexadecimal(io.redirect.bits.roqIdx.asUInt)}\n")
   }
 
   // replay: from s_dispatched to s_valid
@@ -253,7 +257,7 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int, replayWidth: Int) exten
       Mux(inReplayWalk, dispatchPtr - dispatchReplayStep, dispatchPtr + numDeq))
   )
 
-  headPtr := Mux(exceptionValid, 0.U.asTypeOf(new CircularQueuePtr(size)), headPtr + numCommit)
+  headPtr := Mux(exceptionValid, 0.U.asTypeOf(new CircularQueuePtr(size)), headPtr + PopCount(needDequeue))
 
   /**
     * Part 4: set output and input

src/main/scala/xiangshan/backend/roq/Roq.scala

@@ -38,6 +38,7 @@ class Roq extends XSModule with HasCircularQueuePtrHelper {
     val exeWbResults = Vec(exuParameters.ExuCnt + 1, Flipped(ValidIO(new ExuOutput)))
     val commits = Vec(CommitWidth, Valid(new RoqCommit))
     val bcommit = Output(UInt(BrTagWidth.W))
+    val commitRoqIndex = Output(Valid(new RoqPtr))
     val roqDeqPtr = Output(new RoqPtr)
   })
 
@@ -265,6 +266,9 @@ class Roq extends XSModule with HasCircularQueuePtrHelper {
   }
   val retireCounter = Mux(state === s_idle, commitCnt, 0.U)
   XSInfo(retireCounter > 0.U, "retired %d insts\n", retireCounter)
+  val commitOffset = PriorityEncoder((validCommit :+ false.B).map(!_))
+  io.commitRoqIndex.valid := io.commits(0).valid && !io.commits(0).bits.isWalk
+  io.commitRoqIndex.bits := deqPtrExt + commitOffset
 
   // commit branch to brq
   io.bcommit := PopCount(cfiCommitVec)

src/main/scala/xiangshan/cache/missQueue.scala

@@ -230,6 +230,7 @@ class MissEntry(edge: TLEdgeOut) extends DCacheModule
         when (req.cmd === M_XRD) {
           // normal read
           // read hit, no need to update meta
+          new_coh   := old_coh
           new_state := s_send_resp
         } .otherwise {
           assert(isWrite(req.cmd))
@@ -301,7 +302,12 @@ class MissEntry(edge: TLEdgeOut) extends DCacheModule
 
     when (edge.hasData(io.mem_grant.bits)) {
       when (io.mem_grant.fire()) {
-        assert(should_refill_data)
+        // for AcquireBlock BtoT, we clear should_refill_data
+        // and expect response with no data(Grant, not GrantData)
+        // but block inclusive cache responded with a GrantData!
+        // so we temporarily removed this assertion
+        // we may consider using AcquirePerm BtoT for permission upgrade
+        // assert(should_refill_data)
         refill_ctr := refill_ctr + 1.U
         for (i <- 0 until beatRows) {
           val row = io.mem_grant.bits.data(rowBits * (i + 1) - 1, rowBits * i)

src/main/scala/xiangshan/mem/Memend.scala

@@ -77,6 +77,7 @@ class MemToBackendIO extends XSBundle {
   val commits = Flipped(Vec(CommitWidth, Valid(new RoqCommit)))
   val dp1Req = Vec(RenameWidth, Flipped(DecoupledIO(new MicroOp)))
   val lsIdxs = Output(Vec(RenameWidth, new LSIdx))
+  val oldestStore = Output(Valid(new RoqPtr))
   val roqDeqPtr = Input(new RoqPtr)
 }
 
@@ -145,6 +146,7 @@ class Memend extends XSModule {
   lsroq.io.stout       <> io.backend.stout
   lsroq.io.commits     <> io.backend.commits
   lsroq.io.dp1Req      <> io.backend.dp1Req
+  lsroq.io.oldestStore <> io.backend.oldestStore
   lsroq.io.lsIdxs   <> io.backend.lsIdxs
   lsroq.io.brqRedirect := io.backend.redirect
   lsroq.io.roqDeqPtr := io.backend.roqDeqPtr

src/main/scala/xiangshan/mem/lsqueue/separated/LSQWrapper.scala

@@ -29,6 +29,7 @@ class LsqWrappper extends XSModule with HasDCacheParameters with NeedImpl {
     val dcache = new DCacheLineIO
     val uncache = new DCacheWordIO
     val roqDeqPtr = Input(new RoqPtr)
+    val oldestStore = Output(Valid(new RoqPtr))
   })
   
   if(EnableUnifiedLSQ){
@@ -47,6 +48,7 @@ class LsqWrappper extends XSModule with HasDCacheParameters with NeedImpl {
     lsroq.io.dcache <> io.dcache
     lsroq.io.uncache <> io.uncache
     lsroq.io.roqDeqPtr <> io.roqDeqPtr
+    lsroq.io.oldestStore <> io.oldestStore
     (0 until RenameWidth).map(i => {
       io.lsIdxs(i).lsroqIdx := lsroq.io.lsroqIdxs(i)
     })
@@ -74,6 +76,7 @@ class LsqWrappper extends XSModule with HasDCacheParameters with NeedImpl {
     storeQueue.io.stout <> io.stout
     storeQueue.io.commits <> io.commits
     storeQueue.io.roqDeqPtr <> io.roqDeqPtr
+    storeQueue.io.oldestStore <> io.oldestStore
     
     loadQueue.io.forward <> io.forward
     storeQueue.io.forward <> io.forward // overlap forwardMask & forwardData, DO NOT CHANGE SEQUENCE

src/main/scala/xiangshan/mem/lsqueue/separated/LoadQueue.scala

@@ -300,14 +300,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
     }
   })
 
-  // move tailPtr
-  // allocatedMask: dequeuePtr can go to the next 1-bit
-  val allocatedMask = VecInit((0 until LoadQueueSize).map(i => allocated(i) || !enqDeqMask(i)))
-  // find the first one from deqPtr (ringBufferTail)
-  val nextTail1 = getFirstOneWithFlag(allocatedMask, tailMask, ringBufferTailExtended.flag)
-  val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, ringBufferHeadExtended)
-  ringBufferTailExtended := nextTail
-
   // When load commited, mark it as !allocated, this entry will be recycled later
   (0 until CommitWidth).map(i => {
     when(loadCommit(i)) {
@@ -315,6 +307,8 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP
       XSDebug("load commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc)
     }
   })
+  // move tailPtr
+  ringBufferTailExtended := ringBufferTailExtended + PopCount(loadCommit)
 
   // rollback check
   val rollback = Wire(Vec(StorePipelineWidth, Valid(new Redirect)))

src/main/scala/xiangshan/mem/lsqueue/separated/StoreQueue.scala

@@ -21,7 +21,6 @@ object SqPtr extends HasXSParameter {
   }
 }
 
-
 // Store Queue
 class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper {
   val io = IO(new Bundle() {
@@ -36,6 +35,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     val uncache = new DCacheWordIO
     val roqDeqPtr = Input(new RoqPtr)
     // val refill = Flipped(Valid(new DCacheLineReq ))
+    val oldestStore = Output(Valid(new RoqPtr))
   })
   
   val uop = Reg(Vec(StoreQueueSize, new MicroOp))
@@ -125,9 +125,6 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     }
   })
 
-  // writeback up to 2 store insts to CDB
-  // choose the first two valid store requests from deqPtr
-
   def getFirstOne(mask: Vec[Bool], startMask: UInt) = {
     val length = mask.length
     val highBits = (0 until length).map(i => mask(i) & ~startMask(i))
@@ -156,9 +153,16 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     (selValid, selVec)
   }
 
-  val storeWbSelVec = VecInit((0 until StoreQueueSize).map(i => {
-    allocated(i) && valid(i) && !writebacked(i)
-  }))
+  // select the last writebacked instruction
+  val validStoreVec = VecInit((0 until StoreQueueSize).map(i => !(allocated(i) && valid(i))))
+  val storeNotValid = SqPtr(false.B, getFirstOne(validStoreVec, tailMask))
+  val storeValidIndex = (storeNotValid - 1.U).value
+  io.oldestStore.valid := allocated(ringBufferTailExtended.value) && valid(ringBufferTailExtended.value) && !commited(storeValidIndex)
+  io.oldestStore.bits := uop(storeValidIndex).roqIdx
+
+  // writeback up to 2 store insts to CDB
+  // choose the first two valid store requests from deqPtr
+  val storeWbSelVec = VecInit((0 until StoreQueueSize).map(i => allocated(i) && valid(i) && !writebacked(i)))
   val (storeWbValid, storeWbSel) = selectFirstTwo(storeWbSelVec, tailMask)
 
   (0 until StorePipelineWidth).map(i => {
@@ -176,15 +180,6 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     }
   })
 
-  // remove retired insts from sq, add retired store to sbuffer
-
-  // move tailPtr
-  // allocatedMask: dequeuePtr can go to the next 1-bit
-  val allocatedMask = VecInit((0 until StoreQueueSize).map(i => allocated(i) || !enqDeqMask(i)))
-  // find the first one from deqPtr (ringBufferTail)
-  val nextTail1 = getFirstOneWithFlag(allocatedMask, tailMask, ringBufferTailExtended.flag)
-  val nextTail = Mux(Cat(allocatedMask).orR, nextTail1, ringBufferHeadExtended)
-  ringBufferTailExtended := nextTail
 
   // load forward query
   // check over all lq entries and forward data from the first matched store
@@ -246,13 +241,13 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     }
   })
 
+  // remove retired insts from sq, add retired store to sbuffer
   val storeCommitSelVec = VecInit((0 until StoreQueueSize).map(i => {
     allocated(i) && commited(i)
   }))
   val (storeCommitValid, storeCommitSel) = selectFirstTwo(storeCommitSelVec, tailMask)
-  
-  // get no more than 2 commited store from storeCommitedQueue
   // send selected store inst to sbuffer
+  val dequeueValid = Wire(Vec(2, Bool()))
   (0 until 2).map(i => {
     val ptr = storeCommitSel(i)
     val mmio = data(ptr).mmio
@@ -268,11 +263,14 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue
     io.sbuffer(i).bits.meta.mask     := data(ptr).mask
     
     // update sq meta if store inst is send to sbuffer
-    when(storeCommitValid(i) && (mmio || io.sbuffer(i).ready)) {
+    dequeueValid(i) := storeCommitValid(i) && (mmio || io.sbuffer(i).ready)
+    when (dequeueValid(i)) {
       allocated(ptr) := false.B
     }
   })
-  
+  // move tailPtr
+  ringBufferTailExtended := ringBufferTailExtended + PopCount(dequeueValid)
+
   // Memory mapped IO / other uncached operations
   
   // setup misc mem access req

src/main/scala/xiangshan/mem/lsqueue/unified/Lsroq.scala

@@ -33,6 +33,7 @@ class Lsroq extends XSModule with HasDCacheParameters with HasCircularQueuePtrHe
   val io = IO(new Bundle() {
     val dp1Req = Vec(RenameWidth, Flipped(DecoupledIO(new MicroOp)))
     val lsroqIdxs = Output(Vec(RenameWidth, UInt(LsroqIdxWidth.W)))
+    val oldestStore = Output(Valid(new RoqPtr))
     val brqRedirect = Input(Valid(new Redirect))
     val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LsPipelineBundle)))
     val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle)))
@@ -164,7 +165,12 @@ class Lsroq extends XSModule with HasDCacheParameters with HasCircularQueuePtrHe
         pending(io.loadIn(i).bits.uop.lsroqIdx) := io.loadIn(i).bits.mmio
       }
     })
-    
+
+    // find first store req that has not been writebacked
+    val storeNotWritebacked = VecInit((0 until LsroqSize).map(i => store(i) && !writebacked(i)))
+    val firstStore = getFirstOne(storeNotWritebacked, tailMask)
+    io.oldestStore.valid := false.B
+    io.oldestStore.bits := DontCare
     // writeback store
     (0 until StorePipelineWidth).map(i => {
       when(io.storeIn(i).fire()) {
@@ -179,16 +185,20 @@ class Lsroq extends XSModule with HasDCacheParameters with HasCircularQueuePtrHe
         store(io.storeIn(i).bits.uop.lsroqIdx) := true.B
         pending(io.storeIn(i).bits.uop.lsroqIdx) := io.storeIn(i).bits.mmio
         XSInfo("store write to lsroq idx %d pc 0x%x vaddr %x paddr %x data %x miss %x mmio %x roll %x exc %x\n",
-        io.storeIn(i).bits.uop.lsroqIdx(InnerLsroqIdxWidth - 1, 0),
-        io.storeIn(i).bits.uop.cf.pc,
-        io.storeIn(i).bits.vaddr,
-        io.storeIn(i).bits.paddr,
-        io.storeIn(i).bits.data,
-        io.storeIn(i).bits.miss,
-        io.storeIn(i).bits.mmio,
-        io.storeIn(i).bits.rollback,
-        io.storeIn(i).bits.uop.cf.exceptionVec.asUInt
+          io.storeIn(i).bits.uop.lsroqIdx(InnerLsroqIdxWidth - 1, 0),
+          io.storeIn(i).bits.uop.cf.pc,
+          io.storeIn(i).bits.vaddr,
+          io.storeIn(i).bits.paddr,
+          io.storeIn(i).bits.data,
+          io.storeIn(i).bits.miss,
+          io.storeIn(i).bits.mmio,
+          io.storeIn(i).bits.rollback,
+          io.storeIn(i).bits.uop.cf.exceptionVec.asUInt
         )
+        when (io.storeIn(i).bits.uop.lsroqIdx(InnerLsroqIdxWidth - 1, 0) === firstStore) {
+          io.oldestStore.valid := true.B
+          io.oldestStore.bits := io.storeIn(i).bits.uop.roqIdx
+        }
       }
   })
 

src/test/csrc/emu.cpp

@@ -74,6 +74,7 @@ Emulator::Emulator(int argc, const char *argv[]):
   cycles(0), hascommit(0), trapCode(STATE_RUNNING)
 {
   args = parse_args(argc, argv);
+  printf("Emu compiled at %s, %s UTC\n", __DATE__, __TIME__);
 
   // srand
   srand(args.seed);

src/test/csrc/ram.cpp

@@ -100,7 +100,6 @@ void init_ram(const char *img) {
     assert(0);
   }
 
-  printf("Emu compiled at %s, %s\n", __DATE__, __TIME__);
   printf("The image is %s\n", img);
 
   fseek(fp, 0, SEEK_END);

src/test/scala/cache/L2CacheTest.scala

@@ -7,6 +7,7 @@ import chiseltest.experimental.TestOptionBuilder._
 import chiseltest.internal.VerilatorBackendAnnotation
 import chiseltest._
 import chisel3.experimental.BundleLiterals._
+import firrtl.stage.RunFirrtlTransformAnnotation
 import chiseltest.ChiselScalatestTester
 import device.AXI4RAM
 import freechips.rocketchip.amba.axi4.AXI4UserYanker
@@ -18,6 +19,7 @@ import utils.{DebugIdentityNode, HoldUnless, XSDebug}
 import xiangshan.HasXSLog
 import xiangshan.cache.{DCache, DCacheLineReq, DCacheWordReq, MemoryOpConstants}
 import xiangshan.testutils.AddSinks
+import xstransforms.PrintModuleName
 
 import scala.util.Random
 
@@ -28,13 +30,28 @@ case class L2CacheTestParams
   banks: Int = 1,
   capacityKB: Int = 4,
   blockBytes: Int = 64,
-  beatBytes: Int = 8
+  beatBytes: Int = 32,
+  writeBytes: Int = 8
 ) {
   require(blockBytes >= beatBytes)
 }
 
 case object L2CacheTestKey extends Field[L2CacheTestParams]
 
+case class L3CacheTestParams
+(
+  ways: Int = 4,
+  banks: Int = 1,
+  capacityKB: Int = 4,
+  blockBytes: Int = 64,
+  beatBytes: Int = 32,
+  writeBytes: Int = 8
+) {
+  require(blockBytes >= beatBytes)
+}
+
+case object L3CacheTestKey extends Field[L3CacheTestParams]
+
 
 class L2TestTopIO extends Bundle {
   val in = Flipped(DecoupledIO(new Bundle() {
@@ -50,19 +67,34 @@ class L2TestTopIO extends Bundle {
 class L2TestTop()(implicit p: Parameters) extends LazyModule{
 
   val cores = Array.fill(2)(LazyModule(new DCache()))
-
   val l2params = p(L2CacheTestKey)
-
-  val l2 = LazyModule(new InclusiveCache(
+  val l2s = (0 until 2) map (i =>
+    LazyModule(new InclusiveCache(
     CacheParameters(
       level = 2,
       ways = l2params.ways,
       sets = l2params.capacityKB * 1024 / (l2params.blockBytes * l2params.ways * l2params.banks),
       blockBytes = l2params.blockBytes,
-      beatBytes = l2params.beatBytes
+      beatBytes = l2params.beatBytes,
+      cacheName = s"L2_$i"
     ),
     InclusiveCacheMicroParameters(
-      writeBytes = l2params.beatBytes
+      writeBytes = l2params.writeBytes
+    )
+  )))
+
+  val l3params = p(L3CacheTestKey)
+  val l3 = LazyModule(new InclusiveCache(
+    CacheParameters(
+      level = 3,
+      ways = l3params.ways,
+      sets = l3params.capacityKB * 1024 / (l3params.blockBytes * l3params.ways * l3params.banks),
+      blockBytes = l3params.blockBytes,
+      beatBytes = l3params.beatBytes,
+      cacheName = "L3"
+    ),
+    InclusiveCacheMicroParameters(
+      writeBytes = l3params.writeBytes
     )
   ))
 
@@ -74,18 +106,20 @@ class L2TestTop()(implicit p: Parameters) extends LazyModule{
 
   val xbar = TLXbar()
 
-  for(core <- cores){
-    xbar := TLBuffer() := DebugIdentityNode() := core.clientNode
+  for(i <- 0 until 2) {
+    val core = cores(i)
+    val l2 = l2s(i)
+    xbar := l2.node := core.clientNode
   }
 
-  l2.node := TLBuffer() := DebugIdentityNode() := xbar
+  l3.node := xbar
 
   ram.node :=
     AXI4UserYanker() :=
     TLToAXI4() :=
     TLBuffer() :=
     TLCacheCork() :=
-    l2.node
+    l3.node
 
   lazy val module = new LazyModuleImp(this) with HasXSLog {
 
@@ -96,7 +130,7 @@ class L2TestTop()(implicit p: Parameters) extends LazyModule{
     cores.foreach(_.module.io <> DontCare)
 
     val storePorts = cores.map(_.module.io.lsu.store)
-    val loadPorts  = cores.map(_.module.io.lsu.lsroq)
+    val loadPorts  = cores.map(_.module.io.lsu.atomics)
 
     def sendStoreReq(addr: UInt, data: UInt): DCacheLineReq = {
       val req = Wire(new DCacheLineReq)
@@ -110,9 +144,9 @@ class L2TestTop()(implicit p: Parameters) extends LazyModule{
 
     def sendLoadReq(addr: UInt): DCacheWordReq = {
       val req = Wire(new DCacheWordReq)
-      req.cmd := MemoryOpConstants.M_XRD
+      req.cmd := MemoryOpConstants.M_XA_ADD
       req.addr := addr
-      req.data := DontCare
+      req.data := 0.U
       req.mask := Fill(req.mask.getWidth, true.B)
       req.meta := DontCare
       req
@@ -226,8 +260,18 @@ class L2CacheTest extends FlatSpec with ChiselScalatestTester with Matchers{
     implicit val p = Parameters((site, up, here) => {
       case L2CacheTestKey =>
         L2CacheTestParams()
+      case L3CacheTestKey =>
+        L3CacheTestParams()
     })
 
+    /*
+    test(LazyModule(new L2TestTopWrapper()).module)
+      .withAnnotations(Seq(
+        VerilatorBackendAnnotation,
+        RunFirrtlTransformAnnotation(new PrintModuleName)
+      )){ c =>
+        */
+
     test(LazyModule(new L2TestTopWrapper()).module)
       .withAnnotations(Seq(VerilatorBackendAnnotation)){ c =>
 
@@ -236,7 +280,7 @@ class L2CacheTest extends FlatSpec with ChiselScalatestTester with Matchers{
 
         c.clock.step(100)
 
-        for(i <- 0 until 100){
+        for(i <- 0 until 100000){
           val addr = Random.nextInt(0xfffff) & 0xffe00 // align to block size
           val data = Random.nextLong() & 0x7fffffffffffffffL
           c.io.in.enqueue(chiselTypeOf(c.io.in.bits).Lit(