DCacheWrapper.scala 26.9 KB
Newer Older
L
Lemover 已提交
1 2
/***************************************************************************************
* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
Y
Yinan Xu 已提交
3
* Copyright (c) 2020-2021 Peng Cheng Laboratory
L
Lemover 已提交
4 5 6 7 8 9 10 11 12 13 14 15 16
*
* XiangShan is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
*          http://license.coscl.org.cn/MulanPSL2
*
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
*
* See the Mulan PSL v2 for more details.
***************************************************************************************/

A
Allen 已提交
17 18 19 20
package xiangshan.cache

import chipsalliance.rocketchip.config.Parameters
import chisel3._
21
import chisel3.experimental.ExtModule
A
Allen 已提交
22 23 24 25
import chisel3.util._
import xiangshan._
import utils._
import freechips.rocketchip.diplomacy.{IdRange, LazyModule, LazyModuleImp, TransferSizes}
26
import freechips.rocketchip.tilelink._
J
Jiawei Lin 已提交
27
import freechips.rocketchip.util.{BundleFieldBase, UIntToOH1}
28
import device.RAMHelper
J
Jiawei Lin 已提交
29 30
import huancun.{AliasField, AliasKey, DirtyField, PreferCacheField, PrefetchField}

Z
zhanglinjuan 已提交
31
import scala.math.max
32 33 34 35 36 37 38 39 40

// DCache specific parameters
case class DCacheParameters
(
  nSets: Int = 256,
  nWays: Int = 8,
  rowBits: Int = 128,
  tagECC: Option[String] = None,
  dataECC: Option[String] = None,
W
William Wang 已提交
41
  replacer: Option[String] = Some("setplru"),
42 43 44 45 46
  nMissEntries: Int = 1,
  nProbeEntries: Int = 1,
  nReleaseEntries: Int = 1,
  nMMIOEntries: Int = 1,
  nMMIOs: Int = 1,
47 48
  blockBytes: Int = 64,
  alwaysReleaseData: Boolean = true
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
) extends L1CacheParameters {
  // if sets * blockBytes > 4KB(page size),
  // cache alias will happen,
  // we need to avoid this by recoding additional bits in L2 cache
  val setBytes = nSets * blockBytes
  val aliasBitsOpt = if(setBytes > pageSize) Some(log2Ceil(setBytes / pageSize)) else None
  val reqFields: Seq[BundleFieldBase] = Seq(
    PrefetchField(),
    PreferCacheField()
  ) ++ aliasBitsOpt.map(AliasField)
  val echoFields: Seq[BundleFieldBase] = Seq(DirtyField())

  def tagCode: Code = Code.fromString(tagECC)

  def dataCode: Code = Code.fromString(dataECC)
}

//           Physical Address
// --------------------------------------
// |   Physical Tag |  PIndex  | Offset |
// --------------------------------------
//                  |
//                  DCacheTagOffset
Y
Yinan Xu 已提交
72
//
73 74 75 76 77
//           Virtual Address
// --------------------------------------
// | Above index  | Set | Bank | Offset |
// --------------------------------------
//                |     |      |        |
78
//                |     |      |        0
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
//                |     |      DCacheBankOffset
//                |     DCacheSetOffset
//                DCacheAboveIndexOffset

// Default DCache size = 64 sets * 8 ways * 8 banks * 8 Byte = 32K Byte

trait HasDCacheParameters extends HasL1CacheParameters {
  val cacheParams = dcacheParameters
  val cfg = cacheParams

  def encWordBits = cacheParams.dataCode.width(wordBits)

  def encRowBits = encWordBits * rowWords // for DuplicatedDataArray only
  def eccBits = encWordBits - wordBits

94 95 96
  def encTagBits = cacheParams.tagCode.width(tagBits)
  def eccTagBits = encTagBits - tagBits

97 98 99 100 101 102 103 104 105
  def lrscCycles = LRSCCycles // ISA requires 16-insn LRSC sequences to succeed
  def lrscBackoff = 3 // disallow LRSC reacquisition briefly
  def blockProbeAfterGrantCycles = 8 // give the processor some time to issue a request after a grant

  def nSourceType = 3
  def sourceTypeWidth = log2Up(nSourceType)
  def LOAD_SOURCE = 0
  def STORE_SOURCE = 1
  def AMO_SOURCE = 2
106
  def SOFT_PREFETCH = 3
107 108 109 110

  // each source use a id to distinguish its multiple reqs
  def reqIdWidth = 64

W
William Wang 已提交
111 112 113 114 115
  require(isPow2(cfg.nMissEntries)) // TODO
  // require(isPow2(cfg.nReleaseEntries))
  require(cfg.nMissEntries < cfg.nReleaseEntries)
  val nEntries = cfg.nMissEntries + cfg.nReleaseEntries
  val releaseIdBase = cfg.nMissEntries
Z
zhanglinjuan 已提交
116

117 118 119 120 121
  // banked dcache support
  val DCacheSets = cacheParams.nSets
  val DCacheWays = cacheParams.nWays
  val DCacheBanks = 8
  val DCacheSRAMRowBits = 64 // hardcoded
122 123
  val DCacheWordBits = 64 // hardcoded
  val DCacheWordBytes = DCacheWordBits / 8
124

125 126 127
  val DCacheSizeBits = DCacheSRAMRowBits * DCacheBanks * DCacheWays * DCacheSets
  val DCacheSizeBytes = DCacheSizeBits / 8
  val DCacheSizeWords = DCacheSizeBits / 64 // TODO
128 129 130 131

  val DCacheSameVPAddrLength = 12

  val DCacheSRAMRowBytes = DCacheSRAMRowBits / 8
132 133 134
  val DCacheWordOffset = log2Up(DCacheWordBytes)

  val DCacheBankOffset = log2Up(DCacheSRAMRowBytes)
135 136 137
  val DCacheSetOffset = DCacheBankOffset + log2Up(DCacheBanks)
  val DCacheAboveIndexOffset = DCacheSetOffset + log2Up(DCacheSets)
  val DCacheTagOffset = DCacheAboveIndexOffset min DCacheSameVPAddrLength
138
  val DCacheLineOffset = DCacheSetOffset
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
  val DCacheIndexOffset = DCacheBankOffset

  def addr_to_dcache_bank(addr: UInt) = {
    require(addr.getWidth >= DCacheSetOffset)
    addr(DCacheSetOffset-1, DCacheBankOffset)
  }

  def addr_to_dcache_set(addr: UInt) = {
    require(addr.getWidth >= DCacheAboveIndexOffset)
    addr(DCacheAboveIndexOffset-1, DCacheSetOffset)
  }

  def get_data_of_bank(bank: Int, data: UInt) = {
    require(data.getWidth >= (bank+1)*DCacheSRAMRowBits)
    data(DCacheSRAMRowBits * (bank + 1) - 1, DCacheSRAMRowBits * bank)
  }

  def get_mask_of_bank(bank: Int, data: UInt) = {
    require(data.getWidth >= (bank+1)*DCacheSRAMRowBytes)
    data(DCacheSRAMRowBytes * (bank + 1) - 1, DCacheSRAMRowBytes * bank)
  }

Z
zhanglinjuan 已提交
161 162
  val numReplaceRespPorts = 2

163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
  require(isPow2(nSets), s"nSets($nSets) must be pow2")
  require(isPow2(nWays), s"nWays($nWays) must be pow2")
  require(full_divide(rowBits, wordBits), s"rowBits($rowBits) must be multiple of wordBits($wordBits)")
  require(full_divide(beatBits, rowBits), s"beatBits($beatBits) must be multiple of rowBits($rowBits)")
}

abstract class DCacheModule(implicit p: Parameters) extends L1CacheModule
  with HasDCacheParameters

abstract class DCacheBundle(implicit p: Parameters) extends L1CacheBundle
  with HasDCacheParameters

class ReplacementAccessBundle(implicit p: Parameters) extends DCacheBundle {
  val set = UInt(log2Up(nSets).W)
  val way = UInt(log2Up(nWays).W)
}
A
Allen 已提交
179

Z
zhanglinjuan 已提交
180 181 182 183 184
class ReplacementWayReqIO(implicit p: Parameters) extends DCacheBundle {
  val set = ValidIO(UInt(log2Up(nSets).W))
  val way = Input(UInt(log2Up(nWays).W))
}

A
Allen 已提交
185
// memory request in word granularity(load, mmio, lr/sc, atomics)
186
class DCacheWordReq(implicit p: Parameters)  extends DCacheBundle
A
Allen 已提交
187 188 189 190 191 192
{
  val cmd    = UInt(M_SZ.W)
  val addr   = UInt(PAddrBits.W)
  val data   = UInt(DataBits.W)
  val mask   = UInt((DataBits/8).W)
  val id     = UInt(reqIdWidth.W)
193
  val instrtype   = UInt(sourceTypeWidth.W)
A
Allen 已提交
194 195 196 197 198 199 200
  def dump() = {
    XSDebug("DCacheWordReq: cmd: %x addr: %x data: %x mask: %x id: %d\n",
      cmd, addr, data, mask, id)
  }
}

// memory request in word granularity(store)
201
class DCacheLineReq(implicit p: Parameters)  extends DCacheBundle
A
Allen 已提交
202 203
{
  val cmd    = UInt(M_SZ.W)
204
  val vaddr  = UInt(VAddrBits.W)
A
Allen 已提交
205 206 207 208 209 210 211 212
  val addr   = UInt(PAddrBits.W)
  val data   = UInt((cfg.blockBytes * 8).W)
  val mask   = UInt(cfg.blockBytes.W)
  val id     = UInt(reqIdWidth.W)
  def dump() = {
    XSDebug("DCacheLineReq: cmd: %x addr: %x data: %x mask: %x id: %d\n",
      cmd, addr, data, mask, id)
  }
Z
zhanglinjuan 已提交
213
  def idx: UInt = get_idx(vaddr)
A
Allen 已提交
214 215
}

216 217
class DCacheWordReqWithVaddr(implicit p: Parameters) extends DCacheWordReq {
  val vaddr = UInt(VAddrBits.W)
218
  val wline = Bool()
219 220
}

221
class DCacheWordResp(implicit p: Parameters) extends DCacheBundle
A
Allen 已提交
222 223 224 225 226
{
  val data         = UInt(DataBits.W)
  // cache req missed, send it to miss queue
  val miss   = Bool()
  // cache req nacked, replay it later
227 228
  val miss_enter = Bool()
  // cache miss, and enter the missqueue successfully. just for softprefetch
A
Allen 已提交
229 230 231 232 233 234 235 236
  val replay = Bool()
  val id     = UInt(reqIdWidth.W)
  def dump() = {
    XSDebug("DCacheWordResp: data: %x id: %d miss: %b replay: %b\n",
      data, id, miss, replay)
  }
}

237
class DCacheLineResp(implicit p: Parameters) extends DCacheBundle
A
Allen 已提交
238 239 240 241 242 243 244 245 246 247 248 249 250
{
  val data   = UInt((cfg.blockBytes * 8).W)
  // cache req missed, send it to miss queue
  val miss   = Bool()
  // cache req nacked, replay it later
  val replay = Bool()
  val id     = UInt(reqIdWidth.W)
  def dump() = {
    XSDebug("DCacheLineResp: data: %x id: %d miss: %b replay: %b\n",
      data, id, miss, replay)
  }
}

251
class Refill(implicit p: Parameters) extends DCacheBundle
A
Allen 已提交
252 253
{
  val addr   = UInt(PAddrBits.W)
254
  val data   = UInt(l1BusDataWidth.W)
255 256 257 258
  // for debug usage
  val data_raw = UInt((cfg.blockBytes * 8).W)
  val hasdata = Bool()
  val refill_done = Bool()
A
Allen 已提交
259 260 261 262 263
  def dump() = {
    XSDebug("Refill: addr: %x data: %x\n", addr, data)
  }
}

W
William Wang 已提交
264 265 266 267 268 269 270 271
class Release(implicit p: Parameters) extends DCacheBundle
{
  val paddr  = UInt(PAddrBits.W)
  def dump() = {
    XSDebug("Release: paddr: %x\n", paddr(PAddrBits-1, DCacheTagOffset))
  }
}

272
class DCacheWordIO(implicit p: Parameters) extends DCacheBundle
A
Allen 已提交
273 274 275 276 277
{
  val req  = DecoupledIO(new DCacheWordReq)
  val resp = Flipped(DecoupledIO(new DCacheWordResp))
}

278 279 280 281 282 283
class DCacheWordIOWithVaddr(implicit p: Parameters) extends DCacheBundle
{
  val req  = DecoupledIO(new DCacheWordReqWithVaddr)
  val resp = Flipped(DecoupledIO(new DCacheWordResp))
}

A
Allen 已提交
284
// used by load unit
285
class DCacheLoadIO(implicit p: Parameters) extends DCacheWordIO
286 287 288
{
  // kill previous cycle's req
  val s1_kill  = Output(Bool())
L
Lemover 已提交
289
  val s2_kill  = Output(Bool())
290 291 292
  // cycle 0: virtual address: req.addr
  // cycle 1: physical address: s1_paddr
  val s1_paddr = Output(UInt(PAddrBits.W))
293
  val s1_hit_way = Input(UInt(nWays.W))
294
  val s1_disable_fast_wakeup = Input(Bool())
295
  val s1_bank_conflict = Input(Bool())
296 297
}

298
class DCacheLineIO(implicit p: Parameters) extends DCacheBundle
A
Allen 已提交
299
{
300
  val req  = DecoupledIO(new DCacheLineReq)
A
Allen 已提交
301 302 303
  val resp = Flipped(DecoupledIO(new DCacheLineResp))
}

Z
zhanglinjuan 已提交
304 305 306 307 308 309 310 311 312 313 314 315
class DCacheToSbufferIO(implicit p: Parameters) extends DCacheBundle { 
  // sbuffer will directly send request to dcache main pipe
  val req = Flipped(Decoupled(new DCacheLineReq))

  val main_pipe_hit_resp = ValidIO(new DCacheLineResp)
  val refill_hit_resp = ValidIO(new DCacheLineResp)

  val replay_resp = ValidIO(new DCacheLineResp)

  def hit_resps: Seq[ValidIO[DCacheLineResp]] = Seq(main_pipe_hit_resp, refill_hit_resp)
}

316
class DCacheToLsuIO(implicit p: Parameters) extends DCacheBundle {
A
Allen 已提交
317 318
  val load  = Vec(LoadPipelineWidth, Flipped(new DCacheLoadIO)) // for speculative load
  val lsq = ValidIO(new Refill)  // refill to load queue, wake up load misses
Z
zhanglinjuan 已提交
319
  val store = new DCacheToSbufferIO // for sbuffer
320
  val atomics  = Flipped(new DCacheWordIOWithVaddr)  // atomics reqs
W
William Wang 已提交
321
  val release = ValidIO(new Release) // cacheline release hint for ld-ld violation check 
A
Allen 已提交
322 323
}

324
class DCacheIO(implicit p: Parameters) extends DCacheBundle {
J
Jiawei Lin 已提交
325
  val hartId = Input(UInt(8.W))
A
Allen 已提交
326
  val lsu = new DCacheToLsuIO
327
  val csr = new L1CacheToCsrIO
328
  val error = new L1CacheErrorInfo
329
  val mshrFull = Output(Bool())
A
Allen 已提交
330 331 332 333 334 335 336 337
}


class DCache()(implicit p: Parameters) extends LazyModule with HasDCacheParameters {

  val clientParameters = TLMasterPortParameters.v1(
    Seq(TLMasterParameters.v1(
      name = "dcache",
Z
zhanglinjuan 已提交
338
      sourceId = IdRange(0, nEntries + 1),
A
Allen 已提交
339
      supportsProbe = TransferSizes(cfg.blockBytes)
340 341 342
    )),
    requestFields = cacheParams.reqFields,
    echoFields = cacheParams.echoFields
A
Allen 已提交
343 344 345 346 347 348 349 350
  )

  val clientNode = TLClientNode(Seq(clientParameters))

  lazy val module = new DCacheImp(this)
}


L
ljw 已提交
351
class DCacheImp(outer: DCache) extends LazyModuleImp(outer) with HasDCacheParameters {
A
Allen 已提交
352 353 354 355 356 357

  val io = IO(new DCacheIO)

  val (bus, edge) = outer.clientNode.out.head
  require(bus.d.bits.data.getWidth == l1BusDataWidth, "DCache: tilelink width does not match")

Y
Yinan Xu 已提交
358 359 360 361 362 363 364 365 366 367
  println("DCache:")
  println("  DCacheSets: " + DCacheSets)
  println("  DCacheWays: " + DCacheWays)
  println("  DCacheBanks: " + DCacheBanks)
  println("  DCacheSRAMRowBits: " + DCacheSRAMRowBits)
  println("  DCacheWordOffset: " + DCacheWordOffset)
  println("  DCacheBankOffset: " + DCacheBankOffset)
  println("  DCacheSetOffset: " + DCacheSetOffset)
  println("  DCacheTagOffset: " + DCacheTagOffset)
  println("  DCacheAboveIndexOffset: " + DCacheAboveIndexOffset)
368

A
Allen 已提交
369 370
  //----------------------------------------
  // core data structures
371
  val bankedDataArray = Module(new BankedDataArray)
Z
zhanglinjuan 已提交
372 373
  val metaArray = Module(new AsynchronousMetaArray(readPorts = 4, writePorts = 3))
  val tagArray = Module(new DuplicatedTagArray(readPorts = LoadPipelineWidth + 1))
374 375 376
  bankedDataArray.dump()

  val errors = bankedDataArray.io.errors ++ metaArray.io.errors
377
  io.error <> RegNext(Mux1H(errors.map(e => e.ecc_error.valid -> e)))
378
  // assert(!io.error.ecc_error.valid)
A
Allen 已提交
379 380 381

  //----------------------------------------
  // core modules
382
  val ldu = Seq.tabulate(LoadPipelineWidth)({ i => Module(new LoadPipe(i))})
A
Allen 已提交
383
  val atomicsReplayUnit = Module(new AtomicsReplayEntry)
384
  val mainPipe   = Module(new MainPipe)
Z
zhanglinjuan 已提交
385 386
  val refillPipe = Module(new RefillPipe)
  val replacePipe = Module(new ReplacePipe)
A
Allen 已提交
387 388
  val missQueue  = Module(new MissQueue(edge))
  val probeQueue = Module(new ProbeQueue(edge))
389
  val wb         = Module(new WritebackQueue(edge))
A
Allen 已提交
390

J
Jiawei Lin 已提交
391 392
  missQueue.io.hartId := io.hartId

A
Allen 已提交
393 394
  //----------------------------------------
  // meta array
Z
zhanglinjuan 已提交
395 396 397 398 399 400 401 402 403 404 405 406 407 408
  val meta_read_ports = ldu.map(_.io.meta_read) ++
    Seq(mainPipe.io.meta_read,
      replacePipe.io.meta_read)
  val meta_resp_ports = ldu.map(_.io.meta_resp) ++
    Seq(mainPipe.io.meta_resp,
      replacePipe.io.meta_resp)
  val meta_write_ports = Seq(
    mainPipe.io.meta_write,
    refillPipe.io.meta_write,
    replacePipe.io.meta_write
  )
  meta_read_ports.zip(metaArray.io.read).foreach { case (p, r) => r <> p }
  meta_resp_ports.zip(metaArray.io.resp).foreach { case (p, r) => p := r }
  meta_write_ports.zip(metaArray.io.write).foreach { case (p, w) => w <> p }
A
Allen 已提交
409

Z
zhanglinjuan 已提交
410 411 412 413 414 415 416
  //----------------------------------------
  // tag array
  require(tagArray.io.read.size == (ldu.size + 1))
  ldu.zipWithIndex.foreach {
    case (ld, i) =>
      tagArray.io.read(i) <> ld.io.tag_read
      ld.io.tag_resp := tagArray.io.resp(i)
A
Allen 已提交
417
  }
Z
zhanglinjuan 已提交
418 419 420 421 422 423 424
  tagArray.io.read.last <> mainPipe.io.tag_read
  mainPipe.io.tag_resp := tagArray.io.resp.last

  val tag_write_arb = Module(new Arbiter(new TagWriteReq, 2))
  tag_write_arb.io.in(0) <> refillPipe.io.tag_write
  tag_write_arb.io.in(1) <> mainPipe.io.tag_write
  tagArray.io.write <> tag_write_arb.io.out
A
Allen 已提交
425 426 427 428

  //----------------------------------------
  // data array

Z
zhanglinjuan 已提交
429 430 431 432 433 434 435 436 437
  val dataReadLineArb = Module(new Arbiter(new L1BankedDataReadLineReq, 2))
  dataReadLineArb.io.in(0) <> replacePipe.io.data_read
  dataReadLineArb.io.in(1) <> mainPipe.io.data_read

  val dataWriteArb = Module(new Arbiter(new L1BankedDataWriteReq, 2))
  dataWriteArb.io.in(0) <> refillPipe.io.data_write
  dataWriteArb.io.in(1) <> mainPipe.io.data_write

  bankedDataArray.io.write <> dataWriteArb.io.out
438 439
  bankedDataArray.io.read(0) <> ldu(0).io.banked_data_read
  bankedDataArray.io.read(1) <> ldu(1).io.banked_data_read
Z
zhanglinjuan 已提交
440
  bankedDataArray.io.readline <> dataReadLineArb.io.out
A
Allen 已提交
441

442 443
  ldu(0).io.banked_data_resp := bankedDataArray.io.resp
  ldu(1).io.banked_data_resp := bankedDataArray.io.resp
Z
zhanglinjuan 已提交
444 445
  mainPipe.io.data_resp := bankedDataArray.io.resp
  replacePipe.io.data_resp := bankedDataArray.io.resp
A
Allen 已提交
446

447 448 449 450
  ldu(0).io.bank_conflict_fast := bankedDataArray.io.bank_conflict_fast(0)
  ldu(1).io.bank_conflict_fast := bankedDataArray.io.bank_conflict_fast(1)
  ldu(0).io.bank_conflict_slow := bankedDataArray.io.bank_conflict_slow(0)
  ldu(1).io.bank_conflict_slow := bankedDataArray.io.bank_conflict_slow(1)
A
Allen 已提交
451 452 453 454 455 456

  //----------------------------------------
  // load pipe
  // the s1 kill signal
  // only lsu uses this, replay never kills
  for (w <- 0 until LoadPipelineWidth) {
457
    ldu(w).io.lsu <> io.lsu.load(w)
A
Allen 已提交
458 459 460 461

    // replay and nack not needed anymore
    // TODO: remove replay and nack
    ldu(w).io.nack := false.B
462

Y
Yinan Xu 已提交
463
    ldu(w).io.disable_ld_fast_wakeup :=
464
      bankedDataArray.io.bank_conflict_fast(w) // load pipe fast wake up should be disabled when bank conflict
A
Allen 已提交
465 466 467 468 469
  }

  //----------------------------------------
  // atomics
  // atomics not finished yet
A
Allen 已提交
470
  io.lsu.atomics <> atomicsReplayUnit.io.lsu
W
William Wang 已提交
471
  atomicsReplayUnit.io.pipe_resp := RegNext(mainPipe.io.atomic_resp)
A
Allen 已提交
472 473 474 475 476 477 478

  //----------------------------------------
  // miss queue
  val MissReqPortCount = LoadPipelineWidth + 1
  val MainPipeMissReqPort = 0

  // Request
W
William Wang 已提交
479
  val missReqArb = Module(new Arbiter(new MissReq, MissReqPortCount))
A
Allen 已提交
480

W
William Wang 已提交
481
  missReqArb.io.in(MainPipeMissReqPort) <> mainPipe.io.miss_req
A
Allen 已提交
482 483
  for (w <- 0 until LoadPipelineWidth) { missReqArb.io.in(w + 1) <> ldu(w).io.miss_req }

484 485 486
  wb.io.miss_req.valid := missReqArb.io.out.valid
  wb.io.miss_req.bits  := missReqArb.io.out.bits.addr

W
William Wang 已提交
487 488 489 490 491 492
  // block_decoupled(missReqArb.io.out, missQueue.io.req, wb.io.block_miss_req)
  missReqArb.io.out <> missQueue.io.req
  when(wb.io.block_miss_req) {
    missQueue.io.req.bits.cancel := true.B
    missReqArb.io.out.ready := false.B
  }
A
Allen 已提交
493 494

  // refill to load queue
Z
zhanglinjuan 已提交
495
  io.lsu.lsq <> missQueue.io.refill_to_ldq
A
Allen 已提交
496 497 498 499

  // tilelink stuff
  bus.a <> missQueue.io.mem_acquire
  bus.e <> missQueue.io.mem_finish
Z
zhanglinjuan 已提交
500 501
  missQueue.io.probe_addr := bus.b.bits.address

W
William Wang 已提交
502
  missQueue.io.main_pipe_resp := RegNext(mainPipe.io.atomic_resp)
A
Allen 已提交
503 504 505

  //----------------------------------------
  // probe
506 507
  // probeQueue.io.mem_probe <> bus.b
  block_decoupled(bus.b, probeQueue.io.mem_probe, missQueue.io.probe_block)
Z
zhanglinjuan 已提交
508
  probeQueue.io.lrsc_locked_block <> mainPipe.io.lrsc_locked_block
W
William Wang 已提交
509
  probeQueue.io.update_resv_set <> mainPipe.io.update_resv_set
A
Allen 已提交
510 511 512

  //----------------------------------------
  // mainPipe
Z
zhanglinjuan 已提交
513 514
  // when a req enters main pipe, if it is set-conflict with replace pipe or refill pipe,
  // block the req in main pipe
515
  val refillPipeStatus, replacePipeStatusS0 = Wire(Valid(UInt(idxBits.W)))
Z
zhanglinjuan 已提交
516
  refillPipeStatus.valid := refillPipe.io.req.valid
517 518 519
  refillPipeStatus.bits := get_idx(refillPipe.io.req.bits.paddrWithVirtualAlias)
  replacePipeStatusS0.valid := replacePipe.io.req.valid
  replacePipeStatusS0.bits := get_idx(replacePipe.io.req.bits.vaddr)
Z
zhanglinjuan 已提交
520
  val blockMainPipeReqs = Seq(
W
William Wang 已提交
521
	  replacePipeStatusS0,
Z
zhanglinjuan 已提交
522 523 524
    replacePipe.io.status.s1_set,
    replacePipe.io.status.s2_set
  )
W
William Wang 已提交
525 526
  val storeShouldBeBlocked = refillPipeStatus.valid || Cat(blockMainPipeReqs.map(r => r.valid && r.bits === io.lsu.store.req.bits.idx)).orR
  val probeShouldBeBlocked = refillPipeStatus.valid || Cat(blockMainPipeReqs.map(r => r.valid && r.bits === get_idx(probeQueue.io.pipe_req.bits.vaddr))).orR
A
Allen 已提交
527

Z
zhanglinjuan 已提交
528 529
  block_decoupled(probeQueue.io.pipe_req, mainPipe.io.probe_req, probeShouldBeBlocked)
  block_decoupled(io.lsu.store.req, mainPipe.io.store_req, storeShouldBeBlocked)
A
Allen 已提交
530

W
William Wang 已提交
531
  io.lsu.store.replay_resp := RegNext(mainPipe.io.store_replay_resp)
Z
zhanglinjuan 已提交
532 533 534 535 536 537 538
  io.lsu.store.main_pipe_hit_resp := mainPipe.io.store_hit_resp

  val mainPipeAtomicReqArb = Module(new Arbiter(new MainPipeReq, 2))
  mainPipeAtomicReqArb.io.in(0) <> missQueue.io.main_pipe_req
  mainPipeAtomicReqArb.io.in(1) <> atomicsReplayUnit.io.pipe_req
  mainPipe.io.atomic_req <> mainPipeAtomicReqArb.io.out

W
William Wang 已提交
539
  mainPipe.io.invalid_resv_set := RegNext(wb.io.req.fire && wb.io.req.bits.addr === mainPipe.io.lrsc_locked_block.bits)
Z
zhanglinjuan 已提交
540 541 542 543 544 545

  //----------------------------------------
  // replace pipe
  val mpStatus = mainPipe.io.status
  val replaceSet = addr_to_dcache_set(missQueue.io.replace_pipe_req.bits.vaddr)
  val replaceWayEn = missQueue.io.replace_pipe_req.bits.way_en
W
William Wang 已提交
546 547
  val replaceShouldBeBlocked = mpStatus.s1.valid ||
    Cat(Seq(mpStatus.s2, mpStatus.s3).map(s =>
Z
zhanglinjuan 已提交
548 549 550 551 552 553 554
      s.valid && s.bits.set === replaceSet && s.bits.way_en === replaceWayEn
    )).orR()
  block_decoupled(missQueue.io.replace_pipe_req, replacePipe.io.req, replaceShouldBeBlocked)
  missQueue.io.replace_pipe_resp := replacePipe.io.resp

  //----------------------------------------
  // refill pipe
555 556 557 558 559 560
  val refillShouldBeBlocked = (mpStatus.s1.valid && mpStatus.s1.bits.set === missQueue.io.refill_pipe_req.bits.idx) ||
    Cat(Seq(mpStatus.s2, mpStatus.s3).map(s =>
      s.valid &&
        s.bits.set === missQueue.io.refill_pipe_req.bits.idx &&
        s.bits.way_en === missQueue.io.refill_pipe_req.bits.way_en
    )).orR
Z
zhanglinjuan 已提交
561
  block_decoupled(missQueue.io.refill_pipe_req, refillPipe.io.req, refillShouldBeBlocked)
W
William Wang 已提交
562
  io.lsu.store.refill_hit_resp := RegNext(refillPipe.io.store_resp)
563

A
Allen 已提交
564 565 566
  //----------------------------------------
  // wb
  // add a queue between MainPipe and WritebackUnit to reduce MainPipe stalls due to WritebackUnit busy
Z
zhanglinjuan 已提交
567 568 569
  val wbArb = Module(new Arbiter(new WritebackReq, 2))
  wbArb.io.in.zip(Seq(mainPipe.io.wb, replacePipe.io.wb)).foreach { case (arb, pipe) => arb <> pipe }
  wb.io.req <> wbArb.io.out
A
Allen 已提交
570
  bus.c     <> wb.io.mem_release
Z
zhanglinjuan 已提交
571 572
  wb.io.release_wakeup := refillPipe.io.release_wakeup
  wb.io.release_update := mainPipe.io.release_update
W
William Wang 已提交
573 574
  io.lsu.release.valid := RegNext(bus.c.fire())
  io.lsu.release.bits.paddr := RegNext(bus.c.bits.address)
A
Allen 已提交
575

576
  // connect bus d
A
Allen 已提交
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
  missQueue.io.mem_grant.valid := false.B
  missQueue.io.mem_grant.bits  := DontCare

  wb.io.mem_grant.valid := false.B
  wb.io.mem_grant.bits  := DontCare

  // in L1DCache, we ony expect Grant[Data] and ReleaseAck
  bus.d.ready := false.B
  when (bus.d.bits.opcode === TLMessages.Grant || bus.d.bits.opcode === TLMessages.GrantData) {
    missQueue.io.mem_grant <> bus.d
  } .elsewhen (bus.d.bits.opcode === TLMessages.ReleaseAck) {
    wb.io.mem_grant <> bus.d
  } .otherwise {
    assert (!bus.d.fire())
  }

Z
zhanglinjuan 已提交
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616
  //----------------------------------------
  // replacement algorithm
  val replacer = ReplacementPolicy.fromString(cacheParams.replacer, nWays, nSets)

  val replWayReqs = ldu.map(_.io.replace_way) ++ Seq(mainPipe.io.replace_way)
  replWayReqs.foreach{
    case req =>
      req.way := DontCare
      when (req.set.valid) { req.way := replacer.way(req.set.bits) }
  }

  val replAccessReqs = ldu.map(_.io.replace_access) ++ Seq(
    mainPipe.io.replace_access,
    refillPipe.io.replace_access
  )
  val touchWays = Seq.fill(replAccessReqs.size)(Wire(ValidIO(UInt(log2Up(nWays).W))))
  touchWays.zip(replAccessReqs).foreach {
    case (w, req) =>
      w.valid := req.valid
      w.bits := req.bits.way
  }
  val touchSets = replAccessReqs.map(_.bits.set)
  replacer.access(touchSets, touchWays)

617 618
  //----------------------------------------
  // assertions
A
Allen 已提交
619 620 621 622 623 624 625 626 627 628 629
  // dcache should only deal with DRAM addresses
  when (bus.a.fire()) {
    assert(bus.a.bits.address >= 0x80000000L.U)
  }
  when (bus.b.fire()) {
    assert(bus.b.bits.address >= 0x80000000L.U)
  }
  when (bus.c.fire()) {
    assert(bus.c.bits.address >= 0x80000000L.U)
  }

630 631
  //----------------------------------------
  // utility functions
632 633 634 635 636
  def block_decoupled[T <: Data](source: DecoupledIO[T], sink: DecoupledIO[T], block_signal: Bool) = {
    sink.valid   := source.valid && !block_signal
    source.ready := sink.ready   && !block_signal
    sink.bits    := source.bits
  }
637

638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
  //----------------------------------------
  // Customized csr cache op support
  val cacheOpDecoder = Module(new CSRCacheOpDecoder("dcache", CacheInstrucion.COP_ID_DCACHE))
  cacheOpDecoder.io.csr <> io.csr
  bankedDataArray.io.cacheOp.req := cacheOpDecoder.io.cache.req
  metaArray.io.cacheOp.req := cacheOpDecoder.io.cache.req
  tagArray.io.cacheOp.req := cacheOpDecoder.io.cache.req
  cacheOpDecoder.io.cache.resp.valid := bankedDataArray.io.cacheOp.resp.valid ||
    metaArray.io.cacheOp.resp.valid ||
    tagArray.io.cacheOp.resp.valid
  cacheOpDecoder.io.cache.resp.bits := Mux1H(List(
    bankedDataArray.io.cacheOp.resp.valid -> bankedDataArray.io.cacheOp.resp.bits,
    metaArray.io.cacheOp.resp.valid -> metaArray.io.cacheOp.resp.bits,
    tagArray.io.cacheOp.resp.valid -> tagArray.io.cacheOp.resp.bits,
  ))
  assert(!((bankedDataArray.io.cacheOp.resp.valid +& metaArray.io.cacheOp.resp.valid +& tagArray.io.cacheOp.resp.valid) > 1.U))

655 656 657
  //----------------------------------------
  // performance counters
  val num_loads = PopCount(ldu.map(e => e.io.lsu.req.fire()))
658
  XSPerfAccumulate("num_loads", num_loads)
659 660

  io.mshrFull := missQueue.io.full
Z
zhanglinjuan 已提交
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685

  // performance counter
  val ld_access = Wire(Vec(LoadPipelineWidth, missQueue.io.debug_early_replace.last.cloneType))
  val st_access = Wire(ld_access.last.cloneType)
  ld_access.zip(ldu).foreach {
    case (a, u) =>
      a.valid := RegNext(u.io.lsu.req.fire()) && !u.io.lsu.s1_kill
      a.bits.idx := RegNext(get_idx(u.io.lsu.req.bits.addr))
      a.bits.tag := get_tag(u.io.lsu.s1_paddr)
  }
  st_access.valid := RegNext(mainPipe.io.store_req.fire())
  st_access.bits.idx := RegNext(get_idx(mainPipe.io.store_req.bits.vaddr))
  st_access.bits.tag := RegNext(get_tag(mainPipe.io.store_req.bits.addr))
  val access_info = ld_access.toSeq ++ Seq(st_access)
  val early_replace = RegNext(missQueue.io.debug_early_replace)
  val access_early_replace = access_info.map {
    case acc =>
      Cat(early_replace.map {
        case r =>
          acc.valid && r.valid &&
            acc.bits.tag === r.bits.tag &&
            acc.bits.idx === r.bits.idx
      })
  }
  XSPerfAccumulate("access_early_replace", PopCount(Cat(access_early_replace)))
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702

  val wb_perf      = wb.perfEvents.map(_._1).zip(wb.perfinfo.perfEvents.perf_events)
  val mainp_perf     = mainPipe.perfEvents.map(_._1).zip(mainPipe.perfinfo.perfEvents.perf_events)
  val missq_perf     = missQueue.perfEvents.map(_._1).zip(missQueue.perfinfo.perfEvents.perf_events)
  val probq_perf     = probeQueue.perfEvents.map(_._1).zip(probeQueue.perfinfo.perfEvents.perf_events)
  val ldu_0_perf     = ldu(0).perfEvents.map(_._1).zip(ldu(0).perfinfo.perfEvents.perf_events)
  val ldu_1_perf     = ldu(1).perfEvents.map(_._1).zip(ldu(1).perfinfo.perfEvents.perf_events)
  val perfEvents = wb_perf ++ mainp_perf ++ missq_perf ++ probq_perf ++ ldu_0_perf ++ ldu_1_perf
  val perflist = wb.perfinfo.perfEvents.perf_events ++ mainPipe.perfinfo.perfEvents.perf_events ++
                 missQueue.perfinfo.perfEvents.perf_events ++ probeQueue.perfinfo.perfEvents.perf_events ++
                 ldu(0).perfinfo.perfEvents.perf_events ++ ldu(1).perfinfo.perfEvents.perf_events
  val perf_length = perflist.length
  val perfinfo = IO(new Bundle(){
    val perfEvents = Output(new PerfEventsBundle(perflist.length))
  })
  perfinfo.perfEvents.perf_events := perflist

A
Allen 已提交
703
}
704

705
class AMOHelper() extends ExtModule {
J
Jiawei Lin 已提交
706 707 708 709 710 711 712
  val clock  = IO(Input(Clock()))
  val enable = IO(Input(Bool()))
  val cmd    = IO(Input(UInt(5.W)))
  val addr   = IO(Input(UInt(64.W)))
  val wdata  = IO(Input(UInt(64.W)))
  val mask   = IO(Input(UInt(8.W)))
  val rdata  = IO(Output(UInt(64.W)))
713 714
}

J
Jiawei Lin 已提交
715
class DCacheWrapper()(implicit p: Parameters) extends LazyModule with HasXSParameter {
716

J
Jiawei Lin 已提交
717 718 719 720
  val useDcache = coreParams.dcacheParametersOpt.nonEmpty
  val clientNode = if (useDcache) TLIdentityNode() else null
  val dcache = if (useDcache) LazyModule(new DCache()) else null
  if (useDcache) {
721 722 723 724 725
    clientNode := dcache.clientNode
  }

  lazy val module = new LazyModuleImp(this) {
    val io = IO(new DCacheIO)
726 727 728 729
    val perfinfo = IO(new Bundle(){
      val perfEvents = Output(new PerfEventsBundle(dcache.asInstanceOf[DCache].module.perf_length))
    })
    val perfEvents = dcache.asInstanceOf[DCache].module.perfEvents.map(_._1).zip(dcache.asInstanceOf[DCache].module.perfinfo.perfEvents.perf_events)
J
Jiawei Lin 已提交
730 731
    if (!useDcache) {
      // a fake dcache which uses dpi-c to access memory, only for debug usage!
732 733 734 735 736
      val fake_dcache = Module(new FakeDCache())
      io <> fake_dcache.io
    }
    else {
      io <> dcache.module.io
737
      perfinfo := dcache.asInstanceOf[DCache].module.perfinfo
738 739 740
    }
  }
}