From af04e3ac713187aa1e58d643afc02d6614718edf Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 20 Jan 2023 15:41:59 -0800
Subject: [PATCH 01/47] Relax HasTiles to support Attachable only

---
 .../groundtest/GroundTestSubsystem.scala      |  5 +--
 src/main/scala/subsystem/HasTiles.scala       | 31 ++++++++++---------
 .../scala/subsystem/RocketSubsystem.scala     |  8 ++++-
 3 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index ba3215f7f53..f5eb965a016 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -26,8 +26,9 @@ class GroundTestSubsystem(implicit p: Parameters)
 
   val tileStatusNodes = tiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
 
-  // no debug module
-  val debugNode = NullIntSyncSource()
+  val clintOpt = None
+  val debugOpt = None
+  val plicOpt = None
 
   override lazy val module = new GroundTestSubsystemModuleImp(this)
 }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 5c65ed4cd81..22c62fdfc18 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -5,7 +5,8 @@ package freechips.rocketchip.subsystem
 import chisel3._
 import chisel3.dontTouch
 import org.chipsalliance.cde.config.{Field, Parameters}
-import freechips.rocketchip.devices.tilelink.{BasicBusBlocker, BasicBusBlockerParams, CLINTConsts, PLICKey, CanHavePeripheryPLIC, CanHavePeripheryCLINT}
+import freechips.rocketchip.devices.tilelink._
+import freechips.rocketchip.devices.debug.{TLDebugModule}
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.tile._
@@ -98,11 +99,8 @@ case class TileSlavePortParams(
 /** These are sources of interrupts that are driven into the tile.
   * They need to be instantiated before tiles are attached to the subsystem containing them.
   */
-trait HasTileInterruptSources
-  extends CanHavePeripheryPLIC
-  with CanHavePeripheryCLINT
-  with InstantiatesTiles
-{ this: BaseSubsystem => // TODO ideally this bound would be softened to LazyModule
+trait HasTileInterruptSources extends InstantiatesTiles
+{ this: LazyModule with Attachable =>
   /** meipNode is used to create a single bit subsystem input in Configs without a PLIC */
   val meipNode = p(PLICKey) match {
     case Some(_) => None
@@ -137,7 +135,7 @@ trait HasTileInterruptSources
   * they may be either tied to a contant value or programmed during boot or reset.
   * They need to be instantiated before tiles are attached within the subsystem containing them.
   */
-trait HasTileInputConstants extends InstantiatesTiles { this: BaseSubsystem =>
+trait HasTileInputConstants extends InstantiatesTiles { this: LazyModule with Attachable =>
   /** tileHartIdNode is used to collect publishers and subscribers of hartids. */
   val tileHartIdNode = BundleBridgeEphemeralNode[UInt]()
 
@@ -228,9 +226,11 @@ trait DefaultTileContextType
   with HasTileInterruptSources
   with HasTileNotificationSinks
   with HasTileInputConstants
-{ this: BaseSubsystem =>
-  val debugNode: IntSyncOutwardNode
-} // TODO: ideally this bound would be softened to LazyModule
+{ this: LazyModule with Attachable =>
+  val debugOpt: Option[TLDebugModule]
+  val clintOpt: Option[CLINT]
+  val plicOpt: Option[TLPLIC]
+}
 
 /** Standardized interface by which parameterized tiles can be attached to contexts containing interconnect resources.
   *
@@ -291,7 +291,9 @@ trait CanAttachTile {
     //       we stub out missing interrupts with constant sources here.
 
     // 1. Debug interrupt is definitely asynchronous in all cases.
-    domain.tile.intInwardNode := domain { IntSyncAsyncCrossingSink(3) } := context.debugNode
+    domain.tile.intInwardNode :=
+      domain { IntSyncAsyncCrossingSink(3) } :=
+        context.debugOpt.map(_.intnode).getOrElse(IntSyncXbar() := NullIntSyncSource())
 
     // 2. The CLINT and PLIC output interrupts are synchronous to the TileLink bus clock,
     //    so might need to be synchronized depending on the Tile's crossing type.
@@ -400,7 +402,8 @@ case class CloneTileAttachParams(
 /** InstantiatesTiles adds a Config-urable sequence of tiles of any type
   *   to the subsystem class into which it is mixed.
   */
-trait InstantiatesTiles { this: BaseSubsystem =>
+trait InstantiatesTiles { this: LazyModule with Attachable =>
+  val location: HierarchicalLocation
   /** Record the order in which to instantiate all tiles, based on statically-assigned ids.
     *
     * Note that these ids, which are often used as the tiles' default hartid input,
@@ -426,8 +429,8 @@ trait InstantiatesTiles { this: BaseSubsystem =>
 }
 
 /** HasTiles instantiates and also connects a Config-urable sequence of tiles of any type to subsystem interconnect resources. */
-trait HasTiles extends InstantiatesTiles with HasCoreMonitorBundles with DefaultTileContextType
-{ this: BaseSubsystem => // TODO: ideally this bound would be softened to Attachable
+trait HasTiles extends HasCoreMonitorBundles with DefaultTileContextType
+{ this: LazyModule with Attachable =>
   implicit val p: Parameters
 
   // connect all the tiles to interconnect attachment points made available in this subsystem context
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index e9e1f555515..efef5c5b7df 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -7,6 +7,7 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.prci.{ResetCrossingType, NoResetCrossing}
 import freechips.rocketchip.tile._
 import freechips.rocketchip.devices.debug.{HasPeripheryDebug}
+import freechips.rocketchip.devices.tilelink.{CanHavePeripheryCLINT, CanHavePeripheryPLIC}
 
 case class RocketCrossingParams(
   crossingType: ClockCrossingType = SynchronousCrossing(),
@@ -30,9 +31,14 @@ trait HasRocketTiles extends HasTiles { this: BaseSubsystem =>
   }).toList
 }
 
-class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem with HasRocketTiles with HasPeripheryDebug {
+class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
+    with HasPeripheryDebug
+    with CanHavePeripheryCLINT
+    with CanHavePeripheryPLIC
+    with HasRocketTiles {
   override lazy val module = new RocketSubsystemModuleImp(this)
 }
 
 class RocketSubsystemModuleImp[+L <: RocketSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
     with HasTilesModuleImp
+

From d1b141cd12f36c1b773dacc696ab22d3af101308 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 20 Jan 2023 18:24:11 -0800
Subject: [PATCH 02/47] Force tiles to provide names

---
 src/main/scala/groundtest/TraceGen.scala       |  2 +-
 src/main/scala/subsystem/HasTiles.scala        | 12 ++++++------
 src/main/scala/subsystem/RocketSubsystem.scala |  3 ++-
 src/main/scala/tile/BaseTile.scala             |  2 +-
 src/main/scala/tile/RocketTile.scala           |  2 +-
 5 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/main/scala/groundtest/TraceGen.scala b/src/main/scala/groundtest/TraceGen.scala
index 5c5cba9b5f7..bd991656f7f 100644
--- a/src/main/scala/groundtest/TraceGen.scala
+++ b/src/main/scala/groundtest/TraceGen.scala
@@ -76,7 +76,7 @@ case class TraceGenParams(
   }
   val beuAddr = None
   val blockerCtrlAddr = None
-  val name = None
+  val name = s"tracegen_$hartId"
   val clockSinkParams = ClockSinkParameters()
 }
 
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 22c62fdfc18..c96c1566c6d 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -246,7 +246,7 @@ trait CanAttachTile {
 
   /** Narrow waist through which all tiles are intended to pass while being instantiated. */
   def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: Seq[TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
-    val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(s"${tileParams.name.getOrElse("core")}_${tileParams.hartId}"))
+    val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.name))
     val tile_prci_domain = LazyModule(new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
       val tile = self.tile_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
     })
@@ -267,7 +267,7 @@ trait CanAttachTile {
   def connectMasterPorts(domain: TilePRCIDomain[TileType], context: Attachable): Unit = {
     implicit val p = context.p
     val dataBus = context.locateTLBusWrapper(crossingParams.master.where)
-    dataBus.coupleFrom(tileParams.name.getOrElse("tile")) { bus =>
+    dataBus.coupleFrom(tileParams.name) { bus =>
       bus :=* crossingParams.master.injectNode(context) :=* domain.crossMasterPort(crossingParams.crossingType)
     }
   }
@@ -277,7 +277,7 @@ trait CanAttachTile {
     implicit val p = context.p
     DisableMonitors { implicit p =>
       val controlBus = context.locateTLBusWrapper(crossingParams.slave.where)
-      controlBus.coupleTo(tileParams.name.getOrElse("tile")) { bus =>
+      controlBus.coupleTo(tileParams.name) { bus =>
         domain.crossSlavePort(crossingParams.crossingType) :*= crossingParams.slave.injectNode(context) :*= TLWidthWidget(controlBus.beatBytes) :*= bus
       }
     }
@@ -384,17 +384,17 @@ case class CloneTileAttachParams(
 
   def tileParams = cloneParams.tileParams
   def crossingParams = cloneParams.crossingParams
-  require(sourceHart < tileParams.hartId)
 
   override def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: Seq[TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
-    val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(s"${tileParams.name.getOrElse("core")}_${tileParams.hartId}"))
+    require(sourceHart < instantiatedTiles.size)
+    val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.name))
     val tile_prci_domain = CloneLazyModule(
       new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
         val tile = self.tile_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
       },
       instantiatedTiles(sourceHart).asInstanceOf[TilePRCIDomain[TileType]]
     )
-    tile_prci_domain 
+    tile_prci_domain
   }
 }
 
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index efef5c5b7df..fac4eeb4c68 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -35,7 +35,8 @@ class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
     with HasPeripheryDebug
     with CanHavePeripheryCLINT
     with CanHavePeripheryPLIC
-    with HasRocketTiles {
+    with HasRocketTiles
+{
   override lazy val module = new RocketSubsystemModuleImp(this)
 }
 
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index df20aa0aa23..c29db319c2b 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -26,7 +26,7 @@ trait TileParams {
   val hartId: Int
   val beuAddr: Option[BigInt]
   val blockerCtrlAddr: Option[BigInt]
-  val name: Option[String]
+  val name: String
   val clockSinkParams: ClockSinkParameters
 }
 
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index 2527e135e1d..1d765a28e41 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -22,7 +22,6 @@ case class RocketTileParams(
     dcache: Option[DCacheParams] = Some(DCacheParams()),
     btb: Option[BTBParams] = Some(BTBParams()),
     dataScratchpadBytes: Int = 0,
-    name: Option[String] = Some("tile"),
     hartId: Int = 0,
     beuAddr: Option[BigInt] = None,
     blockerCtrlAddr: Option[BigInt] = None,
@@ -31,6 +30,7 @@ case class RocketTileParams(
     ) extends InstantiableTileParams[RocketTile] {
   require(icache.isDefined)
   require(dcache.isDefined)
+  val name = s"rockettile_$hartId"
   def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
     new RocketTile(this, crossing, lookup)
   }

From 71eb59fecf0b1e64870ed734cc2e29737f402967 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 20 Jan 2023 18:37:34 -0800
Subject: [PATCH 03/47] HasCoreIO to HasRocketCoreIO

---
 src/main/scala/groundtest/TraceGen.scala |  1 -
 src/main/scala/rocket/CSR.scala          |  9 ++++----
 src/main/scala/rocket/RocketCore.scala   | 29 ++++++++++++++++++++++--
 src/main/scala/tile/BaseTile.scala       |  1 -
 src/main/scala/tile/Core.scala           | 24 --------------------
 5 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/src/main/scala/groundtest/TraceGen.scala b/src/main/scala/groundtest/TraceGen.scala
index bd991656f7f..eca2e6d4461 100644
--- a/src/main/scala/groundtest/TraceGen.scala
+++ b/src/main/scala/groundtest/TraceGen.scala
@@ -74,7 +74,6 @@ case class TraceGenParams(
   def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
     new TraceGenTile(this, crossing, lookup)
   }
-  val beuAddr = None
   val blockerCtrlAddr = None
   val name = s"tracegen_$hartId"
   val clockSinkParams = ClockSinkParameters()
diff --git a/src/main/scala/rocket/CSR.scala b/src/main/scala/rocket/CSR.scala
index e8cd587efde..8aa9d3736a5 100644
--- a/src/main/scala/rocket/CSR.scala
+++ b/src/main/scala/rocket/CSR.scala
@@ -254,10 +254,10 @@ class CSRDecodeIO(implicit p: Parameters) extends CoreBundle {
   val virtual_system_illegal = Output(Bool())
 }
 
-class CSRFileIO(implicit p: Parameters) extends CoreBundle
+class CSRFileIO(hasBeu: Boolean)(implicit p: Parameters) extends CoreBundle
     with HasCoreParameters {
   val ungated_clock = Input(Clock())
-  val interrupts = Input(new CoreInterrupts())
+  val interrupts = Input(new CoreInterrupts(hasBeu))
   val hartid = Input(UInt(hartIdLen.W))
   val rw = new Bundle {
     val addr = Input(UInt(CSR.ADDRSZ.W))
@@ -375,10 +375,11 @@ class VType(implicit p: Parameters) extends CoreBundle {
 class CSRFile(
   perfEventSets: EventSets = new EventSets(Seq()),
   customCSRs: Seq[CustomCSR] = Nil,
-  roccCSRs: Seq[CustomCSR] = Nil)(implicit p: Parameters)
+  roccCSRs: Seq[CustomCSR] = Nil,
+  hasBeu: Boolean = false)(implicit p: Parameters)
     extends CoreModule()(p)
     with HasCoreParameters {
-  val io = IO(new CSRFileIO {
+  val io = IO(new CSRFileIO(hasBeu) {
     val customCSRs = Vec(CSRFile.this.customCSRs.size, new CustomCSRIO)
     val roccCSRs = Vec(CSRFile.this.roccCSRs.size, new CustomCSRIO)
   })
diff --git a/src/main/scala/rocket/RocketCore.scala b/src/main/scala/rocket/RocketCore.scala
index 34f16d273c3..6946a643723 100644
--- a/src/main/scala/rocket/RocketCore.scala
+++ b/src/main/scala/rocket/RocketCore.scala
@@ -118,9 +118,34 @@ class RocketCustomCSRs(implicit p: Parameters) extends CustomCSRs with HasRocket
   override def decls = super.decls :+ marchid :+ mvendorid :+ mimpid
 }
 
+class CoreInterrupts(val hasBeu: Boolean)(implicit p: Parameters) extends TileInterrupts()(p) {
+  val buserror = Option.when(hasBeu)(Bool())
+}
+
+trait HasRocketCoreIO extends HasRocketCoreParameters {
+  implicit val p: Parameters
+  def nTotalRoCCCSRs: Int
+  val io = IO(new CoreBundle()(p) {
+    val hartid = Input(UInt(hartIdLen.W))
+    val reset_vector = Input(UInt(resetVectorLen.W))
+    val interrupts = Input(new CoreInterrupts(tileParams.asInstanceOf[RocketTileParams].beuAddr.isDefined))
+    val imem  = new FrontendIO
+    val dmem = new HellaCacheIO
+    val ptw = Flipped(new DatapathPTWIO())
+    val fpu = Flipped(new FPUCoreIO())
+    val rocc = Flipped(new RoCCCoreIO(nTotalRoCCCSRs))
+    val trace = Output(new TraceBundle)
+    val bpwatch = Output(Vec(coreParams.nBreakpoints, new BPWatch(coreParams.retireWidth)))
+    val cease = Output(Bool())
+    val wfi = Output(Bool())
+    val traceStall = Input(Bool())
+  })
+}
+
+
 class Rocket(tile: RocketTile)(implicit p: Parameters) extends CoreModule()(p)
     with HasRocketCoreParameters
-    with HasCoreIO {
+    with HasRocketCoreIO {
   def nTotalRoCCCSRs = tile.roccCSRs.flatten.size
 
   val clock_en_reg = RegInit(true.B)
@@ -305,7 +330,7 @@ class Rocket(tile: RocketTile)(implicit p: Parameters) extends CoreModule()(p)
   val ctrl_killd = Wire(Bool())
   val id_npc = (ibuf.io.pc.asSInt + ImmGen(IMM_UJ, id_inst(0))).asUInt
 
-  val csr = Module(new CSRFile(perfEvents, coreParams.customCSRs.decls, tile.roccCSRs.flatten))
+  val csr = Module(new CSRFile(perfEvents, coreParams.customCSRs.decls, tile.roccCSRs.flatten, tile.rocketParams.beuAddr.isDefined))
   val id_csr_en = id_ctrl.csr.isOneOf(CSR.S, CSR.C, CSR.W)
   val id_system_insn = id_ctrl.csr === CSR.I
   val id_csr_ren = id_ctrl.csr.isOneOf(CSR.S, CSR.C) && id_expanded_inst(0).rs1 === 0.U
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index c29db319c2b..9704598ddc0 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -24,7 +24,6 @@ trait TileParams {
   val dcache: Option[DCacheParams]
   val btb: Option[BTBParams]
   val hartId: Int
-  val beuAddr: Option[BigInt]
   val blockerCtrlAddr: Option[BigInt]
   val name: String
   val clockSinkParams: ClockSinkParameters
diff --git a/src/main/scala/tile/Core.scala b/src/main/scala/tile/Core.scala
index bb4d7abf6ae..12b4fade904 100644
--- a/src/main/scala/tile/Core.scala
+++ b/src/main/scala/tile/Core.scala
@@ -145,33 +145,9 @@ abstract class CoreModule(implicit val p: Parameters) extends Module
 abstract class CoreBundle(implicit val p: Parameters) extends ParameterizedBundle()(p)
   with HasCoreParameters
 
-class CoreInterrupts(implicit p: Parameters) extends TileInterrupts()(p) {
-  val buserror = tileParams.beuAddr.map(a => Bool())
-}
-
 // This is a raw commit trace from the core, not the TraceCoreInterface
 class TraceBundle(implicit val p: Parameters) extends Bundle with HasCoreParameters {
   val insns = Vec(coreParams.retireWidth, new TracedInstruction)
   val time = UInt(64.W)
   val custom = coreParams.traceCustom
 }
-
-trait HasCoreIO extends HasTileParameters {
-  implicit val p: Parameters
-  def nTotalRoCCCSRs: Int
-  val io = IO(new CoreBundle()(p) {
-    val hartid = Input(UInt(hartIdLen.W))
-    val reset_vector = Input(UInt(resetVectorLen.W))
-    val interrupts = Input(new CoreInterrupts())
-    val imem  = new FrontendIO
-    val dmem = new HellaCacheIO
-    val ptw = Flipped(new DatapathPTWIO())
-    val fpu = Flipped(new FPUCoreIO())
-    val rocc = Flipped(new RoCCCoreIO(nTotalRoCCCSRs))
-    val trace = Output(new TraceBundle)
-    val bpwatch = Output(Vec(coreParams.nBreakpoints, new BPWatch(coreParams.retireWidth)))
-    val cease = Output(Bool())
-    val wfi = Output(Bool())
-    val traceStall = Input(Bool())
-  })
-}

From 6b7644566d03cf07cfd9e18270184c6d87adb6f3 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 20 Jan 2023 22:19:00 -0800
Subject: [PATCH 04/47] Add abstract Element class

---
 src/main/scala/groundtest/TraceGen.scala      |  8 ++--
 src/main/scala/subsystem/Element.scala        | 47 +++++++++++++++++++
 src/main/scala/subsystem/HasTiles.scala       | 30 ++----------
 .../scala/subsystem/RocketSubsystem.scala     |  4 +-
 src/main/scala/tile/BaseTile.scala            | 10 ++--
 src/main/scala/tile/RocketTile.scala          |  6 +--
 src/main/scala/tile/TilePRCIDomain.scala      |  4 +-
 7 files changed, 66 insertions(+), 43 deletions(-)
 create mode 100644 src/main/scala/subsystem/Element.scala

diff --git a/src/main/scala/groundtest/TraceGen.scala b/src/main/scala/groundtest/TraceGen.scala
index eca2e6d4461..1b4457f50b7 100644
--- a/src/main/scala/groundtest/TraceGen.scala
+++ b/src/main/scala/groundtest/TraceGen.scala
@@ -26,7 +26,7 @@ import freechips.rocketchip.diplomacy.{ClockCrossingType}
 import freechips.rocketchip.rocket._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.subsystem.{TileCrossingParamsLike, CanAttachTile}
+import freechips.rocketchip.subsystem.{ElementCrossingParamsLike, CanAttachTile}
 import freechips.rocketchip.util._
 import freechips.rocketchip.prci.{ClockSinkParameters}
 
@@ -71,7 +71,7 @@ case class TraceGenParams(
     hartId: Int = 0
 ) extends InstantiableTileParams[TraceGenTile] with GroundTestTileParams
 {
-  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
+  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
     new TraceGenTile(this, crossing, lookup)
   }
   val blockerCtrlAddr = None
@@ -104,7 +104,7 @@ trait HasTraceGenParams {
 
 case class TraceGenTileAttachParams(
   tileParams: TraceGenParams,
-  crossingParams: TileCrossingParamsLike
+  crossingParams: ElementCrossingParamsLike
 ) extends CanAttachTile {
   type TileType = TraceGenTile
   val lookup: LookupByHartIdImpl = HartsWontDeduplicate(tileParams)
@@ -616,7 +616,7 @@ class TraceGenTile private(
   q: Parameters
 ) extends GroundTestTile(params, crossing, lookup, q)
 {
-  def this(params: TraceGenParams, crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: TraceGenParams, crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   val masterNode: TLOutwardNode = TLIdentityNode() := visibilityNode := dcacheOpt.map(_.node).getOrElse(TLTempNode())
diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
new file mode 100644
index 00000000000..02d27c98a42
--- /dev/null
+++ b/src/main/scala/subsystem/Element.scala
@@ -0,0 +1,47 @@
+package freechips.rocketchip.subsystem
+
+import chisel3._
+import chisel3.util._
+
+import org.chipsalliance.cde.config.Parameters
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.interrupts._
+import freechips.rocketchip.prci._
+import freechips.rocketchip.tile.{LookupByHartIdImpl}
+import freechips.rocketchip.subsystem._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.devices.debug.{TLDebugModule}
+import freechips.rocketchip.devices.tilelink._
+
+trait ElementParams {
+  val name: String
+  val clockSinkParams: ClockSinkParameters
+}
+
+abstract class InstantiableElementParams[ElementType <: LazyModule] extends ElementParams {
+  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): ElementType
+}
+
+/** An interface for describing the parameteization of how Elements are connected to interconnects */
+trait ElementCrossingParamsLike {
+  /** The type of clock crossing that should be inserted at the element boundary. */
+  def crossingType: ClockCrossingType
+  /** Parameters describing the contents and behavior of the point where the element is attached as an interconnect master. */
+  def master: ElementPortParamsLike
+  /** Parameters describing the contents and behavior of the point where the element is attached as an interconnect slave. */
+  def slave: ElementPortParamsLike
+  /** The subnetwork location of the device selecting the apparent base address of MMIO devices inside the element */
+  def mmioBaseAddressPrefixWhere: TLBusWrapperLocation
+  /** Inject a reset management subgraph that effects the element child reset only */
+  def resetCrossingType: ResetCrossingType
+  /** Keep the element clock separate from the interconnect clock (e.g. even if they are synchronous to one another) */
+  def forceSeparateClockReset: Boolean
+}
+
+/** An interface for describing the parameterization of how a particular element port is connected to an interconnect */
+trait ElementPortParamsLike {
+  /** The subnetwork location of the interconnect to which this element port should be connected. */
+  def where: TLBusWrapperLocation
+  /** Allows port-specific adapters to be injected into the interconnect side of the attachment point. */
+  def injectNode(context: Attachable)(implicit p: Parameters): TLNode
+}
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index c96c1566c6d..6b8ebea3a02 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -37,30 +37,6 @@ case object SubsystemExternalHartIdWidthKey extends Field[Option[Int]](None)
   */
 case object SubsystemExternalResetVectorKey extends Field[Boolean](true)
 
-/** An interface for describing the parameteization of how Tiles are connected to interconnects */
-trait TileCrossingParamsLike {
-  /** The type of clock crossing that should be inserted at the tile boundary. */
-  def crossingType: ClockCrossingType
-  /** Parameters describing the contents and behavior of the point where the tile is attached as an interconnect master. */
-  def master: TilePortParamsLike
-  /** Parameters describing the contents and behavior of the point where the tile is attached as an interconnect slave. */
-  def slave: TilePortParamsLike
-  /** The subnetwork location of the device selecting the apparent base address of MMIO devices inside the tile */
-  def mmioBaseAddressPrefixWhere: TLBusWrapperLocation
-  /** Inject a reset management subgraph that effects the tile child reset only */
-  def resetCrossingType: ResetCrossingType
-  /** Keep the tile clock separate from the interconnect clock (e.g. even if they are synchronous to one another) */
-  def forceSeparateClockReset: Boolean
-}
-
-/** An interface for describing the parameterization of how a particular tile port is connected to an interconnect */
-trait TilePortParamsLike {
-  /** The subnetwork location of the interconnect to which this tile port should be connected. */
-  def where: TLBusWrapperLocation
-  /** Allows port-specific adapters to be injected into the interconnect side of the attachment point. */
-  def injectNode(context: Attachable)(implicit p: Parameters): TLNode
-}
-
 /** A default implementation of parameterizing the connectivity of the port where the tile is the master.
   *   Optional timing buffers and/or an optional CacheCork can be inserted in the interconnect's clock domain.
   */
@@ -68,7 +44,7 @@ case class TileMasterPortParams(
   buffers: Int = 0,
   cork: Option[Boolean] = None,
   where: TLBusWrapperLocation = SBUS
-) extends TilePortParamsLike {
+) extends ElementPortParamsLike {
   def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
     (TLBuffer.chainNode(buffers) :=* cork.map { u => TLCacheCork(unsafe = u) } .getOrElse { TLTempNode() })
   }
@@ -82,7 +58,7 @@ case class TileSlavePortParams(
   blockerCtrlAddr: Option[BigInt] = None,
   blockerCtrlWhere: TLBusWrapperLocation = CBUS,
   where: TLBusWrapperLocation = CBUS
-) extends TilePortParamsLike {
+) extends ElementPortParamsLike {
   def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
     val controlBus = context.locateTLBusWrapper(where)
     val blockerBus = context.locateTLBusWrapper(blockerCtrlWhere)
@@ -242,7 +218,7 @@ trait CanAttachTile {
   type TileType <: BaseTile
   type TileContextType <: DefaultTileContextType
   def tileParams: InstantiableTileParams[TileType]
-  def crossingParams: TileCrossingParamsLike
+  def crossingParams: ElementCrossingParamsLike
 
   /** Narrow waist through which all tiles are intended to pass while being instantiated. */
   def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: Seq[TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index fac4eeb4c68..a72785df427 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -11,12 +11,12 @@ import freechips.rocketchip.devices.tilelink.{CanHavePeripheryCLINT, CanHavePeri
 
 case class RocketCrossingParams(
   crossingType: ClockCrossingType = SynchronousCrossing(),
-  master: TilePortParamsLike = TileMasterPortParams(),
+  master: ElementPortParamsLike = TileMasterPortParams(),
   slave: TileSlavePortParams = TileSlavePortParams(),
   mmioBaseAddressPrefixWhere: TLBusWrapperLocation = CBUS,
   resetCrossingType: ResetCrossingType = NoResetCrossing(),
   forceSeparateClockReset: Boolean = false
-) extends TileCrossingParamsLike
+) extends ElementCrossingParamsLike
 
 case class RocketTileAttachParams(
   tileParams: RocketTileParams,
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 9704598ddc0..3803ba36272 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -18,19 +18,19 @@ case object TileVisibilityNodeKey extends Field[TLEphemeralNode]
 case object TileKey extends Field[TileParams]
 case object LookupByHartId extends Field[LookupByHartIdImpl]
 
-trait TileParams {
+trait TileParams extends ElementParams {
   val core: CoreParams
   val icache: Option[ICacheParams]
   val dcache: Option[DCacheParams]
   val btb: Option[BTBParams]
   val hartId: Int
   val blockerCtrlAddr: Option[BigInt]
-  val name: String
-  val clockSinkParams: ClockSinkParameters
 }
 
-abstract class InstantiableTileParams[TileType <: BaseTile] extends TileParams {
-  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)
+abstract class InstantiableTileParams[TileType <: BaseTile]
+    extends InstantiableElementParams[TileType]
+    with TileParams {
+  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)
                  (implicit p: Parameters): TileType
 }
 
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index 1d765a28e41..bee8fff538e 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -10,7 +10,7 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.rocket._
-import freechips.rocketchip.subsystem.TileCrossingParamsLike
+import freechips.rocketchip.subsystem.ElementCrossingParamsLike
 import freechips.rocketchip.util._
 import freechips.rocketchip.prci.{ClockSinkParameters}
 
@@ -31,7 +31,7 @@ case class RocketTileParams(
   require(icache.isDefined)
   require(dcache.isDefined)
   val name = s"rockettile_$hartId"
-  def instantiate(crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
+  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
     new RocketTile(this, crossing, lookup)
   }
 }
@@ -49,7 +49,7 @@ class RocketTile private(
     with HasICacheFrontend
 {
   // Private constructor ensures altered LazyModule.p is used implicitly
-  def this(params: RocketTileParams, crossing: TileCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: RocketTileParams, crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   val intOutwardNode = IntIdentityNode()
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index 73972bec4de..ac2b05b2cea 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -8,7 +8,7 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.prci._
 import freechips.rocketchip.rocket.{TracedInstruction}
-import freechips.rocketchip.subsystem.{TileCrossingParamsLike, CrossesToOnlyOneResetDomain}
+import freechips.rocketchip.subsystem.{ElementCrossingParamsLike, CrossesToOnlyOneResetDomain}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util.{TraceCoreInterface}
 
@@ -37,7 +37,7 @@ class TileResetDomain(clockSinkParams: ClockSinkParameters, resetCrossingType: R
   */
 abstract class TilePRCIDomain[T <: BaseTile](
   clockSinkParams: ClockSinkParameters,
-  crossingParams: TileCrossingParamsLike)
+  crossingParams: ElementCrossingParamsLike)
   (implicit p: Parameters)
     extends ClockDomain
 {

From 4b95f260796eea4889318bcee168d84cab062534 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 20 Jan 2023 22:29:50 -0800
Subject: [PATCH 05/47] Move more stuff to element

---
 src/main/scala/subsystem/Element.scala | 24 +++++++++++++++++++++++-
 src/main/scala/tile/BaseTile.scala     | 19 +++----------------
 2 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
index 02d27c98a42..594fcde1d8e 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/Element.scala
@@ -18,7 +18,7 @@ trait ElementParams {
   val clockSinkParams: ClockSinkParameters
 }
 
-abstract class InstantiableElementParams[ElementType <: LazyModule] extends ElementParams {
+abstract class InstantiableElementParams[ElementType <: BaseElement] extends ElementParams {
   def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): ElementType
 }
 
@@ -45,3 +45,25 @@ trait ElementPortParamsLike {
   /** Allows port-specific adapters to be injected into the interconnect side of the attachment point. */
   def injectNode(context: Attachable)(implicit p: Parameters): TLNode
 }
+
+abstract class BaseElement (val crossing: ClockCrossingType)(implicit p: Parameters)
+    extends LazyModule()(p)
+    with CrossesToOnlyOneClockDomain
+{
+  def module: BaseElementModuleImp[BaseElement]
+  def masterNode: TLOutwardNode
+  def slaveNode: TLInwardNode
+  def intInwardNode: IntInwardNode    // Interrupts to the core from external devices
+  def intOutwardNode: IntOutwardNode  // Interrupts from tile-internal devices (e.g. BEU)
+  def haltNode: IntOutwardNode        // Unrecoverable error has occurred; suggest reset
+  def ceaseNode: IntOutwardNode       // Tile has ceased to retire instructions
+  def wfiNode: IntOutwardNode         // Tile is waiting for an interrupt
+
+  protected val tlOtherMastersNode = TLIdentityNode()
+  protected val tlMasterXbar = LazyModule(new TLXbar)
+  protected val tlSlaveXbar = LazyModule(new TLXbar)
+  protected val intXbar = LazyModule(new IntXbar)
+
+}
+
+abstract class BaseElementModuleImp[+L <: BaseElement](val outer: L) extends LazyModuleImp(outer)
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 3803ba36272..79e675a4c48 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -195,9 +195,8 @@ trait HasTileParameters extends HasNonDiplomaticTileParameters {
 }
 
 /** Base class for all Tiles that use TileLink */
-abstract class BaseTile private (val crossing: ClockCrossingType, q: Parameters)
-    extends LazyModule()(q)
-    with CrossesToOnlyOneClockDomain
+abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
+    extends BaseElement(crossing)(q)
     with HasNonDiplomaticTileParameters
 {
   // Public constructor alters Parameters to supply some legacy compatibility keys
@@ -210,18 +209,6 @@ abstract class BaseTile private (val crossing: ClockCrossingType, q: Parameters)
   }
 
   def module: BaseTileModuleImp[BaseTile]
-  def masterNode: TLOutwardNode
-  def slaveNode: TLInwardNode
-  def intInwardNode: IntInwardNode    // Interrupts to the core from external devices
-  def intOutwardNode: IntOutwardNode  // Interrupts from tile-internal devices (e.g. BEU)
-  def haltNode: IntOutwardNode        // Unrecoverable error has occurred; suggest reset
-  def ceaseNode: IntOutwardNode       // Tile has ceased to retire instructions
-  def wfiNode: IntOutwardNode         // Tile is waiting for an interrupt
-
-  protected val tlOtherMastersNode = TLIdentityNode()
-  protected val tlMasterXbar = LazyModule(new TLXbar)
-  protected val tlSlaveXbar = LazyModule(new TLXbar)
-  protected val intXbar = LazyModule(new IntXbar)
 
   /** Node for broadcasting a hart id to diplomatic consumers within the tile. */
   val hartIdNexusNode: BundleBridgeNode[UInt] = BundleBroadcast[UInt](registered = p(InsertTimingClosureRegistersOnHartIds))
@@ -380,4 +367,4 @@ abstract class BaseTile private (val crossing: ClockCrossingType, q: Parameters)
   this.suggestName(tileParams.name)
 }
 
-abstract class BaseTileModuleImp[+L <: BaseTile](val outer: L) extends LazyModuleImp(outer) with HasTileParameters
+abstract class BaseTileModuleImp[+L <: BaseTile](outer: L) extends BaseElementModuleImp[L](outer)

From eec3127a5149962269a602756a70b4f463ac3ac3 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 10:25:04 -0800
Subject: [PATCH 06/47] Generalize more things to Element

---
 src/main/scala/subsystem/Configs.scala        |  4 +-
 src/main/scala/subsystem/Element.scala        | 29 ++++++--
 src/main/scala/subsystem/HasTiles.scala       | 68 +++++--------------
 .../scala/subsystem/RocketSubsystem.scala     |  4 +-
 src/main/scala/tile/BaseTile.scala            | 17 +----
 src/main/scala/tile/TilePRCIDomain.scala      | 20 +++---
 6 files changed, 58 insertions(+), 84 deletions(-)

diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 7b4a8368ac1..9c753482d58 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -210,7 +210,7 @@ class With1TinyCore extends Config((site, here, up) => {
       tiny,
       RocketCrossingParams(
         crossingType = SynchronousCrossing(),
-        master = TileMasterPortParams())
+        master = ElementMasterPortParams())
     ))
   }
 })
@@ -291,7 +291,7 @@ class WithIncoherentTiles extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
     case tp: RocketTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
       master = tp.crossingParams.master match {
-        case x: TileMasterPortParams => x.copy(cork = Some(true))
+        case x: ElementMasterPortParams => x.copy(cork = Some(true))
         case _ => throw new Exception("Unrecognized type for RocketCrossingParams.master")
       }))
     case t => t
diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
index 594fcde1d8e..fdb8e9958c6 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/Element.scala
@@ -10,17 +10,16 @@ import freechips.rocketchip.prci._
 import freechips.rocketchip.tile.{LookupByHartIdImpl}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tilelink._
+import freechips.rocketchip.util._
+import freechips.rocketchip.rocket.{TracedInstruction}
 import freechips.rocketchip.devices.debug.{TLDebugModule}
 import freechips.rocketchip.devices.tilelink._
 
 trait ElementParams {
   val name: String
-  val clockSinkParams: ClockSinkParameters
 }
 
-abstract class InstantiableElementParams[ElementType <: BaseElement] extends ElementParams {
-  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): ElementType
-}
+abstract class InstantiableElementParams[ElementType <: BaseElement] extends ElementParams
 
 /** An interface for describing the parameteization of how Elements are connected to interconnects */
 trait ElementCrossingParamsLike {
@@ -64,6 +63,28 @@ abstract class BaseElement (val crossing: ClockCrossingType)(implicit p: Paramet
   protected val tlSlaveXbar = LazyModule(new TLXbar)
   protected val intXbar = LazyModule(new IntXbar)
 
+  val traceCoreNode: BundleBridgeOutwardNode[TraceCoreInterface]
+  val traceNode: BundleBridgeOutwardNode[Vec[TracedInstruction]]
+
+
+  /** Helper function to insert additional buffers on master ports at the boundary of the tile.
+    *
+    * The boundary buffering needed to cut feed-through paths is
+    * microarchitecture specific, so this may need to be overridden
+    * in subclasses of this class.
+    */
+  def makeMasterBoundaryBuffers(crossing: ClockCrossingType)(implicit p: Parameters) = TLBuffer(BufferParams.none)
+
+  /** Helper function to insert additional buffers on slave ports at the boundary of the tile.
+    *
+    * The boundary buffering needed to cut feed-through paths is
+    * microarchitecture specific, so this may need to be overridden
+    * in subclasses of this class.
+    */
+  def makeSlaveBoundaryBuffers(crossing: ClockCrossingType)(implicit p: Parameters) = TLBuffer(BufferParams.none)
+
+
 }
 
 abstract class BaseElementModuleImp[+L <: BaseElement](val outer: L) extends LazyModuleImp(outer)
+
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 6b8ebea3a02..44370f0944a 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -37,40 +37,6 @@ case object SubsystemExternalHartIdWidthKey extends Field[Option[Int]](None)
   */
 case object SubsystemExternalResetVectorKey extends Field[Boolean](true)
 
-/** A default implementation of parameterizing the connectivity of the port where the tile is the master.
-  *   Optional timing buffers and/or an optional CacheCork can be inserted in the interconnect's clock domain.
-  */
-case class TileMasterPortParams(
-  buffers: Int = 0,
-  cork: Option[Boolean] = None,
-  where: TLBusWrapperLocation = SBUS
-) extends ElementPortParamsLike {
-  def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
-    (TLBuffer.chainNode(buffers) :=* cork.map { u => TLCacheCork(unsafe = u) } .getOrElse { TLTempNode() })
-  }
-}
-
-/** A default implementation of parameterizing the connectivity of the port giving access to slaves inside the tile.
-  *   Optional timing buffers and/or an optional BusBlocker adapter can be inserted in the interconnect's clock domain.
-  */
-case class TileSlavePortParams(
-  buffers: Int = 0,
-  blockerCtrlAddr: Option[BigInt] = None,
-  blockerCtrlWhere: TLBusWrapperLocation = CBUS,
-  where: TLBusWrapperLocation = CBUS
-) extends ElementPortParamsLike {
-  def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
-    val controlBus = context.locateTLBusWrapper(where)
-    val blockerBus = context.locateTLBusWrapper(blockerCtrlWhere)
-    blockerCtrlAddr
-      .map { BasicBusBlockerParams(_, blockerBus.beatBytes, controlBus.beatBytes) }
-      .map { bbbp =>
-        val blocker = LazyModule(new BasicBusBlocker(bbbp))
-        blockerBus.coupleTo("tile_slave_port_bus_blocker") { blocker.controlNode := TLFragmenter(blockerBus) := _ }
-        blocker.node :*= TLBuffer.chainNode(buffers)
-      } .getOrElse { TLBuffer.chainNode(buffers) }
-  }
-}
 
 /** These are sources of interrupts that are driven into the tile.
   * They need to be instantiated before tiles are attached to the subsystem containing them.
@@ -224,7 +190,7 @@ trait CanAttachTile {
   def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: Seq[TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
     val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.name))
     val tile_prci_domain = LazyModule(new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
-      val tile = self.tile_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
+      val element = self.tile_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
     })
     tile_prci_domain
   }
@@ -267,9 +233,10 @@ trait CanAttachTile {
     //       we stub out missing interrupts with constant sources here.
 
     // 1. Debug interrupt is definitely asynchronous in all cases.
-    domain.tile.intInwardNode :=
-      domain { IntSyncAsyncCrossingSink(3) } :=
-        context.debugOpt.map(_.intnode).getOrElse(IntSyncXbar() := NullIntSyncSource())
+    domain.element.intInwardNode :=
+      context.debugOpt
+        .map { domain { IntSyncAsyncCrossingSink(3) } := _.intnode }
+        .getOrElse { NullIntSource() }
 
     // 2. The CLINT and PLIC output interrupts are synchronous to the TileLink bus clock,
     //    so might need to be synchronized depending on the Tile's crossing type.
@@ -285,33 +252,33 @@ trait CanAttachTile {
         .getOrElse { context.meipNode.get }
 
     //    From PLIC: "seip" (only if supervisor mode is enabled)
-    if (domain.tile.tileParams.core.hasSupervisorMode) {
+    if (domain.element.tileParams.core.hasSupervisorMode) {
       domain.crossIntIn(crossingParams.crossingType) :=
         context.plicOpt .map { _.intnode }
           .getOrElse { context.seipNode.get }
     }
 
     // 3. Local Interrupts ("lip") are required to already be synchronous to the Tile's clock.
-    // (they are connected to domain.tile.intInwardNode in a seperate trait)
+    // (they are connected to domain.element.intInwardNode in a seperate trait)
 
     // 4. Interrupts coming out of the tile are sent to the PLIC,
     //    so might need to be synchronized depending on the Tile's crossing type.
     context.plicOpt.foreach { plic =>
       FlipRendering { implicit p =>
-        plic.intnode :=* domain.crossIntOut(crossingParams.crossingType, domain.tile.intOutwardNode)
+        plic.intnode :=* domain.crossIntOut(crossingParams.crossingType, domain.element.intOutwardNode)
       }
     }
 
     // 5. Connect NMI inputs to the tile. These inputs are synchronous to the respective core_clock.
-    domain.tile.nmiNode := context.tileNMINode
+    domain.element.nmiNode := context.tileNMINode
   }
 
   /** Notifications of tile status are connected to be broadcast without needing to be clock-crossed. */
   def connectOutputNotifications(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
     implicit val p = context.p
-    context.tileHaltXbarNode  :=* domain.crossIntOut(NoCrossing, domain.tile.haltNode)
-    context.tileWFIXbarNode   :=* domain.crossIntOut(NoCrossing, domain.tile.wfiNode)
-    context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.tile.ceaseNode)
+    context.tileHaltXbarNode  :=* domain.crossIntOut(NoCrossing, domain.element.haltNode)
+    context.tileWFIXbarNode   :=* domain.crossIntOut(NoCrossing, domain.element.wfiNode)
+    context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.element.ceaseNode)
     // TODO should context be forced to have a trace sink connected here?
     //      for now this just ensures domain.trace[Core]Node has been crossed without connecting it externally
     domain.crossTracesOut()
@@ -321,9 +288,9 @@ trait CanAttachTile {
   def connectInputConstants(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
     implicit val p = context.p
     val tlBusToGetPrefixFrom = context.locateTLBusWrapper(crossingParams.mmioBaseAddressPrefixWhere)
-    domain.tile.hartIdNode := context.tileHartIdNode
-    domain.tile.resetVectorNode := context.tileResetVectorNode
-    tlBusToGetPrefixFrom.prefixNode.foreach { domain.tile.mmioAddressPrefixNode := _ }
+    domain.element.hartIdNode := context.tileHartIdNode
+    domain.element.resetVectorNode := context.tileResetVectorNode
+    tlBusToGetPrefixFrom.prefixNode.foreach { domain.element.mmioAddressPrefixNode := _ }
   }
 
   /** Connect power/reset/clock resources. */
@@ -366,7 +333,7 @@ case class CloneTileAttachParams(
     val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.name))
     val tile_prci_domain = CloneLazyModule(
       new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
-        val tile = self.tile_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
+        val element = self.tile_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
       },
       instantiatedTiles(sourceHart).asInstanceOf[TilePRCIDomain[TileType]]
     )
@@ -380,6 +347,7 @@ case class CloneTileAttachParams(
   */
 trait InstantiatesTiles { this: LazyModule with Attachable =>
   val location: HierarchicalLocation
+
   /** Record the order in which to instantiate all tiles, based on statically-assigned ids.
     *
     * Note that these ids, which are often used as the tiles' default hartid input,
@@ -394,7 +362,7 @@ trait InstantiatesTiles { this: LazyModule with Attachable =>
     case (instantiated, params) => instantiated :+ params.instantiate(tileParams, instantiated)(p)
   }
 
-  val tiles: Seq[BaseTile] = tile_prci_domains.map(_.tile.asInstanceOf[BaseTile])
+  val tiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile])
 
   // Helper functions for accessing certain parameters that are popular to refer to in subsystem code
   def nTiles: Int = tileAttachParams.size
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index a72785df427..5427c8bace1 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -11,8 +11,8 @@ import freechips.rocketchip.devices.tilelink.{CanHavePeripheryCLINT, CanHavePeri
 
 case class RocketCrossingParams(
   crossingType: ClockCrossingType = SynchronousCrossing(),
-  master: ElementPortParamsLike = TileMasterPortParams(),
-  slave: TileSlavePortParams = TileSlavePortParams(),
+  master: ElementPortParamsLike = ElementMasterPortParams(),
+  slave: ElementSlavePortParams = ElementSlavePortParams(),
   mmioBaseAddressPrefixWhere: TLBusWrapperLocation = CBUS,
   resetCrossingType: ResetCrossingType = NoResetCrossing(),
   forceSeparateClockReset: Boolean = false
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 79e675a4c48..a405ac61166 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -25,6 +25,7 @@ trait TileParams extends ElementParams {
   val btb: Option[BTBParams]
   val hartId: Int
   val blockerCtrlAddr: Option[BigInt]
+  val clockSinkParams: ClockSinkParameters
 }
 
 abstract class InstantiableTileParams[TileType <: BaseTile]
@@ -338,22 +339,6 @@ abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
       "hardware-exec-breakpoint-count" -> tileParams.core.nBreakpoints.asProperty
   )
 
-  /** Helper function to insert additional buffers on master ports at the boundary of the tile.
-    *
-    * The boundary buffering needed to cut feed-through paths is
-    * microarchitecture specific, so this may need to be overridden
-    * in subclasses of this class.
-    */
-  def makeMasterBoundaryBuffers(crossing: ClockCrossingType)(implicit p: Parameters) = TLBuffer(BufferParams.none)
-
-  /** Helper function to insert additional buffers on slave ports at the boundary of the tile.
-    *
-    * The boundary buffering needed to cut feed-through paths is
-    * microarchitecture specific, so this may need to be overridden
-    * in subclasses of this class.
-    */
- def makeSlaveBoundaryBuffers(crossing: ClockCrossingType)(implicit p: Parameters) = TLBuffer(BufferParams.none)
-
   /** Can be used to access derived params calculated by HasCoreParameters
     *
     * However, callers must ensure they do not access a diplomatically-determined parameter
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index ac2b05b2cea..6c621c540b3 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -8,7 +8,7 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.prci._
 import freechips.rocketchip.rocket.{TracedInstruction}
-import freechips.rocketchip.subsystem.{ElementCrossingParamsLike, CrossesToOnlyOneResetDomain}
+import freechips.rocketchip.subsystem.{ElementCrossingParamsLike, CrossesToOnlyOneResetDomain, BaseElement}
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util.{TraceCoreInterface}
 
@@ -35,13 +35,13 @@ class TileResetDomain(clockSinkParams: ClockSinkParameters, resetCrossingType: R
   * hierarchical P&R boundary buffers, core-local interrupt handling,
   * and any other IOs related to PRCI control.
   */
-abstract class TilePRCIDomain[T <: BaseTile](
+abstract class TilePRCIDomain[T <: BaseElement](
   clockSinkParams: ClockSinkParameters,
   crossingParams: ElementCrossingParamsLike)
   (implicit p: Parameters)
     extends ClockDomain
 {
-  val tile: T
+  val element: T
   val tile_reset_domain = LazyModule(new TileResetDomain(clockSinkParams, crossingParams.resetCrossingType))
   val tapClockNode = ClockIdentityNode()
   val clockNode = FixedClockBroadcast(None) :=* tapClockNode
@@ -59,18 +59,18 @@ abstract class TilePRCIDomain[T <: BaseTile](
     val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
       resetCrossingType = crossingParams.resetCrossingType,
       name = Some(traceSignalName))
-    traceNode :*= traceNexusNode := tile.traceNode
+    traceNode :*= traceNexusNode := element.traceNode
 
     val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
       resetCrossingType = crossingParams.resetCrossingType,
       name = Some(traceCoreSignalName))
-    traceCoreNode :*= traceCoreNexusNode := tile.traceCoreNode
+    traceCoreNode :*= traceCoreNexusNode := element.traceCoreNode
   }
 
   /** External code looking to connect and clock-cross the interrupts driven into this tile can call this. */
   def crossIntIn(crossingType: ClockCrossingType): IntInwardNode = {
     // Unlike the other crossing helpers, here nothing is is blocked during reset because we know these are inputs and assume that tile reset is longer than uncore reset
-    val intInClockXing = this.crossIn(tile.intInwardNode)
+    val intInClockXing = this.crossIn(element.intInwardNode)
     intInClockXing(crossingType)
   }
 
@@ -92,8 +92,8 @@ abstract class TilePRCIDomain[T <: BaseTile](
     */
   def crossSlavePort(crossingType: ClockCrossingType): TLInwardNode = { DisableMonitors { implicit p => FlipRendering { implicit p =>
     val tlSlaveResetXing = this {
-      tile_reset_domain.crossTLIn(tile.slaveNode) :*=
-        tile { tile.makeSlaveBoundaryBuffers(crossingType) }
+      tile_reset_domain.crossTLIn(element.slaveNode) :*=
+        element { element.makeSlaveBoundaryBuffers(crossingType) }
     }
     val tlSlaveClockXing = this.crossIn(tlSlaveResetXing)
     tlSlaveClockXing(crossingType)
@@ -104,8 +104,8 @@ abstract class TilePRCIDomain[T <: BaseTile](
     */
   def crossMasterPort(crossingType: ClockCrossingType): TLOutwardNode = {
     val tlMasterResetXing = this { DisableMonitors { implicit p =>
-      tile { tile.makeMasterBoundaryBuffers(crossingType) } :=*
-        tile_reset_domain.crossTLOut(tile.masterNode)
+      element { element.makeMasterBoundaryBuffers(crossingType) } :=*
+        tile_reset_domain.crossTLOut(element.masterNode)
     } }
     val tlMasterClockXing = this.crossOut(tlMasterResetXing)
     tlMasterClockXing(crossingType)

From 59a9b8f723121756b3007d79564e4f1c8d81cb3b Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 10:29:21 -0800
Subject: [PATCH 07/47] Rename SubsystemExternal to HasTiles

---
 src/main/scala/groundtest/Configs.scala |  2 +-
 src/main/scala/subsystem/Configs.scala  |  2 +-
 src/main/scala/subsystem/HasTiles.scala | 12 ++++++------
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/main/scala/groundtest/Configs.scala b/src/main/scala/groundtest/Configs.scala
index bb0ccd71ab5..4a4b37d2bcd 100644
--- a/src/main/scala/groundtest/Configs.scala
+++ b/src/main/scala/groundtest/Configs.scala
@@ -30,7 +30,7 @@ class GroundTestBaseConfig extends Config(
     case DebugModuleKey => None
     case CLINTKey => None
     case PLICKey => None
-    case SubsystemExternalResetVectorKey => true
+    case HasTilesExternalResetVectorKey => true
   })
 )
 
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 9c753482d58..9b654b5135f 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -38,7 +38,7 @@ class BaseSubsystemConfig extends Config ((site, here, up) => {
     blockBytes = site(CacheBlockBytes))
   // Additional device Parameters
   case BootROMLocated(InSubsystem) => Some(BootROMParams(contentFileName = "./bootrom/bootrom.img"))
-  case SubsystemExternalResetVectorKey => false
+  case HasTilesExternalResetVectorKey => false
   case DebugModuleKey => Some(DefaultDebugModuleParams(site(XLen)))
   case CLINTKey => Some(CLINTParams())
   case PLICKey => Some(PLICParams())
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 44370f0944a..7465c266ef5 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -25,17 +25,17 @@ case class TilesLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachTi
   */
 case object InsertTimingClosureRegistersOnHartIds extends Field[Boolean](false)
 
-/** Whether per-tile hart ids are going to be driven as inputs into the subsystem,
+/** Whether per-tile hart ids are going to be driven as inputs into a HasTiles block,
   * and if so, what their width should be.
   */
-case object SubsystemExternalHartIdWidthKey extends Field[Option[Int]](None)
+case object HasTilesExternalHartIdWidthKey extends Field[Option[Int]](None)
 
-/** Whether per-tile reset vectors are going to be driven as inputs into the subsystem.
+/** Whether per-tile reset vectors are going to be driven as inputs into a HasTiles block.
   *
   * Unlike the hart ids, the reset vector width is determined by the sinks within the tiles,
   * based on the size of the address map visible to the tiles.
   */
-case object SubsystemExternalResetVectorKey extends Field[Boolean](true)
+case object HasTilesExternalResetVectorKey extends Field[Boolean](true)
 
 
 /** These are sources of interrupts that are driven into the tile.
@@ -117,7 +117,7 @@ trait HasTileInputConstants extends InstantiatesTiles { this: LazyModule with At
     *
     *   Or, if such IOs are not configured to exist, tileHartIdNexusNode is used to supply an id to each tile.
     */
-  val tileHartIdIONodes: Seq[BundleBridgeSource[UInt]] = p(SubsystemExternalHartIdWidthKey) match {
+  val tileHartIdIONodes: Seq[BundleBridgeSource[UInt]] = p(HasTilesExternalHartIdWidthKey) match {
     case Some(w) => Seq.fill(tiles.size) {
       val hartIdSource = BundleBridgeSource(() => UInt(w.W))
       tileHartIdNode := hartIdSource
@@ -130,7 +130,7 @@ trait HasTileInputConstants extends InstantiatesTiles { this: LazyModule with At
     *
     *   Or, if such IOs are not configured to exist, tileResetVectorNexusNode is used to supply a single reset vector to every tile.
     */
-  val tileResetVectorIONodes: Seq[BundleBridgeSource[UInt]] = p(SubsystemExternalResetVectorKey) match {
+  val tileResetVectorIONodes: Seq[BundleBridgeSource[UInt]] = p(HasTilesExternalResetVectorKey) match {
     case true => Seq.fill(tiles.size) {
       val resetVectorSource = BundleBridgeSource[UInt]()
       tileResetVectorNode := resetVectorSource

From f7889ee6023ce9dad3fa34f1e90b7ca6ab6574b3 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 10:46:57 -0800
Subject: [PATCH 08/47] Generalize InstantiatesTiles to InstantiatesElements

---
 .../groundtest/GroundTestSubsystem.scala      |  2 +-
 src/main/scala/subsystem/HasTiles.scala       | 45 +++----------------
 .../scala/subsystem/RocketSubsystem.scala     |  2 +-
 3 files changed, 9 insertions(+), 40 deletions(-)

diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index f5eb965a016..0c8497b469f 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -24,7 +24,7 @@ class GroundTestSubsystem(implicit p: Parameters)
   // No PLIC in ground test; so just sink the interrupts to nowhere
   IntSinkNode(IntSinkPortSimple()) :=* ibus.toPLIC
 
-  val tileStatusNodes = tiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
+  val tileStatusNodes = totalTiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
 
   val clintOpt = None
   val debugOpt = None
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 7465c266ef5..0e17f0f35c0 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -41,7 +41,7 @@ case object HasTilesExternalResetVectorKey extends Field[Boolean](true)
 /** These are sources of interrupts that are driven into the tile.
   * They need to be instantiated before tiles are attached to the subsystem containing them.
   */
-trait HasTileInterruptSources extends InstantiatesTiles
+trait HasTileInterruptSources extends InstantiatesElements
 { this: LazyModule with Attachable =>
   /** meipNode is used to create a single bit subsystem input in Configs without a PLIC */
   val meipNode = p(PLICKey) match {
@@ -63,7 +63,7 @@ trait HasTileInterruptSources extends InstantiatesTiles
   /** Source of Non-maskable Interrupt (NMI) input bundle to each tile. */
   val tileNMINode = BundleBridgeEphemeralNode[NMI]()
   val tileNMIIONodes: Seq[BundleBridgeSource[NMI]] = {
-    Seq.fill(tiles.size) {
+    Seq.fill(nTotalTiles) {
       val nmiSource = BundleBridgeSource[NMI]()
       tileNMINode := nmiSource
       nmiSource
@@ -77,7 +77,7 @@ trait HasTileInterruptSources extends InstantiatesTiles
   * they may be either tied to a contant value or programmed during boot or reset.
   * They need to be instantiated before tiles are attached within the subsystem containing them.
   */
-trait HasTileInputConstants extends InstantiatesTiles { this: LazyModule with Attachable =>
+trait HasTileInputConstants extends InstantiatesElements { this: LazyModule with Attachable =>
   /** tileHartIdNode is used to collect publishers and subscribers of hartids. */
   val tileHartIdNode = BundleBridgeEphemeralNode[UInt]()
 
@@ -96,7 +96,7 @@ trait HasTileInputConstants extends InstantiatesTiles { this: LazyModule with At
   val tileHartIdNexusNode = LazyModule(new BundleBridgeNexus[UInt](
     inputFn = BundleBridgeNexus.orReduction[UInt](registered = p(InsertTimingClosureRegistersOnHartIds)) _,
     outputFn = (prefix: UInt, n: Int) =>  Seq.tabulate(n) { i =>
-      val y = dontTouch(prefix | hartIdList(i).U(p(MaxHartIdBits).W)) // dontTouch to keep constant prop from breaking tile dedup
+      val y = dontTouch(prefix | totalHartIdList(i).U(p(MaxHartIdBits).W)) // dontTouch to keep constant prop from breaking tile dedup
       if (p(InsertTimingClosureRegistersOnHartIds)) BundleBridgeNexus.safeRegNext(y) else y
     },
     default = Some(() => 0.U(p(MaxHartIdBits).W)),
@@ -118,7 +118,7 @@ trait HasTileInputConstants extends InstantiatesTiles { this: LazyModule with At
     *   Or, if such IOs are not configured to exist, tileHartIdNexusNode is used to supply an id to each tile.
     */
   val tileHartIdIONodes: Seq[BundleBridgeSource[UInt]] = p(HasTilesExternalHartIdWidthKey) match {
-    case Some(w) => Seq.fill(tiles.size) {
+    case Some(w) => Seq.fill(nTotalTiles) {
       val hartIdSource = BundleBridgeSource(() => UInt(w.W))
       tileHartIdNode := hartIdSource
       hartIdSource
@@ -131,7 +131,7 @@ trait HasTileInputConstants extends InstantiatesTiles { this: LazyModule with At
     *   Or, if such IOs are not configured to exist, tileResetVectorNexusNode is used to supply a single reset vector to every tile.
     */
   val tileResetVectorIONodes: Seq[BundleBridgeSource[UInt]] = p(HasTilesExternalResetVectorKey) match {
-    case true => Seq.fill(tiles.size) {
+    case true => Seq.fill(nTotalTiles) {
       val resetVectorSource = BundleBridgeSource[UInt]()
       tileResetVectorNode := resetVectorSource
       resetVectorSource
@@ -341,37 +341,6 @@ case class CloneTileAttachParams(
   }
 }
 
-
-/** InstantiatesTiles adds a Config-urable sequence of tiles of any type
-  *   to the subsystem class into which it is mixed.
-  */
-trait InstantiatesTiles { this: LazyModule with Attachable =>
-  val location: HierarchicalLocation
-
-  /** Record the order in which to instantiate all tiles, based on statically-assigned ids.
-    *
-    * Note that these ids, which are often used as the tiles' default hartid input,
-    * may or may not be those actually reflected at runtime in e.g. the $mhartid CSR
-    */
-  val tileAttachParams: Seq[CanAttachTile] = p(TilesLocated(location)).sortBy(_.tileParams.hartId)
-  val tileParams: Seq[TileParams] = tileAttachParams.map(_.tileParams)
-  val tileCrossingTypes: Seq[ClockCrossingType] = tileAttachParams.map(_.crossingParams.crossingType)
-
-  /** The actual list of instantiated tiles in this subsystem. */
-  val tile_prci_domains: Seq[TilePRCIDomain[_]] = tileAttachParams.foldLeft(Seq[TilePRCIDomain[_]]()) {
-    case (instantiated, params) => instantiated :+ params.instantiate(tileParams, instantiated)(p)
-  }
-
-  val tiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile])
-
-  // Helper functions for accessing certain parameters that are popular to refer to in subsystem code
-  def nTiles: Int = tileAttachParams.size
-  def hartIdList: Seq[Int] = tileParams.map(_.hartId)
-  def localIntCounts: Seq[Int] = tileParams.map(_.core.nLocalInterrupts)
-
-  require(hartIdList.distinct.size == tiles.size, s"Every tile must be statically assigned a unique id, but got:\n${hartIdList}")
-}
-
 /** HasTiles instantiates and also connects a Config-urable sequence of tiles of any type to subsystem interconnect resources. */
 trait HasTiles extends HasCoreMonitorBundles with DefaultTileContextType
 { this: LazyModule with Attachable =>
@@ -402,5 +371,5 @@ trait HasTilesModuleImp extends LazyModuleImp {
       (outer.seipNode.get.out(i)._1)(0) := pin
     }
   }
-  val nmi = outer.tiles.zip(outer.tileNMIIONodes).zipWithIndex.map { case ((tile, n), i) => tile.tileParams.core.useNMI.option(n.makeIO(s"nmi_$i")) }
+  val nmi = outer.totalTiles.zip(outer.tileNMIIONodes).zipWithIndex.map { case ((tile, n), i) => tile.tileParams.core.useNMI.option(n.makeIO(s"nmi_$i")) }
 }
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index 5427c8bace1..508146bcd86 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -24,7 +24,7 @@ case class RocketTileAttachParams(
 ) extends CanAttachTile { type TileType = RocketTile }
 
 trait HasRocketTiles extends HasTiles { this: BaseSubsystem =>
-  val rocketTiles = tiles.collect { case r: RocketTile => r }
+  val rocketTiles = totalTiles.collect { case r: RocketTile => r }
 
   def coreMonitorBundles = (rocketTiles map { t =>
     t.module.core.rocketImpl.coreMonitorBundle

From 293a0bc2ec8f2af854943c6a3232ecdf70e69240 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 10:57:26 -0800
Subject: [PATCH 09/47] Rename HasTilesModuleImp to HasTilesRootModuleImp

---
 src/main/scala/subsystem/HasTiles.scala        | 6 ++++--
 src/main/scala/subsystem/RocketSubsystem.scala | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 0e17f0f35c0..448365866c0 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -352,8 +352,10 @@ trait HasTiles extends HasCoreMonitorBundles with DefaultTileContextType
   }
 }
 
-/** Provides some Chisel connectivity to certain tile IOs */
-trait HasTilesModuleImp extends LazyModuleImp {
+/** Provides some Chisel connectivity to certain tile IOs
+  * This trait is intended for the root subsystem
+  */
+trait HasTilesRootModuleImp extends LazyModuleImp {
   val outer: HasTiles with HasTileInterruptSources with HasTileInputConstants
 
   val reset_vector = outer.tileResetVectorIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"reset_vector_$i") }
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index 508146bcd86..76ed297b3e6 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -41,5 +41,5 @@ class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
 }
 
 class RocketSubsystemModuleImp[+L <: RocketSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
-    with HasTilesModuleImp
+    with HasTilesRootModuleImp
 

From 37320f8a6e740583375bec736109cf94fcd508a1 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 11:04:47 -0800
Subject: [PATCH 10/47] HasTilesModuleImp to HasElementsModuleImp

---
 src/main/scala/devices/tilelink/BootROM.scala |  4 +--
 .../groundtest/GroundTestSubsystem.scala      |  4 +--
 src/main/scala/subsystem/HasTiles.scala       | 33 -------------------
 .../scala/subsystem/RocketSubsystem.scala     |  4 +--
 4 files changed, 6 insertions(+), 39 deletions(-)

diff --git a/src/main/scala/devices/tilelink/BootROM.scala b/src/main/scala/devices/tilelink/BootROM.scala
index 9ffdac9e65e..e92f944788e 100644
--- a/src/main/scala/devices/tilelink/BootROM.scala
+++ b/src/main/scala/devices/tilelink/BootROM.scala
@@ -5,7 +5,7 @@ package freechips.rocketchip.devices.tilelink
 import chisel3._
 import chisel3.util.log2Ceil
 import org.chipsalliance.cde.config.{Field, Parameters}
-import freechips.rocketchip.subsystem.{BaseSubsystem, HierarchicalLocation, HasTiles, TLBusWrapperLocation}
+import freechips.rocketchip.subsystem.{BaseSubsystem, HierarchicalLocation, HasElements, TLBusWrapperLocation}
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.prci.{ClockSinkDomain}
@@ -66,7 +66,7 @@ object BootROM {
     *    at a configurable location, but also drives the tiles' reset vectors to point
     *    at its 'hang' address parameter value.
     */
-  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasTiles, where: TLBusWrapperLocation)
+  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasElements, where: TLBusWrapperLocation)
             (implicit p: Parameters): TLROM = {
     val tlbus = subsystem.locateTLBusWrapper(where)
     val bootROMDomainWrapper = LazyModule(new ClockSinkDomain(take = None))
diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index 0c8497b469f..67fd262a65c 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -6,13 +6,13 @@ import chisel3._
 import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.diplomacy.{AddressSet, LazyModule}
 import freechips.rocketchip.interrupts.{IntSinkNode, IntSinkPortSimple}
-import freechips.rocketchip.subsystem.{BaseSubsystem, BaseSubsystemModuleImp, HasTiles, CanHaveMasterAXI4MemPort}
+import freechips.rocketchip.subsystem.{BaseSubsystem, BaseSubsystemModuleImp, HasElements, CanHaveMasterAXI4MemPort}
 import freechips.rocketchip.tilelink.{TLRAM, TLFragmenter}
 import freechips.rocketchip.interrupts.{NullIntSyncSource}
 
 class GroundTestSubsystem(implicit p: Parameters)
   extends BaseSubsystem
-  with HasTiles
+  with HasElements
   with CanHaveMasterAXI4MemPort
 {
   val testram = LazyModule(new TLRAM(AddressSet(0x52000000, 0xfff), beatBytes=pbus.beatBytes))
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 448365866c0..4c21b481463 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -341,37 +341,4 @@ case class CloneTileAttachParams(
   }
 }
 
-/** HasTiles instantiates and also connects a Config-urable sequence of tiles of any type to subsystem interconnect resources. */
-trait HasTiles extends HasCoreMonitorBundles with DefaultTileContextType
-{ this: LazyModule with Attachable =>
-  implicit val p: Parameters
-
-  // connect all the tiles to interconnect attachment points made available in this subsystem context
-  tileAttachParams.zip(tile_prci_domains).foreach { case (params, td) =>
-    params.connect(td.asInstanceOf[TilePRCIDomain[params.TileType]], this.asInstanceOf[params.TileContextType])
-  }
-}
-
-/** Provides some Chisel connectivity to certain tile IOs
-  * This trait is intended for the root subsystem
-  */
-trait HasTilesRootModuleImp extends LazyModuleImp {
-  val outer: HasTiles with HasTileInterruptSources with HasTileInputConstants
 
-  val reset_vector = outer.tileResetVectorIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"reset_vector_$i") }
-  val tile_hartids = outer.tileHartIdIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"tile_hartids_$i") }
-
-  val meip = if(outer.meipNode.isDefined) Some(IO(Input(Vec(outer.meipNode.get.out.size, Bool())))) else None
-  meip.foreach { m =>
-    m.zipWithIndex.foreach{ case (pin, i) =>
-      (outer.meipNode.get.out(i)._1)(0) := pin
-    }
-  }
-  val seip = if(outer.seipNode.isDefined) Some(IO(Input(Vec(outer.seipNode.get.out.size, Bool())))) else None
-  seip.foreach { s =>
-    s.zipWithIndex.foreach{ case (pin, i) =>
-      (outer.seipNode.get.out(i)._1)(0) := pin
-    }
-  }
-  val nmi = outer.totalTiles.zip(outer.tileNMIIONodes).zipWithIndex.map { case ((tile, n), i) => tile.tileParams.core.useNMI.option(n.makeIO(s"nmi_$i")) }
-}
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index 76ed297b3e6..ebb781d011f 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -23,7 +23,7 @@ case class RocketTileAttachParams(
   crossingParams: RocketCrossingParams
 ) extends CanAttachTile { type TileType = RocketTile }
 
-trait HasRocketTiles extends HasTiles { this: BaseSubsystem =>
+trait HasRocketTiles extends HasElements { this: BaseSubsystem =>
   val rocketTiles = totalTiles.collect { case r: RocketTile => r }
 
   def coreMonitorBundles = (rocketTiles map { t =>
@@ -41,5 +41,5 @@ class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
 }
 
 class RocketSubsystemModuleImp[+L <: RocketSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
-    with HasTilesRootModuleImp
+    with HasElementsRootModuleImp
 

From df45b74bd474a44846001a7e2dc26474351352ec Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 11:11:31 -0800
Subject: [PATCH 11/47] Create ElementPRCIDomain

---
 src/main/scala/subsystem/HasTiles.scala  |  6 ++---
 src/main/scala/tile/TilePRCIDomain.scala | 32 +++++-------------------
 2 files changed, 9 insertions(+), 29 deletions(-)

diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 4c21b481463..9ee33cf7607 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -190,7 +190,7 @@ trait CanAttachTile {
   def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: Seq[TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
     val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.name))
     val tile_prci_domain = LazyModule(new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
-      val element = self.tile_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
+      val element = self.element_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
     })
     tile_prci_domain
   }
@@ -313,7 +313,7 @@ trait CanAttachTile {
     })
 
     domain {
-      domain.tile_reset_domain.clockNode := crossingParams.resetCrossingType.injectClockNode := domain.clockNode
+      domain.element_reset_domain.clockNode := crossingParams.resetCrossingType.injectClockNode := domain.clockNode
     }
   }
 }
@@ -333,7 +333,7 @@ case class CloneTileAttachParams(
     val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.name))
     val tile_prci_domain = CloneLazyModule(
       new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
-        val element = self.tile_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
+        val element = self.element_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
       },
       instantiatedTiles(sourceHart).asInstanceOf[TilePRCIDomain[TileType]]
     )
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index 6c621c540b3..d125489d70c 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -8,24 +8,10 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.prci._
 import freechips.rocketchip.rocket.{TracedInstruction}
-import freechips.rocketchip.subsystem.{ElementCrossingParamsLike, CrossesToOnlyOneResetDomain, BaseElement}
+import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util.{TraceCoreInterface}
 
-/** A wrapper containing all logic within a managed reset domain for a tile.
-  *
-  * This does not add a layer of the module hierarchy.
-  */
-class TileResetDomain(clockSinkParams: ClockSinkParameters, resetCrossingType: ResetCrossingType)
-                     (implicit p: Parameters)
-    extends ResetDomain
-    with CrossesToOnlyOneResetDomain
-{ 
-  def crossing = resetCrossingType
-  val clockNode = ClockSinkNode(Seq(clockSinkParams))
-  def clockBundle = clockNode.in.head._1
-  override def shouldBeInlined = true
-}
 
 /** A wrapper containing all logic necessary to safely place a tile
   * inside of a particular Power/Reset/Clock/Interrupt domain.
@@ -35,18 +21,12 @@ class TileResetDomain(clockSinkParams: ClockSinkParameters, resetCrossingType: R
   * hierarchical P&R boundary buffers, core-local interrupt handling,
   * and any other IOs related to PRCI control.
   */
-abstract class TilePRCIDomain[T <: BaseElement](
+abstract class TilePRCIDomain[T <: BaseTile](
   clockSinkParams: ClockSinkParameters,
   crossingParams: ElementCrossingParamsLike)
   (implicit p: Parameters)
-    extends ClockDomain
+    extends ElementPRCIDomain[T](clockSinkParams, crossingParams)
 {
-  val element: T
-  val tile_reset_domain = LazyModule(new TileResetDomain(clockSinkParams, crossingParams.resetCrossingType))
-  val tapClockNode = ClockIdentityNode()
-  val clockNode = FixedClockBroadcast(None) :=* tapClockNode
-  lazy val clockBundle = tapClockNode.in.head._1
-
   private val traceSignalName = "trace"
   private val traceCoreSignalName = "tracecore"
   /** Node to broadcast legacy "raw" instruction trace while surpressing it during (async) reset. */
@@ -82,7 +62,7 @@ abstract class TilePRCIDomain[T <: BaseElement](
     * Takes tileNode as an argument because tiles might have multiple outbound interrupt nodes
     */
   def crossIntOut(crossingType: ClockCrossingType, tileNode: IntOutwardNode): IntOutwardNode = {
-    val intOutResetXing = this { tile_reset_domain.crossIntOut(tileNode) }
+    val intOutResetXing = this { element_reset_domain.crossIntOut(tileNode) }
     val intOutClockXing = this.crossOut(intOutResetXing)
     intOutClockXing(crossingType)
   }
@@ -92,7 +72,7 @@ abstract class TilePRCIDomain[T <: BaseElement](
     */
   def crossSlavePort(crossingType: ClockCrossingType): TLInwardNode = { DisableMonitors { implicit p => FlipRendering { implicit p =>
     val tlSlaveResetXing = this {
-      tile_reset_domain.crossTLIn(element.slaveNode) :*=
+      element_reset_domain.crossTLIn(element.slaveNode) :*=
         element { element.makeSlaveBoundaryBuffers(crossingType) }
     }
     val tlSlaveClockXing = this.crossIn(tlSlaveResetXing)
@@ -105,7 +85,7 @@ abstract class TilePRCIDomain[T <: BaseElement](
   def crossMasterPort(crossingType: ClockCrossingType): TLOutwardNode = {
     val tlMasterResetXing = this { DisableMonitors { implicit p =>
       element { element.makeMasterBoundaryBuffers(crossingType) } :=*
-        tile_reset_domain.crossTLOut(element.masterNode)
+        element_reset_domain.crossTLOut(element.masterNode)
     } }
     val tlMasterClockXing = this.crossOut(tlMasterResetXing)
     tlMasterClockXing(crossingType)

From 791c628407cb28ae463568fbee975e1d8abdc824 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 11:36:54 -0800
Subject: [PATCH 12/47] traceNode to Seqs

---
 src/main/scala/subsystem/Element.scala        |   4 +-
 .../scala/subsystem/ElementPRCIDomain.scala   |  51 ++++++++
 src/main/scala/subsystem/HasElements.scala    | 119 ++++++++++++++++++
 src/main/scala/tile/BaseTile.scala            |   7 +-
 4 files changed, 175 insertions(+), 6 deletions(-)
 create mode 100644 src/main/scala/subsystem/ElementPRCIDomain.scala
 create mode 100644 src/main/scala/subsystem/HasElements.scala

diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
index fdb8e9958c6..4d2c456038b 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/Element.scala
@@ -63,8 +63,8 @@ abstract class BaseElement (val crossing: ClockCrossingType)(implicit p: Paramet
   protected val tlSlaveXbar = LazyModule(new TLXbar)
   protected val intXbar = LazyModule(new IntXbar)
 
-  val traceCoreNode: BundleBridgeOutwardNode[TraceCoreInterface]
-  val traceNode: BundleBridgeOutwardNode[Vec[TracedInstruction]]
+  val traceCoreNodes: Seq[BundleBridgeOutwardNode[TraceCoreInterface]]
+  val traceNodes: Seq[BundleBridgeOutwardNode[Vec[TracedInstruction]]]
 
 
   /** Helper function to insert additional buffers on master ports at the boundary of the tile.
diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/ElementPRCIDomain.scala
new file mode 100644
index 00000000000..fdfc11b36be
--- /dev/null
+++ b/src/main/scala/subsystem/ElementPRCIDomain.scala
@@ -0,0 +1,51 @@
+package freechips.rocketchip.subsystem
+
+import chisel3._
+import chisel3.util._
+
+import org.chipsalliance.cde.config.Parameters
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.interrupts._
+import freechips.rocketchip.prci._
+import freechips.rocketchip.tile.{RocketTile}
+import freechips.rocketchip.subsystem._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.devices.debug.{TLDebugModule}
+import freechips.rocketchip.devices.tilelink._
+
+
+/** A wrapper containing all logic within a managed reset domain for a element.
+  *
+  * This does not add a layer of the module hierarchy.
+  */
+class ElementResetDomain(clockSinkParams: ClockSinkParameters, resetCrossingType: ResetCrossingType)
+                        (implicit p: Parameters)
+    extends ResetDomain
+    with CrossesToOnlyOneResetDomain
+{
+  def crossing = resetCrossingType
+  val clockNode = ClockSinkNode(Seq(clockSinkParams))
+  def clockBundle = clockNode.in.head._1
+  override def shouldBeInlined = true
+}
+
+/** A wrapper containing all logic necessary to safely place a tile
+  * inside of a particular Power/Reset/Clock/Interrupt domain.
+  *
+  * This adds a layer to the module hierarchy which is a parent of the tile
+  * and should contain all logic related to clock crossings, isolation cells,
+  * hierarchical P&R boundary buffers, core-local interrupt handling,
+  * and any other IOs related to PRCI control.
+  */
+abstract class ElementPRCIDomain[T <: BaseElement](
+  clockSinkParams: ClockSinkParameters,
+  crossingParams: ElementCrossingParamsLike)
+  (implicit p: Parameters)
+    extends ClockDomain
+{
+  val element: T
+  val element_reset_domain = LazyModule(new ElementResetDomain(clockSinkParams, crossingParams.resetCrossingType))
+  val tapClockNode = ClockIdentityNode()
+  val clockNode = FixedClockBroadcast(None) :=* tapClockNode
+  lazy val clockBundle = tapClockNode.in.head._1
+}
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
new file mode 100644
index 00000000000..d07c7b7639f
--- /dev/null
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -0,0 +1,119 @@
+// See LICENSE.SiFive for license details.
+
+package freechips.rocketchip.subsystem
+
+import chisel3._
+import chisel3.dontTouch
+import org.chipsalliance.cde.config.{Field, Parameters}
+import freechips.rocketchip.devices.debug.{TLDebugModule}
+import freechips.rocketchip.devices.tilelink._
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.interrupts._
+import freechips.rocketchip.tile._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.prci.{ClockGroup, ResetCrossingType, ClockGroupNode}
+import freechips.rocketchip.util._
+
+
+/** A default implementation of parameterizing the connectivity of the port where the tile is the master.
+  *   Optional timing buffers and/or an optional CacheCork can be inserted in the interconnect's clock domain.
+  */
+case class ElementMasterPortParams(
+  buffers: Int = 0,
+  cork: Option[Boolean] = None,
+  where: TLBusWrapperLocation = SBUS
+) extends ElementPortParamsLike {
+  def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
+    (TLBuffer.chainNode(buffers) :=* cork.map { u => TLCacheCork(unsafe = u) } .getOrElse { TLTempNode() })
+  }
+}
+
+/** A default implementation of parameterizing the connectivity of the port giving access to slaves inside the tile.
+  *   Optional timing buffers and/or an optional BusBlocker adapter can be inserted in the interconnect's clock domain.
+  */
+case class ElementSlavePortParams(
+  buffers: Int = 0,
+  blockerCtrlAddr: Option[BigInt] = None,
+  blockerCtrlWhere: TLBusWrapperLocation = CBUS,
+  where: TLBusWrapperLocation = CBUS
+) extends ElementPortParamsLike {
+  def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
+    val controlBus = context.locateTLBusWrapper(where)
+    val blockerBus = context.locateTLBusWrapper(blockerCtrlWhere)
+    blockerCtrlAddr
+      .map { BasicBusBlockerParams(_, blockerBus.beatBytes, controlBus.beatBytes) }
+      .map { bbbp =>
+        val blocker = LazyModule(new BasicBusBlocker(bbbp))
+        blockerBus.coupleTo("tile_slave_port_bus_blocker") { blocker.controlNode := TLFragmenter(blockerBus) := _ }
+        blocker.node :*= TLBuffer.chainNode(buffers)
+      } .getOrElse { TLBuffer.chainNode(buffers) }
+  }
+}
+
+/** InstantiatesTiles adds a Config-urable sequence of Elements of any type
+  *   to the subsystem class into which it is mixed.
+  */
+trait InstantiatesElements { this: LazyModule with Attachable =>
+  val location: HierarchicalLocation
+
+  /** Record the order in which to instantiate all tiles, based on statically-assigned ids.
+    *
+    * Note that these ids, which are often used as the tiles' default hartid input,
+    * may or may not be those actually reflected at runtime in e.g. the $mhartid CSR
+    */
+  val tileAttachParams: Seq[CanAttachTile] = p(TilesLocated(location)).sortBy(_.tileParams.hartId)
+  val tileParams: Seq[TileParams] = tileAttachParams.map(_.tileParams)
+  val tileCrossingTypes: Seq[ClockCrossingType] = tileAttachParams.map(_.crossingParams.crossingType)
+
+  /** The actual list of instantiated tiles in this block. */
+  val tile_prci_domains: Seq[TilePRCIDomain[_]] = tileAttachParams.foldLeft(Seq[TilePRCIDomain[_]]()) {
+    case (instantiated, params) => instantiated :+ params.instantiate(tileParams, instantiated)(p)
+  }
+
+  val leafTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile])
+  val totalTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile])
+
+  // Helper functions for accessing certain parameters that are popular to refer to in subsystem code
+  def nLeafTiles: Int = tileAttachParams.size
+  def nTotalTiles: Int = tileAttachParams.size
+  def leafHartIdList: Seq[Int] = tileParams.map(_.hartId)
+  def totalHartIdList: Seq[Int] = tileParams.map(_.hartId)
+  def localIntCounts: Seq[Int] = tileParams.map(_.core.nLocalInterrupts)
+
+  require(totalHartIdList.distinct.size == totalTiles.size, s"Every tile must be statically assigned a unique id, but got:\n${totalHartIdList}")
+}
+
+/** HasTiles instantiates and also connects a Config-urable sequence of tiles of any type to subsystem interconnect resources. */
+trait HasElements extends HasCoreMonitorBundles with DefaultTileContextType
+{ this: LazyModule with Attachable =>
+  implicit val p: Parameters
+
+  // connect all the tiles to interconnect attachment points made available in this subsystem context
+  tileAttachParams.zip(tile_prci_domains).foreach { case (params, td) =>
+    params.connect(td.asInstanceOf[TilePRCIDomain[params.TileType]], this.asInstanceOf[params.TileContextType])
+  }
+}
+
+/** Provides some Chisel connectivity to certain tile IOs
+  * This trait is intended for the root subsystem
+  */
+trait HasElementsRootModuleImp extends LazyModuleImp {
+  val outer: HasElements with HasTileInterruptSources with HasTileInputConstants
+
+  val reset_vector = outer.tileResetVectorIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"reset_vector_$i") }
+  val tile_hartids = outer.tileHartIdIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"tile_hartids_$i") }
+
+  val meip = if(outer.meipNode.isDefined) Some(IO(Input(Vec(outer.meipNode.get.out.size, Bool())))) else None
+  meip.foreach { m =>
+    m.zipWithIndex.foreach{ case (pin, i) =>
+      (outer.meipNode.get.out(i)._1)(0) := pin
+    }
+  }
+  val seip = if(outer.seipNode.isDefined) Some(IO(Input(Vec(outer.seipNode.get.out.size, Bool())))) else None
+  seip.foreach { s =>
+    s.zipWithIndex.foreach{ case (pin, i) =>
+      (outer.seipNode.get.out(i)._1)(0) := pin
+    }
+  }
+  val nmi = outer.totalTiles.zip(outer.tileNMIIONodes).zipWithIndex.map { case ((tile, n), i) => tile.tileParams.core.useNMI.option(n.makeIO(s"nmi_$i")) }
+}
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index a405ac61166..0f85023b7d9 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -271,15 +271,14 @@ abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
   protected def traceRetireWidth = tileParams.core.retireWidth
   /** Node for the core to drive legacy "raw" instruction trace. */
   val traceSourceNode = BundleBridgeSource(() => new TraceBundle)
-  private val traceNexus = BundleBroadcast[TraceBundle]() // backwards compatiblity; not blocked during stretched reset
   /** Node for external consumers to source a legacy instruction trace from the core. */
-  val traceNode: BundleBridgeOutwardNode[TraceBundle] = traceNexus := traceSourceNode
+  val traceNodes: Seq[BundleBridgeOutwardNode[TraceBundle]] = Seq(traceSourceNode)
 
-  protected def traceCoreParams = new TraceCoreParams()
+  def traceCoreParams = new TraceCoreParams()
   /** Node for core to drive instruction trace conforming to RISC-V Processor Trace spec V1.0 */
   val traceCoreSourceNode = BundleBridgeSource(() => new TraceCoreInterface(traceCoreParams))
   /** Node for external consumers to source  a V1.0 instruction trace from the core. */
-  val traceCoreNode: BundleBridgeOutwardNode[TraceCoreInterface] = traceCoreSourceNode
+  val traceCoreNodes: Seq[BundleBridgeOutwardNode[TraceCoreInterface]] = Seq(traceCoreSourceNode)
 
   /** Node to broadcast collected trace sideband signals into the tile. */
   val traceAuxNexusNode = BundleBridgeNexus[TraceAux](default = Some(() => {

From 1df02efcb119f88223442b11966c1d8ca92575d2 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 12:12:24 -0800
Subject: [PATCH 13/47] Use Nodes in HasElement for interrupts

---
 src/main/scala/devices/debug/Debug.scala     |  3 ++-
 src/main/scala/devices/debug/Periphery.scala |  2 +-
 src/main/scala/devices/tilelink/CLINT.scala  |  1 +
 src/main/scala/devices/tilelink/Plic.scala   |  1 +
 src/main/scala/subsystem/HasTiles.scala      | 23 +++++++++-----------
 5 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/main/scala/devices/debug/Debug.scala b/src/main/scala/devices/debug/Debug.scala
index fec2caa14a3..c1ea7c7f16e 100755
--- a/src/main/scala/devices/debug/Debug.scala
+++ b/src/main/scala/devices/debug/Debug.scala
@@ -693,7 +693,8 @@ class TLDebugModuleOuterAsync(device: Device)(implicit p: Parameters) extends La
   })
 
   val dmOuter = LazyModule( new TLDebugModuleOuter(device))
-  val intnode = IntSyncCrossingSource(alreadyRegistered = true) :*= dmOuter.intnode
+  val intnode = IntSyncIdentityNode()
+  intnode :*= IntSyncCrossingSource(alreadyRegistered = true) :*= dmOuter.intnode
 
   val dmiBypass = LazyModule(new TLBusBypass(beatBytes=4, bufferError=false, maxAtomic=0, maxTransfer=4))
   val dmiInnerNode = TLAsyncCrossingSource() := dmiBypass.node := dmiXbar.node
diff --git a/src/main/scala/devices/debug/Periphery.scala b/src/main/scala/devices/debug/Periphery.scala
index 009372078c6..fcf5826dc0b 100644
--- a/src/main/scala/devices/debug/Periphery.scala
+++ b/src/main/scala/devices/debug/Periphery.scala
@@ -99,7 +99,7 @@ trait HasPeripheryDebug { this: BaseSubsystem =>
     tlDM
   }
 
-  lazy val debugNode = debugOpt.map(_.intnode).getOrElse(IntSyncXbar() := NullIntSyncSource())
+  val debugNode = debugOpt.map(_.intnode)
 
   val psd = InModuleBody {
     val psd = IO(new PSDIO)
diff --git a/src/main/scala/devices/tilelink/CLINT.scala b/src/main/scala/devices/tilelink/CLINT.scala
index 8156706ee8d..7ca4405f4a9 100644
--- a/src/main/scala/devices/tilelink/CLINT.scala
+++ b/src/main/scala/devices/tilelink/CLINT.scala
@@ -116,4 +116,5 @@ trait CanHavePeripheryCLINT { this: BaseSubsystem =>
     clint
 
   }
+  val clintNode = clintOpt.map(_.intnode)
 }
diff --git a/src/main/scala/devices/tilelink/Plic.scala b/src/main/scala/devices/tilelink/Plic.scala
index 3e6de570423..539f076ea11 100644
--- a/src/main/scala/devices/tilelink/Plic.scala
+++ b/src/main/scala/devices/tilelink/Plic.scala
@@ -366,4 +366,5 @@ trait CanHavePeripheryPLIC { this: BaseSubsystem =>
 
     plic
   }
+  val plicNode = plicOpt.map(_.intnode)
 }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 9ee33cf7607..e196a93e127 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -169,9 +169,9 @@ trait DefaultTileContextType
   with HasTileNotificationSinks
   with HasTileInputConstants
 { this: LazyModule with Attachable =>
-  val debugOpt: Option[TLDebugModule]
-  val clintOpt: Option[CLINT]
-  val plicOpt: Option[TLPLIC]
+  val clintNode: Option[IntNexusNode]
+  val plicNode: Option[IntNexusNode]
+  val debugNode: Option[IntSyncIdentityNode]
 }
 
 /** Standardized interface by which parameterized tiles can be attached to contexts containing interconnect resources.
@@ -234,8 +234,8 @@ trait CanAttachTile {
 
     // 1. Debug interrupt is definitely asynchronous in all cases.
     domain.element.intInwardNode :=
-      context.debugOpt
-        .map { domain { IntSyncAsyncCrossingSink(3) } := _.intnode }
+      context.debugNode
+        .map { node => domain { IntSyncAsyncCrossingSink(3) } := node }
         .getOrElse { NullIntSource() }
 
     // 2. The CLINT and PLIC output interrupts are synchronous to the TileLink bus clock,
@@ -243,19 +243,16 @@ trait CanAttachTile {
 
     //    From CLINT: "msip" and "mtip"
     domain.crossIntIn(crossingParams.crossingType) :=
-      context.clintOpt.map { _.intnode }
-        .getOrElse { NullIntSource(sources = CLINTConsts.ints) }
+      context.clintNode.getOrElse { NullIntSource(sources = CLINTConsts.ints) }
 
     //    From PLIC: "meip"
     domain.crossIntIn(crossingParams.crossingType) :=
-      context.plicOpt .map { _.intnode }
-        .getOrElse { context.meipNode.get }
+      context.plicNode.getOrElse { context.meipNode.get }
 
     //    From PLIC: "seip" (only if supervisor mode is enabled)
     if (domain.element.tileParams.core.hasSupervisorMode) {
       domain.crossIntIn(crossingParams.crossingType) :=
-        context.plicOpt .map { _.intnode }
-          .getOrElse { context.seipNode.get }
+        context.plicNode.getOrElse { context.seipNode.get }
     }
 
     // 3. Local Interrupts ("lip") are required to already be synchronous to the Tile's clock.
@@ -263,9 +260,9 @@ trait CanAttachTile {
 
     // 4. Interrupts coming out of the tile are sent to the PLIC,
     //    so might need to be synchronized depending on the Tile's crossing type.
-    context.plicOpt.foreach { plic =>
+    context.plicNode.foreach { node =>
       FlipRendering { implicit p =>
-        plic.intnode :=* domain.crossIntOut(crossingParams.crossingType, domain.element.intOutwardNode)
+        node :=* domain.crossIntOut(crossingParams.crossingType, domain.element.intOutwardNode)
       }
     }
 

From bf98b808361fea3b560cc877ff83c1ad1a41624e Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 14:26:47 -0800
Subject: [PATCH 14/47] Genericize traceNodes

---
 src/main/scala/groundtest/GroundTestSubsystem.scala | 6 +++---
 src/main/scala/subsystem/Element.scala              | 5 ++---
 src/main/scala/subsystem/ElementPRCIDomain.scala    | 8 +++++++-
 src/main/scala/subsystem/HasTiles.scala             | 8 ++++----
 src/main/scala/tile/TilePRCIDomain.scala            | 8 ++++----
 5 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index 67fd262a65c..901c3a7863a 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -26,9 +26,9 @@ class GroundTestSubsystem(implicit p: Parameters)
 
   val tileStatusNodes = totalTiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
 
-  val clintOpt = None
-  val debugOpt = None
-  val plicOpt = None
+  val clintNode = None
+  val debugNode = None
+  val plicNode = None
 
   override lazy val module = new GroundTestSubsystemModuleImp(this)
 }
diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
index 4d2c456038b..b5ab7b37b1e 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/Element.scala
@@ -7,11 +7,10 @@ import org.chipsalliance.cde.config.Parameters
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.prci._
-import freechips.rocketchip.tile.{LookupByHartIdImpl}
+import freechips.rocketchip.tile.{LookupByHartIdImpl, TraceBundle}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
-import freechips.rocketchip.rocket.{TracedInstruction}
 import freechips.rocketchip.devices.debug.{TLDebugModule}
 import freechips.rocketchip.devices.tilelink._
 
@@ -64,7 +63,7 @@ abstract class BaseElement (val crossing: ClockCrossingType)(implicit p: Paramet
   protected val intXbar = LazyModule(new IntXbar)
 
   val traceCoreNodes: Seq[BundleBridgeOutwardNode[TraceCoreInterface]]
-  val traceNodes: Seq[BundleBridgeOutwardNode[Vec[TracedInstruction]]]
+  val traceNodes: Seq[BundleBridgeOutwardNode[TraceBundle]]
 
 
   /** Helper function to insert additional buffers on master ports at the boundary of the tile.
diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/ElementPRCIDomain.scala
index fdfc11b36be..c3e37b5074f 100644
--- a/src/main/scala/subsystem/ElementPRCIDomain.scala
+++ b/src/main/scala/subsystem/ElementPRCIDomain.scala
@@ -7,11 +7,12 @@ import org.chipsalliance.cde.config.Parameters
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.prci._
-import freechips.rocketchip.tile.{RocketTile}
+import freechips.rocketchip.tile.{RocketTile, TraceBundle}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.devices.debug.{TLDebugModule}
 import freechips.rocketchip.devices.tilelink._
+import freechips.rocketchip.util.{TraceCoreInterface}
 
 
 /** A wrapper containing all logic within a managed reset domain for a element.
@@ -48,4 +49,9 @@ abstract class ElementPRCIDomain[T <: BaseElement](
   val tapClockNode = ClockIdentityNode()
   val clockNode = FixedClockBroadcast(None) :=* tapClockNode
   lazy val clockBundle = tapClockNode.in.head._1
+
+  /** Node to broadcast legacy "raw" instruction trace while surpressing it during (async) reset. */
+  val traceNodes: Seq[BundleBridgeIdentityNode[TraceBundle]]
+  /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
+  val traceCoreNodes: Seq[BundleBridgeIdentityNode[TraceCoreInterface]]
 }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index e196a93e127..2834557255d 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -169,9 +169,9 @@ trait DefaultTileContextType
   with HasTileNotificationSinks
   with HasTileInputConstants
 { this: LazyModule with Attachable =>
-  val clintNode: Option[IntNexusNode]
-  val plicNode: Option[IntNexusNode]
-  val debugNode: Option[IntSyncIdentityNode]
+  val clintNode: Option[IntOutwardNode]
+  val plicNode: Option[IntNode]
+  val debugNode: Option[IntSyncOutwardNode]
 }
 
 /** Standardized interface by which parameterized tiles can be attached to contexts containing interconnect resources.
@@ -262,7 +262,7 @@ trait CanAttachTile {
     //    so might need to be synchronized depending on the Tile's crossing type.
     context.plicNode.foreach { node =>
       FlipRendering { implicit p =>
-        node :=* domain.crossIntOut(crossingParams.crossingType, domain.element.intOutwardNode)
+        node :*= domain.crossIntOut(crossingParams.crossingType, domain.element.intOutwardNode)
       }
     }
 
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index d125489d70c..9c6d99f061a 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -30,21 +30,21 @@ abstract class TilePRCIDomain[T <: BaseTile](
   private val traceSignalName = "trace"
   private val traceCoreSignalName = "tracecore"
   /** Node to broadcast legacy "raw" instruction trace while surpressing it during (async) reset. */
-  val traceNode: BundleBridgeIdentityNode[TraceBundle] = BundleBridgeNameNode(traceSignalName)
+  val traceNodes: Seq[BundleBridgeIdentityNode[TraceBundle]] = Seq(BundleBridgeNameNode(traceSignalName))
   /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
-  val traceCoreNode: BundleBridgeIdentityNode[TraceCoreInterface] = BundleBridgeNameNode(traceCoreSignalName)
+  val traceCoreNodes: Seq[BundleBridgeIdentityNode[TraceCoreInterface]] = Seq(BundleBridgeNameNode(traceCoreSignalName))
 
   /** Function to handle all trace crossings when tile is instantiated inside domains */
   def crossTracesOut(): Unit = this {
     val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
       resetCrossingType = crossingParams.resetCrossingType,
       name = Some(traceSignalName))
-    traceNode :*= traceNexusNode := element.traceNode
+    traceNodes(0) :*= traceNexusNode := element.traceNodes(0)
 
     val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
       resetCrossingType = crossingParams.resetCrossingType,
       name = Some(traceCoreSignalName))
-    traceCoreNode :*= traceCoreNexusNode := element.traceCoreNode
+    traceCoreNodes(0) :*= traceCoreNexusNode := element.traceCoreNodes(0)
   }
 
   /** External code looking to connect and clock-cross the interrupts driven into this tile can call this. */

From 9c2cc0daa68b9fce23913c554144d473f189e045 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 15:05:17 -0800
Subject: [PATCH 15/47] Genericize crossTracesout

---
 src/main/scala/subsystem/ElementPRCIDomain.scala | 16 ++++++++++++++++
 src/main/scala/tile/TilePRCIDomain.scala         | 13 -------------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/ElementPRCIDomain.scala
index c3e37b5074f..6231d982309 100644
--- a/src/main/scala/subsystem/ElementPRCIDomain.scala
+++ b/src/main/scala/subsystem/ElementPRCIDomain.scala
@@ -54,4 +54,20 @@ abstract class ElementPRCIDomain[T <: BaseElement](
   val traceNodes: Seq[BundleBridgeIdentityNode[TraceBundle]]
   /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
   val traceCoreNodes: Seq[BundleBridgeIdentityNode[TraceCoreInterface]]
+  require(element.traceNodes.size == traceNodes.size)
+  require(element.traceCoreNodes.size == traceCoreNodes.size)
+
+  /** Function to handle all trace crossings when tile is instantiated inside domains */
+  def crossTracesOut(): Unit = this {
+    for (i <- 0 until traceNodes.size) {
+      val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
+        resetCrossingType = crossingParams.resetCrossingType)
+      traceNodes(i) :*= traceNexusNode := element.traceNodes(i)
+    }
+    for (i <- 0 until traceCoreNodes.size) {
+      val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
+        resetCrossingType = crossingParams.resetCrossingType)
+      traceCoreNodes(i) :*= traceCoreNexusNode := element.traceCoreNodes(i)
+    }
+  }
 }
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index 9c6d99f061a..72b5a9e52cc 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -34,19 +34,6 @@ abstract class TilePRCIDomain[T <: BaseTile](
   /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
   val traceCoreNodes: Seq[BundleBridgeIdentityNode[TraceCoreInterface]] = Seq(BundleBridgeNameNode(traceCoreSignalName))
 
-  /** Function to handle all trace crossings when tile is instantiated inside domains */
-  def crossTracesOut(): Unit = this {
-    val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
-      resetCrossingType = crossingParams.resetCrossingType,
-      name = Some(traceSignalName))
-    traceNodes(0) :*= traceNexusNode := element.traceNodes(0)
-
-    val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
-      resetCrossingType = crossingParams.resetCrossingType,
-      name = Some(traceCoreSignalName))
-    traceCoreNodes(0) :*= traceCoreNexusNode := element.traceCoreNodes(0)
-  }
-
   /** External code looking to connect and clock-cross the interrupts driven into this tile can call this. */
   def crossIntIn(crossingType: ClockCrossingType): IntInwardNode = {
     // Unlike the other crossing helpers, here nothing is is blocked during reset because we know these are inputs and assume that tile reset is longer than uncore reset

From bfda1c93a03e5eca197c2ddac9947a5e3fb41239 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 15:21:15 -0800
Subject: [PATCH 16/47] Genericize rest of TilePRCIDomain

---
 .../scala/subsystem/ElementPRCIDomain.scala   | 44 +++++++++++++++++++
 src/main/scala/tile/TilePRCIDomain.scala      | 44 -------------------
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/ElementPRCIDomain.scala
index 6231d982309..144bd5dcd93 100644
--- a/src/main/scala/subsystem/ElementPRCIDomain.scala
+++ b/src/main/scala/subsystem/ElementPRCIDomain.scala
@@ -70,4 +70,48 @@ abstract class ElementPRCIDomain[T <: BaseElement](
       traceCoreNodes(i) :*= traceCoreNexusNode := element.traceCoreNodes(i)
     }
   }
+
+  /** External code looking to connect and clock-cross the interrupts driven into this tile can call this. */
+  def crossIntIn(crossingType: ClockCrossingType): IntInwardNode = {
+    // Unlike the other crossing helpers, here nothing is is blocked during reset because we know these are inputs and assume that tile reset is longer than uncore reset
+    val intInClockXing = this.crossIn(element.intInwardNode)
+    intInClockXing(crossingType)
+  }
+
+  /** External code looking to connect and clock/reset-cross
+    *   - interrupts raised by devices inside this tile
+    *   - notifications raise by the cores and caches
+    * can call this function to instantiate the required crossing hardware.
+    * Takes crossingType as an argument because some interrupts are supposed to be synchronous
+    * Takes tileNode as an argument because tiles might have multiple outbound interrupt nodes
+    */
+  def crossIntOut(crossingType: ClockCrossingType, tileNode: IntOutwardNode): IntOutwardNode = {
+    val intOutResetXing = this { element_reset_domain.crossIntOut(tileNode) }
+    val intOutClockXing = this.crossOut(intOutResetXing)
+    intOutClockXing(crossingType)
+  }
+
+  /** External code looking to connect the ports where this tile is slaved to an interconnect
+    * (while also crossing clock domains) can call this.
+    */
+  def crossSlavePort(crossingType: ClockCrossingType): TLInwardNode = { DisableMonitors { implicit p => FlipRendering { implicit p =>
+    val tlSlaveResetXing = this {
+      element_reset_domain.crossTLIn(element.slaveNode) :*=
+        element { element.makeSlaveBoundaryBuffers(crossingType) }
+    }
+    val tlSlaveClockXing = this.crossIn(tlSlaveResetXing)
+    tlSlaveClockXing(crossingType)
+  } } }
+
+  /** External code looking to connect the ports where this tile masters an interconnect
+    * (while also crossing clock domains) can call this.
+    */
+  def crossMasterPort(crossingType: ClockCrossingType): TLOutwardNode = {
+    val tlMasterResetXing = this { DisableMonitors { implicit p =>
+      element { element.makeMasterBoundaryBuffers(crossingType) } :=*
+        element_reset_domain.crossTLOut(element.masterNode)
+    } }
+    val tlMasterClockXing = this.crossOut(tlMasterResetXing)
+    tlMasterClockXing(crossingType)
+  }
 }
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index 72b5a9e52cc..270308706b1 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -33,48 +33,4 @@ abstract class TilePRCIDomain[T <: BaseTile](
   val traceNodes: Seq[BundleBridgeIdentityNode[TraceBundle]] = Seq(BundleBridgeNameNode(traceSignalName))
   /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
   val traceCoreNodes: Seq[BundleBridgeIdentityNode[TraceCoreInterface]] = Seq(BundleBridgeNameNode(traceCoreSignalName))
-
-  /** External code looking to connect and clock-cross the interrupts driven into this tile can call this. */
-  def crossIntIn(crossingType: ClockCrossingType): IntInwardNode = {
-    // Unlike the other crossing helpers, here nothing is is blocked during reset because we know these are inputs and assume that tile reset is longer than uncore reset
-    val intInClockXing = this.crossIn(element.intInwardNode)
-    intInClockXing(crossingType)
-  }
-
-  /** External code looking to connect and clock/reset-cross
-    *   - interrupts raised by devices inside this tile
-    *   - notifications raise by the cores and caches
-    * can call this function to instantiate the required crossing hardware.
-    * Takes crossingType as an argument because some interrupts are supposed to be synchronous
-    * Takes tileNode as an argument because tiles might have multiple outbound interrupt nodes
-    */
-  def crossIntOut(crossingType: ClockCrossingType, tileNode: IntOutwardNode): IntOutwardNode = {
-    val intOutResetXing = this { element_reset_domain.crossIntOut(tileNode) }
-    val intOutClockXing = this.crossOut(intOutResetXing)
-    intOutClockXing(crossingType)
-  }
-
-  /** External code looking to connect the ports where this tile is slaved to an interconnect
-    * (while also crossing clock domains) can call this.
-    */
-  def crossSlavePort(crossingType: ClockCrossingType): TLInwardNode = { DisableMonitors { implicit p => FlipRendering { implicit p =>
-    val tlSlaveResetXing = this {
-      element_reset_domain.crossTLIn(element.slaveNode) :*=
-        element { element.makeSlaveBoundaryBuffers(crossingType) }
-    }
-    val tlSlaveClockXing = this.crossIn(tlSlaveResetXing)
-    tlSlaveClockXing(crossingType)
-  } } }
-
-  /** External code looking to connect the ports where this tile masters an interconnect
-    * (while also crossing clock domains) can call this.
-    */
-  def crossMasterPort(crossingType: ClockCrossingType): TLOutwardNode = {
-    val tlMasterResetXing = this { DisableMonitors { implicit p =>
-      element { element.makeMasterBoundaryBuffers(crossingType) } :=*
-        element_reset_domain.crossTLOut(element.masterNode)
-    } }
-    val tlMasterClockXing = this.crossOut(tlMasterResetXing)
-    tlMasterClockXing(crossingType)
-  }
 }

From f8536817352660ef102f8acd16c55a4f1cc6c3e2 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 23 Jan 2023 16:53:15 -0800
Subject: [PATCH 17/47] Genericize interrupts

---
 src/main/scala/groundtest/Tile.scala          |  2 +-
 src/main/scala/subsystem/Element.scala        | 10 ++---
 .../scala/subsystem/ElementPRCIDomain.scala   |  8 ++--
 src/main/scala/subsystem/HasElements.scala    | 18 ++++++++-
 src/main/scala/subsystem/HasTiles.scala       | 38 ++++++-------------
 src/main/scala/tile/Interrupts.scala          | 14 +++----
 src/main/scala/tile/RocketTile.scala          |  4 +-
 7 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/src/main/scala/groundtest/Tile.scala b/src/main/scala/groundtest/Tile.scala
index 912016f8ea4..3225e671f54 100644
--- a/src/main/scala/groundtest/Tile.scala
+++ b/src/main/scala/groundtest/Tile.scala
@@ -34,7 +34,7 @@ abstract class GroundTestTile(
   with SourcesExternalNotifications
 {
   val cpuDevice: SimpleDevice = new SimpleDevice("groundtest", Nil)
-  val intOutwardNode: IntOutwardNode = IntIdentityNode()
+  val intOutwardNode = Seq(None)
   val slaveNode: TLInwardNode = TLIdentityNode()
   val statusNode = BundleBridgeSource(() => new GroundTestStatus)
 
diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
index b5ab7b37b1e..2b8ca2b54a1 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/Element.scala
@@ -51,11 +51,11 @@ abstract class BaseElement (val crossing: ClockCrossingType)(implicit p: Paramet
   def module: BaseElementModuleImp[BaseElement]
   def masterNode: TLOutwardNode
   def slaveNode: TLInwardNode
-  def intInwardNode: IntInwardNode    // Interrupts to the core from external devices
-  def intOutwardNode: IntOutwardNode  // Interrupts from tile-internal devices (e.g. BEU)
-  def haltNode: IntOutwardNode        // Unrecoverable error has occurred; suggest reset
-  def ceaseNode: IntOutwardNode       // Tile has ceased to retire instructions
-  def wfiNode: IntOutwardNode         // Tile is waiting for an interrupt
+  def intInwardNode: Seq[IntInwardNode]    // Interrupts to the core from external devices
+  def intOutwardNode: Seq[Option[IntOutwardNode]]  // Interrupts from tile-internal devices (e.g. BEU)
+  def haltNode: Seq[IntOutwardNode]        // Unrecoverable error has occurred; suggest reset
+  def ceaseNode: Seq[IntOutwardNode]       // Tile has ceased to retire instructions
+  def wfiNode: Seq[IntOutwardNode]         // Tile is waiting for an interrupt
 
   protected val tlOtherMastersNode = TLIdentityNode()
   protected val tlMasterXbar = LazyModule(new TLXbar)
diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/ElementPRCIDomain.scala
index 144bd5dcd93..2a69648a2ee 100644
--- a/src/main/scala/subsystem/ElementPRCIDomain.scala
+++ b/src/main/scala/subsystem/ElementPRCIDomain.scala
@@ -54,11 +54,11 @@ abstract class ElementPRCIDomain[T <: BaseElement](
   val traceNodes: Seq[BundleBridgeIdentityNode[TraceBundle]]
   /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
   val traceCoreNodes: Seq[BundleBridgeIdentityNode[TraceCoreInterface]]
-  require(element.traceNodes.size == traceNodes.size)
-  require(element.traceCoreNodes.size == traceCoreNodes.size)
 
   /** Function to handle all trace crossings when tile is instantiated inside domains */
   def crossTracesOut(): Unit = this {
+    require(element.traceNodes.size == traceNodes.size)
+    require(element.traceCoreNodes.size == traceCoreNodes.size)
     for (i <- 0 until traceNodes.size) {
       val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
         resetCrossingType = crossingParams.resetCrossingType)
@@ -72,9 +72,9 @@ abstract class ElementPRCIDomain[T <: BaseElement](
   }
 
   /** External code looking to connect and clock-cross the interrupts driven into this tile can call this. */
-  def crossIntIn(crossingType: ClockCrossingType): IntInwardNode = {
+  def crossIntIn(crossingType: ClockCrossingType, tileNode: IntInwardNode): IntInwardNode = {
     // Unlike the other crossing helpers, here nothing is is blocked during reset because we know these are inputs and assume that tile reset is longer than uncore reset
-    val intInClockXing = this.crossIn(element.intInwardNode)
+    val intInClockXing = this.crossIn(tileNode)
     intInClockXing(crossingType)
   }
 
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
index d07c7b7639f..7f7392ee69c 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -84,7 +84,7 @@ trait InstantiatesElements { this: LazyModule with Attachable =>
 }
 
 /** HasTiles instantiates and also connects a Config-urable sequence of tiles of any type to subsystem interconnect resources. */
-trait HasElements extends HasCoreMonitorBundles with DefaultTileContextType
+trait HasElements extends HasCoreMonitorBundles with DefaultElementContextType
 { this: LazyModule with Attachable =>
   implicit val p: Parameters
 
@@ -117,3 +117,19 @@ trait HasElementsRootModuleImp extends LazyModuleImp {
   }
   val nmi = outer.totalTiles.zip(outer.tileNMIIONodes).zipWithIndex.map { case ((tile, n), i) => tile.tileParams.core.useNMI.option(n.makeIO(s"nmi_$i")) }
 }
+
+/** Most tile types require only these traits in order for their standardized connect functions to apply.
+  *
+  *    BaseTiles subtypes with different needs can extend this trait to provide themselves with
+  *    additional external connection points.
+  */
+trait DefaultElementContextType
+  extends Attachable
+  with HasTileInterruptSources
+  with HasTileNotificationSinks
+  with HasTileInputConstants
+{ this: LazyModule with Attachable =>
+  val clintNode: Option[IntOutwardNode]
+  val plicNode: Option[IntNode]
+  val debugNode: Option[IntSyncOutwardNode]
+}
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 2834557255d..44eda307c4f 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -158,22 +158,6 @@ trait HasTileNotificationSinks { this: LazyModule =>
   tileCeaseSinkNode := tileCeaseXbarNode
 }
 
-/** Most tile types require only these traits in order for their standardized connect functions to apply.
-  *
-  *    BaseTiles subtypes with different needs can extend this trait to provide themselves with
-  *    additional external connection points.
-  */
-trait DefaultTileContextType
-  extends Attachable
-  with HasTileInterruptSources
-  with HasTileNotificationSinks
-  with HasTileInputConstants
-{ this: LazyModule with Attachable =>
-  val clintNode: Option[IntOutwardNode]
-  val plicNode: Option[IntNode]
-  val debugNode: Option[IntSyncOutwardNode]
-}
-
 /** Standardized interface by which parameterized tiles can be attached to contexts containing interconnect resources.
   *
   *   Sub-classes of this trait can optionally override the individual connect functions in order to specialize
@@ -182,7 +166,7 @@ trait DefaultTileContextType
   */
 trait CanAttachTile {
   type TileType <: BaseTile
-  type TileContextType <: DefaultTileContextType
+  type TileContextType <: DefaultElementContextType
   def tileParams: InstantiableTileParams[TileType]
   def crossingParams: ElementCrossingParamsLike
 
@@ -233,7 +217,7 @@ trait CanAttachTile {
     //       we stub out missing interrupts with constant sources here.
 
     // 1. Debug interrupt is definitely asynchronous in all cases.
-    domain.element.intInwardNode :=
+    domain.element.intInwardNode(0) :=
       context.debugNode
         .map { node => domain { IntSyncAsyncCrossingSink(3) } := node }
         .getOrElse { NullIntSource() }
@@ -242,16 +226,16 @@ trait CanAttachTile {
     //    so might need to be synchronized depending on the Tile's crossing type.
 
     //    From CLINT: "msip" and "mtip"
-    domain.crossIntIn(crossingParams.crossingType) :=
+    domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode(0)) :=
       context.clintNode.getOrElse { NullIntSource(sources = CLINTConsts.ints) }
 
     //    From PLIC: "meip"
-    domain.crossIntIn(crossingParams.crossingType) :=
+    domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode(0)) :=
       context.plicNode.getOrElse { context.meipNode.get }
 
     //    From PLIC: "seip" (only if supervisor mode is enabled)
     if (domain.element.tileParams.core.hasSupervisorMode) {
-      domain.crossIntIn(crossingParams.crossingType) :=
+      domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode(0)) :=
         context.plicNode.getOrElse { context.seipNode.get }
     }
 
@@ -261,9 +245,9 @@ trait CanAttachTile {
     // 4. Interrupts coming out of the tile are sent to the PLIC,
     //    so might need to be synchronized depending on the Tile's crossing type.
     context.plicNode.foreach { node =>
-      FlipRendering { implicit p =>
-        node :*= domain.crossIntOut(crossingParams.crossingType, domain.element.intOutwardNode)
-      }
+      FlipRendering { implicit p => domain.element.intOutwardNode(0).foreach { out =>
+        node :*= domain.crossIntOut(crossingParams.crossingType, out)
+      }}
     }
 
     // 5. Connect NMI inputs to the tile. These inputs are synchronous to the respective core_clock.
@@ -273,9 +257,9 @@ trait CanAttachTile {
   /** Notifications of tile status are connected to be broadcast without needing to be clock-crossed. */
   def connectOutputNotifications(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
     implicit val p = context.p
-    context.tileHaltXbarNode  :=* domain.crossIntOut(NoCrossing, domain.element.haltNode)
-    context.tileWFIXbarNode   :=* domain.crossIntOut(NoCrossing, domain.element.wfiNode)
-    context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.element.ceaseNode)
+    context.tileHaltXbarNode  :=* domain.crossIntOut(NoCrossing, domain.element.haltNode(0))
+    context.tileWFIXbarNode   :=* domain.crossIntOut(NoCrossing, domain.element.wfiNode(0))
+    context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.element.ceaseNode(0))
     // TODO should context be forced to have a trace sink connected here?
     //      for now this just ensures domain.trace[Core]Node has been crossed without connecting it externally
     domain.crossTracesOut()
diff --git a/src/main/scala/tile/Interrupts.scala b/src/main/scala/tile/Interrupts.scala
index 1efdde5983c..7c41216728d 100644
--- a/src/main/scala/tile/Interrupts.scala
+++ b/src/main/scala/tile/Interrupts.scala
@@ -28,7 +28,7 @@ class TileInterrupts(implicit p: Parameters) extends CoreBundle()(p) {
 // Use diplomatic interrupts to external interrupts from the subsystem into the tile
 trait SinksExternalInterrupts { this: BaseTile =>
 
-  val intInwardNode = intXbar.intnode :=* IntIdentityNode()(ValName("int_local"))
+  val intInwardNode = Seq(intXbar.intnode :=* IntIdentityNode()(ValName("int_local")))
   protected val intSinkNode = IntSinkNode(IntSinkPortSimple())
   intSinkNode := intXbar.intnode
 
@@ -85,10 +85,10 @@ trait SinksExternalInterrupts { this: BaseTile =>
 
 trait SourcesExternalNotifications { this: BaseTile =>
   // Report unrecoverable error conditions
-  val haltNode = IntSourceNode(IntSourcePortSimple())
+  val haltNode = Seq(IntSourceNode(IntSourcePortSimple()))
 
   def reportHalt(could_halt: Option[Bool]): Unit = {
-    val (halt_and_catch_fire, _) = haltNode.out(0)
+    val (halt_and_catch_fire, _) = haltNode(0).out(0)
     halt_and_catch_fire(0) := could_halt.map(RegEnable(true.B, false.B, _)).getOrElse(false.B)
   }
 
@@ -97,7 +97,7 @@ trait SourcesExternalNotifications { this: BaseTile =>
   }
 
   // Report when the tile has ceased to retire instructions
-  val ceaseNode = IntSourceNode(IntSourcePortSimple())
+  val ceaseNode = Seq(IntSourceNode(IntSourcePortSimple()))
 
   def reportCease(could_cease: Option[Bool], quiescenceCycles: Int = 8): Unit = {
     def waitForQuiescence(cease: Bool): Bool = {
@@ -108,7 +108,7 @@ trait SourcesExternalNotifications { this: BaseTile =>
       when (cease && !saturated) { count := count + 1.U }
       saturated
     }
-    val (cease, _) = ceaseNode.out(0)
+    val (cease, _) = ceaseNode(0).out(0)
     cease(0) := could_cease.map{ c => 
       val cease = (waitForQuiescence(c))
       // Test-Only Code --
@@ -119,10 +119,10 @@ trait SourcesExternalNotifications { this: BaseTile =>
   }
 
   // Report when the tile is waiting for an interrupt
-  val wfiNode = IntSourceNode(IntSourcePortSimple())
+  val wfiNode = Seq(IntSourceNode(IntSourcePortSimple()))
 
   def reportWFI(could_wfi: Option[Bool]): Unit = {
-    val (wfi, _) = wfiNode.out(0)
+    val (wfi, _) = wfiNode(0).out(0)
     wfi(0) := could_wfi.map(RegNext(_, init=false.B)).getOrElse(false.B)
   }
 }
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index bee8fff538e..f9b375ca3c8 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -52,7 +52,7 @@ class RocketTile private(
   def this(params: RocketTileParams, crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
-  val intOutwardNode = IntIdentityNode()
+  val intOutwardNode = Seq(rocketParams.beuAddr map { _ => IntIdentityNode() })
   val slaveNode = TLIdentityNode()
   val masterNode = visibilityNode
 
@@ -63,7 +63,7 @@ class RocketTile private(
 
   val bus_error_unit = rocketParams.beuAddr map { a =>
     val beu = LazyModule(new BusErrorUnit(new L1BusErrors, BusErrorUnitParams(a)))
-    intOutwardNode := beu.intNode
+    intOutwardNode(0).get := beu.intNode
     connectTLSlave(beu.node, xBytes)
     beu
   }

From e99eebdd41d1189c8a55bf5cc21b01eab2fdde31 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 25 Jan 2023 10:06:48 -0800
Subject: [PATCH 18/47] Move more stuff to Element

---
 src/main/scala/devices/tilelink/BootROM.scala |  4 +-
 .../groundtest/GroundTestSubsystem.scala      | 20 ++--
 src/main/scala/groundtest/Tile.scala          |  2 +-
 src/main/scala/subsystem/Element.scala        | 13 +--
 src/main/scala/subsystem/HasElements.scala    | 84 ++++++++++++----
 src/main/scala/subsystem/HasTiles.scala       | 96 +++++++------------
 .../scala/subsystem/RocketSubsystem.scala     | 14 ++-
 src/main/scala/tile/BaseTile.scala            | 17 ++--
 src/main/scala/tile/Interrupts.scala          | 14 +--
 src/main/scala/tile/RocketTile.scala          |  6 +-
 10 files changed, 155 insertions(+), 115 deletions(-)

diff --git a/src/main/scala/devices/tilelink/BootROM.scala b/src/main/scala/devices/tilelink/BootROM.scala
index e92f944788e..9431943072b 100644
--- a/src/main/scala/devices/tilelink/BootROM.scala
+++ b/src/main/scala/devices/tilelink/BootROM.scala
@@ -5,7 +5,7 @@ package freechips.rocketchip.devices.tilelink
 import chisel3._
 import chisel3.util.log2Ceil
 import org.chipsalliance.cde.config.{Field, Parameters}
-import freechips.rocketchip.subsystem.{BaseSubsystem, HierarchicalLocation, HasElements, TLBusWrapperLocation}
+import freechips.rocketchip.subsystem._
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.prci.{ClockSinkDomain}
@@ -66,7 +66,7 @@ object BootROM {
     *    at a configurable location, but also drives the tiles' reset vectors to point
     *    at its 'hang' address parameter value.
     */
-  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasElements, where: TLBusWrapperLocation)
+  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasElements with HasTileInputConstants, where: TLBusWrapperLocation)
             (implicit p: Parameters): TLROM = {
     val tlbus = subsystem.locateTLBusWrapper(where)
     val bootROMDomainWrapper = LazyModule(new ClockSinkDomain(take = None))
diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index 901c3a7863a..d29342e9162 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -4,15 +4,20 @@ package freechips.rocketchip.groundtest
 
 import chisel3._
 import org.chipsalliance.cde.config.{Parameters}
-import freechips.rocketchip.diplomacy.{AddressSet, LazyModule}
-import freechips.rocketchip.interrupts.{IntSinkNode, IntSinkPortSimple}
-import freechips.rocketchip.subsystem.{BaseSubsystem, BaseSubsystemModuleImp, HasElements, CanHaveMasterAXI4MemPort}
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.interrupts._
+import freechips.rocketchip.tile.{NMI}
+import freechips.rocketchip.devices.tilelink.{CLINTConsts}
+import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tilelink.{TLRAM, TLFragmenter}
 import freechips.rocketchip.interrupts.{NullIntSyncSource}
 
 class GroundTestSubsystem(implicit p: Parameters)
   extends BaseSubsystem
+  with InstantiatesElements
   with HasElements
+  with HasTileNotificationSinks
+  with HasTileInputConstants
   with CanHaveMasterAXI4MemPort
 {
   val testram = LazyModule(new TLRAM(AddressSet(0x52000000, 0xfff), beatBytes=pbus.beatBytes))
@@ -26,9 +31,12 @@ class GroundTestSubsystem(implicit p: Parameters)
 
   val tileStatusNodes = totalTiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
 
-  val clintNode = None
-  val debugNode = None
-  val plicNode = None
+  lazy val msipNodes = Map[Int, IntOutwardNode]().withDefaultValue(NullIntSource(sources = CLINTConsts.ints))
+  lazy val meipNodes = Map[Int, IntOutwardNode]().withDefaultValue(NullIntSource())
+  lazy val seipNodes = Map[Int, IntOutwardNode]().withDefaultValue(NullIntSource())
+  lazy val plicNodes = Map[Int, IntInwardNode]()
+  lazy val debugNodes = Map[Int, IntSyncOutwardNode]().withDefaultValue(IntSyncCrossingSource() := NullIntSource())
+  lazy val nmiNodes = Map[Int, BundleBridgeOutwardNode[NMI]]().withDefaultValue(BundleBridgeSource[NMI])
 
   override lazy val module = new GroundTestSubsystemModuleImp(this)
 }
diff --git a/src/main/scala/groundtest/Tile.scala b/src/main/scala/groundtest/Tile.scala
index 3225e671f54..f7bef2d439f 100644
--- a/src/main/scala/groundtest/Tile.scala
+++ b/src/main/scala/groundtest/Tile.scala
@@ -34,7 +34,7 @@ abstract class GroundTestTile(
   with SourcesExternalNotifications
 {
   val cpuDevice: SimpleDevice = new SimpleDevice("groundtest", Nil)
-  val intOutwardNode = Seq(None)
+  val intOutwardNode = None
   val slaveNode: TLInwardNode = TLIdentityNode()
   val statusNode = BundleBridgeSource(() => new GroundTestStatus)
 
diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
index 2b8ca2b54a1..dc2405a1960 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/Element.scala
@@ -49,22 +49,17 @@ abstract class BaseElement (val crossing: ClockCrossingType)(implicit p: Paramet
     with CrossesToOnlyOneClockDomain
 {
   def module: BaseElementModuleImp[BaseElement]
-  def masterNode: TLOutwardNode
-  def slaveNode: TLInwardNode
-  def intInwardNode: Seq[IntInwardNode]    // Interrupts to the core from external devices
-  def intOutwardNode: Seq[Option[IntOutwardNode]]  // Interrupts from tile-internal devices (e.g. BEU)
-  def haltNode: Seq[IntOutwardNode]        // Unrecoverable error has occurred; suggest reset
-  def ceaseNode: Seq[IntOutwardNode]       // Tile has ceased to retire instructions
-  def wfiNode: Seq[IntOutwardNode]         // Tile is waiting for an interrupt
 
   protected val tlOtherMastersNode = TLIdentityNode()
   protected val tlMasterXbar = LazyModule(new TLXbar)
   protected val tlSlaveXbar = LazyModule(new TLXbar)
   protected val intXbar = LazyModule(new IntXbar)
 
-  val traceCoreNodes: Seq[BundleBridgeOutwardNode[TraceCoreInterface]]
-  val traceNodes: Seq[BundleBridgeOutwardNode[TraceBundle]]
+  def masterNode: TLOutwardNode
+  def slaveNode: TLInwardNode
 
+  val traceCoreNodes: Map[Int, BundleBridgeOutwardNode[TraceCoreInterface]]
+  val traceNodes: Map[Int, BundleBridgeOutwardNode[TraceBundle]]
 
   /** Helper function to insert additional buffers on master ports at the boundary of the tile.
     *
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
index 7f7392ee69c..c413f5ec495 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -5,7 +5,7 @@ package freechips.rocketchip.subsystem
 import chisel3._
 import chisel3.dontTouch
 import org.chipsalliance.cde.config.{Field, Parameters}
-import freechips.rocketchip.devices.debug.{TLDebugModule}
+import freechips.rocketchip.devices.debug.{TLDebugModule, HasPeripheryDebug}
 import freechips.rocketchip.devices.tilelink._
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
@@ -84,8 +84,8 @@ trait InstantiatesElements { this: LazyModule with Attachable =>
 }
 
 /** HasTiles instantiates and also connects a Config-urable sequence of tiles of any type to subsystem interconnect resources. */
-trait HasElements extends HasCoreMonitorBundles with DefaultElementContextType
-{ this: LazyModule with Attachable =>
+trait HasElements extends DefaultElementContextType
+{ this: LazyModule with Attachable with InstantiatesElements =>
   implicit val p: Parameters
 
   // connect all the tiles to interconnect attachment points made available in this subsystem context
@@ -97,25 +97,27 @@ trait HasElements extends HasCoreMonitorBundles with DefaultElementContextType
 /** Provides some Chisel connectivity to certain tile IOs
   * This trait is intended for the root subsystem
   */
-trait HasElementsRootModuleImp extends LazyModuleImp {
-  val outer: HasElements with HasTileInterruptSources with HasTileInputConstants
+trait HasElementsRootContextModuleImp extends LazyModuleImp {
+  val outer: InstantiatesElements with HasElements with HasElementsRootContext with HasTileInputConstants
 
   val reset_vector = outer.tileResetVectorIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"reset_vector_$i") }
   val tile_hartids = outer.tileHartIdIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"tile_hartids_$i") }
 
-  val meip = if(outer.meipNode.isDefined) Some(IO(Input(Vec(outer.meipNode.get.out.size, Bool())))) else None
+  val meip = if (outer.meipIONode.isDefined) Some(IO(Input(Vec(outer.meipIONode.get.out.size, Bool())))) else None
   meip.foreach { m =>
     m.zipWithIndex.foreach{ case (pin, i) =>
-      (outer.meipNode.get.out(i)._1)(0) := pin
+      (outer.meipIONode.get.out(i)._1)(0) := pin
     }
   }
-  val seip = if(outer.seipNode.isDefined) Some(IO(Input(Vec(outer.seipNode.get.out.size, Bool())))) else None
+  val seip = if (outer.seipIONode.isDefined) Some(IO(Input(Vec(outer.seipIONode.get.out.size, Bool())))) else None
   seip.foreach { s =>
     s.zipWithIndex.foreach{ case (pin, i) =>
-      (outer.seipNode.get.out(i)._1)(0) := pin
+      (outer.seipIONode.get.out(i)._1)(0) := pin
     }
   }
-  val nmi = outer.totalTiles.zip(outer.tileNMIIONodes).zipWithIndex.map { case ((tile, n), i) => tile.tileParams.core.useNMI.option(n.makeIO(s"nmi_$i")) }
+  val nmi = outer.nmiIONodes.map { case (i, node) =>
+    node.makeIO(s"nmi_$i")
+  }
 }
 
 /** Most tile types require only these traits in order for their standardized connect functions to apply.
@@ -124,12 +126,60 @@ trait HasElementsRootModuleImp extends LazyModuleImp {
   *    additional external connection points.
   */
 trait DefaultElementContextType
-  extends Attachable
-  with HasTileInterruptSources
-  with HasTileNotificationSinks
-  with HasTileInputConstants
+    extends Attachable
+    with HasTileNotificationSinks
 { this: LazyModule with Attachable =>
-  val clintNode: Option[IntOutwardNode]
-  val plicNode: Option[IntNode]
-  val debugNode: Option[IntSyncOutwardNode]
+  val msipNodes: Map[Int, IntOutwardNode]
+  val meipNodes: Map[Int, IntOutwardNode]
+  val seipNodes: Map[Int, IntOutwardNode]
+  val plicNodes: Map[Int, IntInwardNode]
+  val debugNodes: Map[Int, IntSyncOutwardNode]
+  val nmiNodes: Map[Int, BundleBridgeOutwardNode[NMI]]
+  val tileHartIdNodes: Map[Int, BundleBridgeOutwardNode[UInt]]
+  val tileResetVectorNodes: Map[Int, BundleBridgeOutwardNode[UInt]]
+}
+
+/** This trait provides the tile attachment context for the root (outermost) subsystem */
+trait HasElementsRootContext
+{ this: HasElements
+    with HasPeripheryDebug
+    with CanHavePeripheryCLINT
+    with CanHavePeripheryPLIC
+    with HasTileNotificationSinks
+    with InstantiatesElements =>
+  val msipNodes: Map[Int, IntOutwardNode] = (0 until nTotalTiles).map { i =>
+    (i, IntEphemeralNode() := clintOpt.map(_.intnode).getOrElse(NullIntSource(sources = CLINTConsts.ints)))
+  }.toMap
+
+  val meipIONode = Option.when(plicOpt.isEmpty)(IntNexusNode(
+    sourceFn = { _ => IntSourcePortParameters(Seq(IntSourceParameters(1))) },
+    sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
+    outputRequiresInput = false,
+    inputRequiresOutput = false))
+  val meipNodes: Map[Int, IntOutwardNode] = (0 until nTotalTiles).map { i =>
+    (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(meipIONode.get))
+  }.toMap
+
+  val seipIONode = Option.when(plicOpt.isEmpty)(IntNexusNode(
+    sourceFn = { _ => IntSourcePortParameters(Seq(IntSourceParameters(1))) },
+    sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
+    outputRequiresInput = false,
+    inputRequiresOutput = false))
+  val seipNodes: Map[Int, IntOutwardNode] = (0 until nTotalTiles).map { i =>
+    (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(seipIONode.get))
+  }.toMap
+
+  val plicNodes: Map[Int, IntInwardNode] = (0 until nTotalTiles).map { i =>
+    plicOpt.map(o => (i, o.intnode :=* IntEphemeralNode()))
+  }.flatten.toMap
+
+  val debugNodes: Map[Int, IntSyncOutwardNode] = (0 until nTotalTiles).map { i =>
+    (i, IntSyncIdentityNode() := debugOpt.map(_.intnode).getOrElse(IntSyncCrossingSource() := NullIntSource()))
+  }.toMap
+
+  val nmiHarts = tileParams.filter(_.core.useNMI).map(_.hartId)
+  val nmiIONodes = nmiHarts.map { i => (i, BundleBridgeSource[NMI]()) }.toMap
+  val nmiNodes: Map[Int, BundleBridgeOutwardNode[NMI]] = nmiIONodes.map { case (i, n) =>
+    (i, BundleBridgeEphemeralNode[NMI]() := n)
+  }.toMap
 }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 44eda307c4f..ce72121ef23 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -37,49 +37,17 @@ case object HasTilesExternalHartIdWidthKey extends Field[Option[Int]](None)
   */
 case object HasTilesExternalResetVectorKey extends Field[Boolean](true)
 
-
-/** These are sources of interrupts that are driven into the tile.
-  * They need to be instantiated before tiles are attached to the subsystem containing them.
-  */
-trait HasTileInterruptSources extends InstantiatesElements
-{ this: LazyModule with Attachable =>
-  /** meipNode is used to create a single bit subsystem input in Configs without a PLIC */
-  val meipNode = p(PLICKey) match {
-    case Some(_) => None
-    case None    => Some(IntNexusNode(
-      sourceFn = { _ => IntSourcePortParameters(Seq(IntSourceParameters(1))) },
-      sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
-      outputRequiresInput = false,
-      inputRequiresOutput = false))
-  }
-  val seipNode = p(PLICKey) match {
-    case Some(_) => None
-    case None    => Some(IntNexusNode(
-      sourceFn = { _ => IntSourcePortParameters(Seq(IntSourceParameters(1))) },
-      sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
-      outputRequiresInput = false,
-      inputRequiresOutput = false))
-  }
-  /** Source of Non-maskable Interrupt (NMI) input bundle to each tile. */
-  val tileNMINode = BundleBridgeEphemeralNode[NMI]()
-  val tileNMIIONodes: Seq[BundleBridgeSource[NMI]] = {
-    Seq.fill(nTotalTiles) {
-      val nmiSource = BundleBridgeSource[NMI]()
-      tileNMINode := nmiSource
-      nmiSource
-    }
-  }
-}
-
 /** These are sources of "constants" that are driven into the tile.
   * 
   * While they are not expected to change dyanmically while the tile is executing code,
   * they may be either tied to a contant value or programmed during boot or reset.
   * They need to be instantiated before tiles are attached within the subsystem containing them.
   */
-trait HasTileInputConstants extends InstantiatesElements { this: LazyModule with Attachable =>
+trait HasTileInputConstants { this: LazyModule with Attachable with InstantiatesElements =>
   /** tileHartIdNode is used to collect publishers and subscribers of hartids. */
-  val tileHartIdNode = BundleBridgeEphemeralNode[UInt]()
+  val tileHartIdNodes: Map[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
+    (i, BundleBridgeEphemeralNode[UInt]())
+  }.toMap
 
   /** tileHartIdNexusNode is a BundleBridgeNexus that collects dynamic hart prefixes.
     *
@@ -106,7 +74,9 @@ trait HasTileInputConstants extends InstantiatesElements { this: LazyModule with
   // TODO: Replace the DebugModuleHartSelFuncs config key with logic to consume the dynamic hart IDs
 
   /** tileResetVectorNode is used to collect publishers and subscribers of tile reset vector addresses. */
-  val tileResetVectorNode = BundleBridgeEphemeralNode[UInt]()
+  val tileResetVectorNodes: Map[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
+    (i, BundleBridgeEphemeralNode[UInt]())
+  }.toMap
 
   /** tileResetVectorNexusNode is a BundleBridgeNexus that accepts a single reset vector source, and broadcasts it to all tiles. */
   val tileResetVectorNexusNode = BundleBroadcast[UInt](
@@ -118,12 +88,15 @@ trait HasTileInputConstants extends InstantiatesElements { this: LazyModule with
     *   Or, if such IOs are not configured to exist, tileHartIdNexusNode is used to supply an id to each tile.
     */
   val tileHartIdIONodes: Seq[BundleBridgeSource[UInt]] = p(HasTilesExternalHartIdWidthKey) match {
-    case Some(w) => Seq.fill(nTotalTiles) {
+    case Some(w) => (0 until nTotalTiles).map { i =>
       val hartIdSource = BundleBridgeSource(() => UInt(w.W))
-      tileHartIdNode := hartIdSource
+      tileHartIdNodes(i) := hartIdSource
       hartIdSource
     }
-    case None => { tileHartIdNode :*= tileHartIdNexusNode; Nil }
+    case None => {
+      (0 until nTotalTiles).map { i => tileHartIdNodes(i) :*= tileHartIdNexusNode }
+      Nil
+    }
   }
 
   /** tileResetVectorIONodes may generate subsystem IOs, one per tile, allowing the parent to assign unique reset vectors.
@@ -131,12 +104,15 @@ trait HasTileInputConstants extends InstantiatesElements { this: LazyModule with
     *   Or, if such IOs are not configured to exist, tileResetVectorNexusNode is used to supply a single reset vector to every tile.
     */
   val tileResetVectorIONodes: Seq[BundleBridgeSource[UInt]] = p(HasTilesExternalResetVectorKey) match {
-    case true => Seq.fill(nTotalTiles) {
+    case true => (0 until nTotalTiles).map { i =>
       val resetVectorSource = BundleBridgeSource[UInt]()
-      tileResetVectorNode := resetVectorSource
+      tileResetVectorNodes(i) := resetVectorSource
       resetVectorSource
     }
-    case false => { tileResetVectorNode :*= tileResetVectorNexusNode; Nil }
+    case false => {
+      (0 until nTotalTiles).map { i => tileResetVectorNodes(i) :*= tileResetVectorNexusNode }
+      Nil
+    }
   }
 }
 
@@ -217,26 +193,24 @@ trait CanAttachTile {
     //       we stub out missing interrupts with constant sources here.
 
     // 1. Debug interrupt is definitely asynchronous in all cases.
-    domain.element.intInwardNode(0) :=
-      context.debugNode
-        .map { node => domain { IntSyncAsyncCrossingSink(3) } := node }
-        .getOrElse { NullIntSource() }
+    domain.element.intInwardNode := domain { IntSyncAsyncCrossingSink(3) } :=
+      context.debugNodes(domain.element.hartId)
 
     // 2. The CLINT and PLIC output interrupts are synchronous to the TileLink bus clock,
     //    so might need to be synchronized depending on the Tile's crossing type.
 
     //    From CLINT: "msip" and "mtip"
-    domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode(0)) :=
-      context.clintNode.getOrElse { NullIntSource(sources = CLINTConsts.ints) }
+    domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
+      context.msipNodes(domain.element.hartId)
 
     //    From PLIC: "meip"
-    domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode(0)) :=
-      context.plicNode.getOrElse { context.meipNode.get }
+    domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
+      context.meipNodes(domain.element.hartId)
 
     //    From PLIC: "seip" (only if supervisor mode is enabled)
     if (domain.element.tileParams.core.hasSupervisorMode) {
-      domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode(0)) :=
-        context.plicNode.getOrElse { context.seipNode.get }
+      domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
+      context.seipNodes(domain.element.hartId)
     }
 
     // 3. Local Interrupts ("lip") are required to already be synchronous to the Tile's clock.
@@ -244,22 +218,22 @@ trait CanAttachTile {
 
     // 4. Interrupts coming out of the tile are sent to the PLIC,
     //    so might need to be synchronized depending on the Tile's crossing type.
-    context.plicNode.foreach { node =>
-      FlipRendering { implicit p => domain.element.intOutwardNode(0).foreach { out =>
+    context.plicNodes.get(domain.element.hartId).foreach { node =>
+      FlipRendering { implicit p => domain.element.intOutwardNode.foreach { out =>
         node :*= domain.crossIntOut(crossingParams.crossingType, out)
       }}
     }
 
     // 5. Connect NMI inputs to the tile. These inputs are synchronous to the respective core_clock.
-    domain.element.nmiNode := context.tileNMINode
+    domain.element.nmiNode.foreach(_ := context.nmiNodes(domain.element.hartId))
   }
 
   /** Notifications of tile status are connected to be broadcast without needing to be clock-crossed. */
   def connectOutputNotifications(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
     implicit val p = context.p
-    context.tileHaltXbarNode  :=* domain.crossIntOut(NoCrossing, domain.element.haltNode(0))
-    context.tileWFIXbarNode   :=* domain.crossIntOut(NoCrossing, domain.element.wfiNode(0))
-    context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.element.ceaseNode(0))
+    context.tileHaltXbarNode  :=* domain.crossIntOut(NoCrossing, domain.element.haltNode)
+    context.tileWFIXbarNode   :=* domain.crossIntOut(NoCrossing, domain.element.wfiNode)
+    context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.element.ceaseNode)
     // TODO should context be forced to have a trace sink connected here?
     //      for now this just ensures domain.trace[Core]Node has been crossed without connecting it externally
     domain.crossTracesOut()
@@ -269,8 +243,8 @@ trait CanAttachTile {
   def connectInputConstants(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
     implicit val p = context.p
     val tlBusToGetPrefixFrom = context.locateTLBusWrapper(crossingParams.mmioBaseAddressPrefixWhere)
-    domain.element.hartIdNode := context.tileHartIdNode
-    domain.element.resetVectorNode := context.tileResetVectorNode
+    domain.element.hartIdNode := context.tileHartIdNodes(domain.element.hartId)
+    domain.element.resetVectorNode := context.tileResetVectorNodes(domain.element.hartId)
     tlBusToGetPrefixFrom.prefixNode.foreach { domain.element.mmioAddressPrefixNode := _ }
   }
 
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index ebb781d011f..7d9ee26e562 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -7,6 +7,7 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.prci.{ResetCrossingType, NoResetCrossing}
 import freechips.rocketchip.tile._
 import freechips.rocketchip.devices.debug.{HasPeripheryDebug}
+import freechips.rocketchip.util.{HasCoreMonitorBundles}
 import freechips.rocketchip.devices.tilelink.{CanHavePeripheryCLINT, CanHavePeripheryPLIC}
 
 case class RocketCrossingParams(
@@ -23,7 +24,8 @@ case class RocketTileAttachParams(
   crossingParams: RocketCrossingParams
 ) extends CanAttachTile { type TileType = RocketTile }
 
-trait HasRocketTiles extends HasElements { this: BaseSubsystem =>
+trait HasRocketTiles {
+  this: BaseSubsystem with InstantiatesElements =>
   val rocketTiles = totalTiles.collect { case r: RocketTile => r }
 
   def coreMonitorBundles = (rocketTiles map { t =>
@@ -32,14 +34,20 @@ trait HasRocketTiles extends HasElements { this: BaseSubsystem =>
 }
 
 class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
-    with HasPeripheryDebug
+    with InstantiatesElements
+    with HasTileNotificationSinks
+    with HasTileInputConstants
     with CanHavePeripheryCLINT
     with CanHavePeripheryPLIC
+    with HasPeripheryDebug
+    with HasElementsRootContext
+    with HasElements
+    with HasCoreMonitorBundles
     with HasRocketTiles
 {
   override lazy val module = new RocketSubsystemModuleImp(this)
 }
 
 class RocketSubsystemModuleImp[+L <: RocketSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
-    with HasElementsRootModuleImp
+    with HasElementsRootContextModuleImp
 
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 0f85023b7d9..4553a41d7fa 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -209,6 +209,11 @@ abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
     )))
   }
 
+  def intInwardNode: IntInwardNode            // Interrupts to the core from external devices
+  def intOutwardNode: Option[IntOutwardNode]  // Interrupts from tile-internal devices (e.g. BEU)
+  def haltNode: IntOutwardNode                // Unrecoverable error has occurred; suggest reset
+  def ceaseNode: IntOutwardNode               // Tile has ceased to retire instructions
+  def wfiNode: IntOutwardNode                 // Tile is waiting for an interrupt
   def module: BaseTileModuleImp[BaseTile]
 
   /** Node for broadcasting a hart id to diplomatic consumers within the tile. */
@@ -242,10 +247,10 @@ abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
     resetVectorSinkNode := resetVectorNexusNode := BundleBridgeNameNode("reset_vector")
 
   /** Nodes for connecting NMI interrupt sources and vectors into the tile */
-  val nmiNexusNode: BundleBridgeNode[NMI] = BundleBroadcast[NMI]()
-  val nmiSinkNode = BundleBridgeSink[NMI](Some(() => new NMI(visiblePhysAddrBits)))
-  val nmiNode: BundleBridgeInwardNode[NMI] =
-    nmiSinkNode := nmiNexusNode := BundleBridgeNameNode("nmi")
+  val nmiSinkNode = Option.when(tileParams.core.useNMI) {
+    BundleBridgeSink[NMI](Some(() => new NMI(visiblePhysAddrBits)))
+  }
+  val nmiNode: Option[BundleBridgeInwardNode[NMI]] = nmiSinkNode.map(_ := BundleBridgeNameNode("nmi"))
 
   /** Node for broadcasting an address prefix to diplomatic consumers within the tile.
     *
@@ -272,13 +277,13 @@ abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
   /** Node for the core to drive legacy "raw" instruction trace. */
   val traceSourceNode = BundleBridgeSource(() => new TraceBundle)
   /** Node for external consumers to source a legacy instruction trace from the core. */
-  val traceNodes: Seq[BundleBridgeOutwardNode[TraceBundle]] = Seq(traceSourceNode)
+  val traceNodes: Map[Int, BundleBridgeOutwardNode[TraceBundle]] = Map(hartId -> traceSourceNode)
 
   def traceCoreParams = new TraceCoreParams()
   /** Node for core to drive instruction trace conforming to RISC-V Processor Trace spec V1.0 */
   val traceCoreSourceNode = BundleBridgeSource(() => new TraceCoreInterface(traceCoreParams))
   /** Node for external consumers to source  a V1.0 instruction trace from the core. */
-  val traceCoreNodes: Seq[BundleBridgeOutwardNode[TraceCoreInterface]] = Seq(traceCoreSourceNode)
+  val traceCoreNodes: Map[Int, BundleBridgeOutwardNode[TraceCoreInterface]] = Map(hartId -> traceCoreSourceNode)
 
   /** Node to broadcast collected trace sideband signals into the tile. */
   val traceAuxNexusNode = BundleBridgeNexus[TraceAux](default = Some(() => {
diff --git a/src/main/scala/tile/Interrupts.scala b/src/main/scala/tile/Interrupts.scala
index 7c41216728d..1efdde5983c 100644
--- a/src/main/scala/tile/Interrupts.scala
+++ b/src/main/scala/tile/Interrupts.scala
@@ -28,7 +28,7 @@ class TileInterrupts(implicit p: Parameters) extends CoreBundle()(p) {
 // Use diplomatic interrupts to external interrupts from the subsystem into the tile
 trait SinksExternalInterrupts { this: BaseTile =>
 
-  val intInwardNode = Seq(intXbar.intnode :=* IntIdentityNode()(ValName("int_local")))
+  val intInwardNode = intXbar.intnode :=* IntIdentityNode()(ValName("int_local"))
   protected val intSinkNode = IntSinkNode(IntSinkPortSimple())
   intSinkNode := intXbar.intnode
 
@@ -85,10 +85,10 @@ trait SinksExternalInterrupts { this: BaseTile =>
 
 trait SourcesExternalNotifications { this: BaseTile =>
   // Report unrecoverable error conditions
-  val haltNode = Seq(IntSourceNode(IntSourcePortSimple()))
+  val haltNode = IntSourceNode(IntSourcePortSimple())
 
   def reportHalt(could_halt: Option[Bool]): Unit = {
-    val (halt_and_catch_fire, _) = haltNode(0).out(0)
+    val (halt_and_catch_fire, _) = haltNode.out(0)
     halt_and_catch_fire(0) := could_halt.map(RegEnable(true.B, false.B, _)).getOrElse(false.B)
   }
 
@@ -97,7 +97,7 @@ trait SourcesExternalNotifications { this: BaseTile =>
   }
 
   // Report when the tile has ceased to retire instructions
-  val ceaseNode = Seq(IntSourceNode(IntSourcePortSimple()))
+  val ceaseNode = IntSourceNode(IntSourcePortSimple())
 
   def reportCease(could_cease: Option[Bool], quiescenceCycles: Int = 8): Unit = {
     def waitForQuiescence(cease: Bool): Bool = {
@@ -108,7 +108,7 @@ trait SourcesExternalNotifications { this: BaseTile =>
       when (cease && !saturated) { count := count + 1.U }
       saturated
     }
-    val (cease, _) = ceaseNode(0).out(0)
+    val (cease, _) = ceaseNode.out(0)
     cease(0) := could_cease.map{ c => 
       val cease = (waitForQuiescence(c))
       // Test-Only Code --
@@ -119,10 +119,10 @@ trait SourcesExternalNotifications { this: BaseTile =>
   }
 
   // Report when the tile is waiting for an interrupt
-  val wfiNode = Seq(IntSourceNode(IntSourcePortSimple()))
+  val wfiNode = IntSourceNode(IntSourcePortSimple())
 
   def reportWFI(could_wfi: Option[Bool]): Unit = {
-    val (wfi, _) = wfiNode(0).out(0)
+    val (wfi, _) = wfiNode.out(0)
     wfi(0) := could_wfi.map(RegNext(_, init=false.B)).getOrElse(false.B)
   }
 }
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index f9b375ca3c8..bcfef753f15 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -52,7 +52,7 @@ class RocketTile private(
   def this(params: RocketTileParams, crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
-  val intOutwardNode = Seq(rocketParams.beuAddr map { _ => IntIdentityNode() })
+  val intOutwardNode = rocketParams.beuAddr map { _ => IntIdentityNode() }
   val slaveNode = TLIdentityNode()
   val masterNode = visibilityNode
 
@@ -63,7 +63,7 @@ class RocketTile private(
 
   val bus_error_unit = rocketParams.beuAddr map { a =>
     val beu = LazyModule(new BusErrorUnit(new L1BusErrors, BusErrorUnitParams(a)))
-    intOutwardNode(0).get := beu.intNode
+    intOutwardNode.get := beu.intNode
     connectTLSlave(beu.node, xBytes)
     beu
   }
@@ -149,7 +149,7 @@ class RocketTileModuleImp(outer: RocketTile) extends BaseTileModuleImp(outer)
     beu.module.io.errors.icache := outer.frontend.module.io.errors
   }
 
-  core.io.interrupts.nmi.foreach { nmi => nmi := outer.nmiSinkNode.bundle }
+  core.io.interrupts.nmi.foreach { nmi => nmi := outer.nmiSinkNode.get.bundle }
 
   // Pass through various external constants and reports that were bundle-bridged into the tile
   outer.traceSourceNode.bundle <> core.io.trace

From e66d9b3d968da3b1b91fd243a248d9de7ab382ad Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 25 Jan 2023 11:31:35 -0800
Subject: [PATCH 19/47] Use map for tracenodes

---
 src/main/scala/subsystem/ElementPRCIDomain.scala | 4 ++--
 src/main/scala/tile/TilePRCIDomain.scala         | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/ElementPRCIDomain.scala
index 2a69648a2ee..c1c9fc99433 100644
--- a/src/main/scala/subsystem/ElementPRCIDomain.scala
+++ b/src/main/scala/subsystem/ElementPRCIDomain.scala
@@ -51,9 +51,9 @@ abstract class ElementPRCIDomain[T <: BaseElement](
   lazy val clockBundle = tapClockNode.in.head._1
 
   /** Node to broadcast legacy "raw" instruction trace while surpressing it during (async) reset. */
-  val traceNodes: Seq[BundleBridgeIdentityNode[TraceBundle]]
+  val traceNodes: Map[Int, BundleBridgeIdentityNode[TraceBundle]]
   /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
-  val traceCoreNodes: Seq[BundleBridgeIdentityNode[TraceCoreInterface]]
+  val traceCoreNodes: Map[Int, BundleBridgeIdentityNode[TraceCoreInterface]]
 
   /** Function to handle all trace crossings when tile is instantiated inside domains */
   def crossTracesOut(): Unit = this {
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index 270308706b1..8d5d7ecfb54 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -30,7 +30,9 @@ abstract class TilePRCIDomain[T <: BaseTile](
   private val traceSignalName = "trace"
   private val traceCoreSignalName = "tracecore"
   /** Node to broadcast legacy "raw" instruction trace while surpressing it during (async) reset. */
-  val traceNodes: Seq[BundleBridgeIdentityNode[TraceBundle]] = Seq(BundleBridgeNameNode(traceSignalName))
+  val traceNode: BundleBridgeIdentityNode[TraceBundle] = BundleBridgeNameNode(traceSignalName)
+  val traceNodes = Map(element.hartId -> traceNode)
   /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
-  val traceCoreNodes: Seq[BundleBridgeIdentityNode[TraceCoreInterface]] = Seq(BundleBridgeNameNode(traceCoreSignalName))
+  val traceCoreNode: BundleBridgeIdentityNode[TraceCoreInterface] = BundleBridgeNameNode(traceCoreSignalName)
+  val traceCoreNodes = Map(element.hartId -> traceCoreNode)
 }

From 60045e1700dc49f4047de43d90d2da912c20dd7c Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 25 Jan 2023 18:26:20 -0800
Subject: [PATCH 20/47] Improve tile context node types

---
 .../groundtest/GroundTestSubsystem.scala      | 19 ++++++----
 src/main/scala/subsystem/BusTopology.scala    |  3 ++
 src/main/scala/subsystem/Element.scala        |  1 +
 src/main/scala/subsystem/HasElements.scala    | 36 +++++++++++--------
 src/main/scala/subsystem/HasTiles.scala       |  2 +-
 src/main/scala/tile/TilePRCIDomain.scala      |  4 +--
 6 files changed, 41 insertions(+), 24 deletions(-)

diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index d29342e9162..99e1435991e 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -31,12 +31,19 @@ class GroundTestSubsystem(implicit p: Parameters)
 
   val tileStatusNodes = totalTiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
 
-  lazy val msipNodes = Map[Int, IntOutwardNode]().withDefaultValue(NullIntSource(sources = CLINTConsts.ints))
-  lazy val meipNodes = Map[Int, IntOutwardNode]().withDefaultValue(NullIntSource())
-  lazy val seipNodes = Map[Int, IntOutwardNode]().withDefaultValue(NullIntSource())
-  lazy val plicNodes = Map[Int, IntInwardNode]()
-  lazy val debugNodes = Map[Int, IntSyncOutwardNode]().withDefaultValue(IntSyncCrossingSource() := NullIntSource())
-  lazy val nmiNodes = Map[Int, BundleBridgeOutwardNode[NMI]]().withDefaultValue(BundleBridgeSource[NMI])
+  val msipNodes = (0 until nTotalTiles).map { i => (i, IntIdentityNode()) }.toMap
+  val meipNodes = (0 until nTotalTiles).map { i => (i, IntIdentityNode()) }.toMap
+  val seipNodes = (0 until nTotalTiles).map { i => (i, IntIdentityNode()) }.toMap
+  val plicNodes = (0 until nTotalTiles).map { i => (i, IntIdentityNode()) }.toMap
+  val debugNodes = (0 until nTotalTiles).map { i => (i, IntSyncIdentityNode()) }.toMap
+  val nmiNodes = (0 until nTotalTiles).map { i => (i, BundleBridgeIdentityNode[NMI]()) }.toMap
+
+  msipNodes.values.foreach(_ := NullIntSource(sources=CLINTConsts.ints))
+  meipNodes.values.foreach(_ := NullIntSource())
+  seipNodes.values.foreach(_ := NullIntSource())
+  plicNodes.values.foreach(n => IntSinkNode(Nil) := n) // sink to nowhere
+  debugNodes.values.foreach(_ := IntSyncCrossingSource() := NullIntSource())
+  nmiNodes.values.foreach(_ := BundleBridgeSource[NMI]())
 
   override lazy val module = new GroundTestSubsystemModuleImp(this)
 }
diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index 239a016647e..7c27645a236 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -30,6 +30,9 @@ case object MBUS extends TLBusWrapperLocation("subsystem_mbus")
 case object CBUS extends TLBusWrapperLocation("subsystem_cbus")
 case object L2   extends TLBusWrapperLocation("subsystem_l2")
 
+case class ClusterSBUS(val clusterId: Int) extends TLBusWrapperLocation(s"cluster_${clusterId}_sbus")
+case class ClusterCBUS(val clusterId: Int) extends TLBusWrapperLocation(s"cluster_${clusterId}_cbus")
+
 /** Parameterizes the subsystem in terms of optional clock-crossings
   *   that are insertable between some of the five traditional tilelink bus wrappers.
   *   This class exists for backwards compatiblity reasons but could eventually be retired
diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
index dc2405a1960..3eabde94917 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/Element.scala
@@ -16,6 +16,7 @@ import freechips.rocketchip.devices.tilelink._
 
 trait ElementParams {
   val name: String
+  val clockSinkParams: ClockSinkParameters
 }
 
 abstract class InstantiableElementParams[ElementType <: BaseElement] extends ElementParams
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
index c413f5ec495..5199f543bcf 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -73,6 +73,8 @@ trait InstantiatesElements { this: LazyModule with Attachable =>
   val leafTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile])
   val totalTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile])
 
+  val element_prci_domains: Seq[ElementPRCIDomain[_]] = tile_prci_domains
+
   // Helper functions for accessing certain parameters that are popular to refer to in subsystem code
   def nLeafTiles: Int = tileAttachParams.size
   def nTotalTiles: Int = tileAttachParams.size
@@ -129,14 +131,14 @@ trait DefaultElementContextType
     extends Attachable
     with HasTileNotificationSinks
 { this: LazyModule with Attachable =>
-  val msipNodes: Map[Int, IntOutwardNode]
-  val meipNodes: Map[Int, IntOutwardNode]
-  val seipNodes: Map[Int, IntOutwardNode]
-  val plicNodes: Map[Int, IntInwardNode]
-  val debugNodes: Map[Int, IntSyncOutwardNode]
-  val nmiNodes: Map[Int, BundleBridgeOutwardNode[NMI]]
-  val tileHartIdNodes: Map[Int, BundleBridgeOutwardNode[UInt]]
-  val tileResetVectorNodes: Map[Int, BundleBridgeOutwardNode[UInt]]
+  val msipNodes: Map[Int, IntNode]
+  val meipNodes: Map[Int, IntNode]
+  val seipNodes: Map[Int, IntNode]
+  val plicNodes: Map[Int, IntNode]
+  val debugNodes: Map[Int, IntSyncNode]
+  val nmiNodes: Map[Int, BundleBridgeNode[NMI]]
+  val tileHartIdNodes: Map[Int, BundleBridgeNode[UInt]]
+  val tileResetVectorNodes: Map[Int, BundleBridgeNode[UInt]]
 }
 
 /** This trait provides the tile attachment context for the root (outermost) subsystem */
@@ -147,7 +149,7 @@ trait HasElementsRootContext
     with CanHavePeripheryPLIC
     with HasTileNotificationSinks
     with InstantiatesElements =>
-  val msipNodes: Map[Int, IntOutwardNode] = (0 until nTotalTiles).map { i =>
+  val msipNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode() := clintOpt.map(_.intnode).getOrElse(NullIntSource(sources = CLINTConsts.ints)))
   }.toMap
 
@@ -156,7 +158,7 @@ trait HasElementsRootContext
     sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
     outputRequiresInput = false,
     inputRequiresOutput = false))
-  val meipNodes: Map[Int, IntOutwardNode] = (0 until nTotalTiles).map { i =>
+  val meipNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(meipIONode.get))
   }.toMap
 
@@ -165,21 +167,25 @@ trait HasElementsRootContext
     sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
     outputRequiresInput = false,
     inputRequiresOutput = false))
-  val seipNodes: Map[Int, IntOutwardNode] = (0 until nTotalTiles).map { i =>
+  val seipNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(seipIONode.get))
   }.toMap
 
-  val plicNodes: Map[Int, IntInwardNode] = (0 until nTotalTiles).map { i =>
+  val plicNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
     plicOpt.map(o => (i, o.intnode :=* IntEphemeralNode()))
   }.flatten.toMap
 
-  val debugNodes: Map[Int, IntSyncOutwardNode] = (0 until nTotalTiles).map { i =>
-    (i, IntSyncIdentityNode() := debugOpt.map(_.intnode).getOrElse(IntSyncCrossingSource() := NullIntSource()))
+  val debugNodes: Map[Int, IntSyncNode] = (0 until nTotalTiles).map { i =>
+    (i, IntSyncIdentityNode())
   }.toMap
 
+  debugNodes.foreach { case (hartid, node) =>
+    node := debugOpt.map(_.intnode).getOrElse(IntSyncCrossingSource() := NullIntSource())
+  }
+
   val nmiHarts = tileParams.filter(_.core.useNMI).map(_.hartId)
   val nmiIONodes = nmiHarts.map { i => (i, BundleBridgeSource[NMI]()) }.toMap
-  val nmiNodes: Map[Int, BundleBridgeOutwardNode[NMI]] = nmiIONodes.map { case (i, n) =>
+  val nmiNodes: Map[Int, BundleBridgeNode[NMI]] = nmiIONodes.map { case (i, n) =>
     (i, BundleBridgeEphemeralNode[NMI]() := n)
   }.toMap
 }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index ce72121ef23..cf8e0e80b18 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -210,7 +210,7 @@ trait CanAttachTile {
     //    From PLIC: "seip" (only if supervisor mode is enabled)
     if (domain.element.tileParams.core.hasSupervisorMode) {
       domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
-      context.seipNodes(domain.element.hartId)
+        context.seipNodes(domain.element.hartId)
     }
 
     // 3. Local Interrupts ("lip") are required to already be synchronous to the Tile's clock.
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index 8d5d7ecfb54..22cc14f1e77 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -31,8 +31,8 @@ abstract class TilePRCIDomain[T <: BaseTile](
   private val traceCoreSignalName = "tracecore"
   /** Node to broadcast legacy "raw" instruction trace while surpressing it during (async) reset. */
   val traceNode: BundleBridgeIdentityNode[TraceBundle] = BundleBridgeNameNode(traceSignalName)
-  val traceNodes = Map(element.hartId -> traceNode)
+  lazy val traceNodes = Map(element.hartId -> traceNode)
   /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
   val traceCoreNode: BundleBridgeIdentityNode[TraceCoreInterface] = BundleBridgeNameNode(traceCoreSignalName)
-  val traceCoreNodes = Map(element.hartId -> traceCoreNode)
+  lazy val traceCoreNodes = Map(element.hartId -> traceCoreNode)
 }

From b02937ee1d0e390802c1b28ac713f9dd070d431e Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 30 Jan 2023 10:52:44 -0800
Subject: [PATCH 21/47] Fix tiles in clusters

---
 .../groundtest/GroundTestSubsystem.scala      | 18 ++-----
 src/main/scala/subsystem/BusTopology.scala    | 15 +++++-
 src/main/scala/subsystem/Configs.scala        | 33 +++++++++++--
 src/main/scala/subsystem/Element.scala        |  3 --
 .../scala/subsystem/ElementPRCIDomain.scala   | 21 --------
 src/main/scala/subsystem/HasElements.scala    | 48 +++++++++++++------
 src/main/scala/subsystem/HasTiles.scala       | 21 ++++++--
 src/main/scala/system/Configs.scala           | 12 +++++
 src/main/scala/tile/BaseTile.scala            |  5 +-
 src/main/scala/tile/TilePRCIDomain.scala      |  8 ----
 10 files changed, 111 insertions(+), 73 deletions(-)

diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index 99e1435991e..232c69204be 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -15,6 +15,7 @@ import freechips.rocketchip.interrupts.{NullIntSyncSource}
 class GroundTestSubsystem(implicit p: Parameters)
   extends BaseSubsystem
   with InstantiatesElements
+  with HasElementsRootContext
   with HasElements
   with HasTileNotificationSinks
   with HasTileInputConstants
@@ -30,20 +31,9 @@ class GroundTestSubsystem(implicit p: Parameters)
   IntSinkNode(IntSinkPortSimple()) :=* ibus.toPLIC
 
   val tileStatusNodes = totalTiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
-
-  val msipNodes = (0 until nTotalTiles).map { i => (i, IntIdentityNode()) }.toMap
-  val meipNodes = (0 until nTotalTiles).map { i => (i, IntIdentityNode()) }.toMap
-  val seipNodes = (0 until nTotalTiles).map { i => (i, IntIdentityNode()) }.toMap
-  val plicNodes = (0 until nTotalTiles).map { i => (i, IntIdentityNode()) }.toMap
-  val debugNodes = (0 until nTotalTiles).map { i => (i, IntSyncIdentityNode()) }.toMap
-  val nmiNodes = (0 until nTotalTiles).map { i => (i, BundleBridgeIdentityNode[NMI]()) }.toMap
-
-  msipNodes.values.foreach(_ := NullIntSource(sources=CLINTConsts.ints))
-  meipNodes.values.foreach(_ := NullIntSource())
-  seipNodes.values.foreach(_ := NullIntSource())
-  plicNodes.values.foreach(n => IntSinkNode(Nil) := n) // sink to nowhere
-  debugNodes.values.foreach(_ := IntSyncCrossingSource() := NullIntSource())
-  nmiNodes.values.foreach(_ := BundleBridgeSource[NMI]())
+  val clintOpt = None
+  val debugOpt = None
+  val plicOpt = None
 
   override lazy val module = new GroundTestSubsystemModuleImp(this)
 }
diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index 7c27645a236..3dd152fd764 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -30,8 +30,8 @@ case object MBUS extends TLBusWrapperLocation("subsystem_mbus")
 case object CBUS extends TLBusWrapperLocation("subsystem_cbus")
 case object L2   extends TLBusWrapperLocation("subsystem_l2")
 
-case class ClusterSBUS(val clusterId: Int) extends TLBusWrapperLocation(s"cluster_${clusterId}_sbus")
-case class ClusterCBUS(val clusterId: Int) extends TLBusWrapperLocation(s"cluster_${clusterId}_cbus")
+case class CSBUS(val clusterId: Int) extends TLBusWrapperLocation(s"cluster_${clusterId}_sbus")
+case class CCBUS(val clusterId: Int) extends TLBusWrapperLocation(s"cluster_${clusterId}_cbus")
 
 /** Parameterizes the subsystem in terms of optional clock-crossings
   *   that are insertable between some of the five traditional tilelink bus wrappers.
@@ -109,3 +109,14 @@ case class CoherentBusTopologyParams(
       nodeBinding = BIND_QUERY))
   )
 )
+
+case class ClusterBusTopologyParams(
+  clusterId: Int,
+  csbus: SystemBusParams,
+  ccbus: PeripheryBusParams
+) extends TLBusWrapperTopology(
+  instantiations = List(
+    (CSBUS(clusterId), csbus),
+    (CCBUS(clusterId), ccbus)),
+  connections = Nil
+)
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 9b654b5135f..b70fc4cb64b 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -16,7 +16,8 @@ class BaseSubsystemConfig extends Config ((site, here, up) => {
   // Tile parameters
   case PgLevels => if (site(XLen) == 64) 3 /* Sv39 */ else 2 /* Sv32 */
   case XLen => 64 // Applies to all cores
-  case MaxHartIdBits => log2Up((site(TilesLocated(InSubsystem)).map(_.tileParams.hartId) :+ 0).max+1)
+  case MaxHartIdBits => log2Up((site(PossibleTileLocations).flatMap(loc => site(TilesLocated(loc)))
+      .map(_.tileParams.hartId) :+ 0).max+1)
   // Interconnect parameters
   case SystemBusKey => SystemBusParams(
     beatBytes = site(XLen)/8,
@@ -43,6 +44,7 @@ class BaseSubsystemConfig extends Config ((site, here, up) => {
   case CLINTKey => Some(CLINTParams())
   case PLICKey => Some(PLICParams())
   case TilesLocated(InSubsystem) => Nil
+  case PossibleTileLocations => Seq(InSubsystem)
 })
 
 /* Composable partial function Configs to set individual parameters */
@@ -90,10 +92,11 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
 class WithNBigCores(
   n: Int,
   overrideIdOffset: Option[Int] = None,
-  crossing: RocketCrossingParams = RocketCrossingParams()
+  crossing: RocketCrossingParams = RocketCrossingParams(),
+  location: HierarchicalLocation = InSubsystem
 ) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => {
-    val prev = up(TilesLocated(InSubsystem), site)
+  case TilesLocated(`location`) => {
+    val prev = up(TilesLocated(location), site)
     val idOffset = overrideIdOffset.getOrElse(prev.size)
     val big = RocketTileParams(
       core   = RocketCoreParams(mulDiv = Some(MulDivParams(
@@ -215,6 +218,24 @@ class With1TinyCore extends Config((site, here, up) => {
   }
 })
 
+class WithCluster(
+  clusterId: Int,
+  location: HierarchicalLocation = InSubsystem,
+  crossing: RocketCrossingParams = RocketCrossingParams() // TODO make this not rocket
+) extends Config((site, here, up) => {
+  case ClustersLocated(`location`) => up(ClustersLocated(location)) :+ ClusterAttachParams(
+    ClusterParams(clusterId = clusterId),
+    crossing)
+  case TLNetworkTopologyLocated(InCluster(`clusterId`)) => List(
+    ClusterBusTopologyParams(
+      clusterId = clusterId,
+      csbus = site(SystemBusKey),
+      ccbus = site(ControlBusKey).copy(errorDevice = None)
+    )
+  )
+  case PossibleTileLocations => up(PossibleTileLocations) :+ InCluster(clusterId)
+})
+
 class WithNBanks(n: Int) extends Config((site, here, up) => {
   case BankedL2Key => up(BankedL2Key, site).copy(nBanks = n)
 })
@@ -664,3 +685,7 @@ class WithCloneRocketTiles(n: Int = 1, cloneHart: Int = 0, overrideIdOffset: Opt
   }
 })
 
+
+class WithNClockGroups(n: Int) extends Config((site, here, up) => {
+  case SubsystemDriveAsyncClockGroupsKey => up(SubsystemDriveAsyncClockGroupsKey).map(_.copy(num=n))
+})
diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
index 3eabde94917..d48f4fa2863 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/Element.scala
@@ -59,9 +59,6 @@ abstract class BaseElement (val crossing: ClockCrossingType)(implicit p: Paramet
   def masterNode: TLOutwardNode
   def slaveNode: TLInwardNode
 
-  val traceCoreNodes: Map[Int, BundleBridgeOutwardNode[TraceCoreInterface]]
-  val traceNodes: Map[Int, BundleBridgeOutwardNode[TraceBundle]]
-
   /** Helper function to insert additional buffers on master ports at the boundary of the tile.
     *
     * The boundary buffering needed to cut feed-through paths is
diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/ElementPRCIDomain.scala
index c1c9fc99433..f80eb29c0d1 100644
--- a/src/main/scala/subsystem/ElementPRCIDomain.scala
+++ b/src/main/scala/subsystem/ElementPRCIDomain.scala
@@ -50,27 +50,6 @@ abstract class ElementPRCIDomain[T <: BaseElement](
   val clockNode = FixedClockBroadcast(None) :=* tapClockNode
   lazy val clockBundle = tapClockNode.in.head._1
 
-  /** Node to broadcast legacy "raw" instruction trace while surpressing it during (async) reset. */
-  val traceNodes: Map[Int, BundleBridgeIdentityNode[TraceBundle]]
-  /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
-  val traceCoreNodes: Map[Int, BundleBridgeIdentityNode[TraceCoreInterface]]
-
-  /** Function to handle all trace crossings when tile is instantiated inside domains */
-  def crossTracesOut(): Unit = this {
-    require(element.traceNodes.size == traceNodes.size)
-    require(element.traceCoreNodes.size == traceCoreNodes.size)
-    for (i <- 0 until traceNodes.size) {
-      val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
-        resetCrossingType = crossingParams.resetCrossingType)
-      traceNodes(i) :*= traceNexusNode := element.traceNodes(i)
-    }
-    for (i <- 0 until traceCoreNodes.size) {
-      val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
-        resetCrossingType = crossingParams.resetCrossingType)
-      traceCoreNodes(i) :*= traceCoreNexusNode := element.traceCoreNodes(i)
-    }
-  }
-
   /** External code looking to connect and clock-cross the interrupts driven into this tile can call this. */
   def crossIntIn(crossingType: ClockCrossingType, tileNode: IntInwardNode): IntInwardNode = {
     // Unlike the other crossing helpers, here nothing is is blocked during reset because we know these are inputs and assume that tile reset is longer than uncore reset
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
index 5199f543bcf..1c2a5759e7d 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -13,7 +13,7 @@ import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.prci.{ClockGroup, ResetCrossingType, ClockGroupNode}
 import freechips.rocketchip.util._
-
+import freechips.rocketchip.rocket.{TracedInstruction}
 
 /** A default implementation of parameterizing the connectivity of the port where the tile is the master.
   *   Optional timing buffers and/or an optional CacheCork can be inserted in the interconnect's clock domain.
@@ -70,17 +70,24 @@ trait InstantiatesElements { this: LazyModule with Attachable =>
     case (instantiated, params) => instantiated :+ params.instantiate(tileParams, instantiated)(p)
   }
 
-  val leafTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile])
-  val totalTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile])
+  val clusterAttachParams: Seq[CanAttachCluster] = p(ClustersLocated(location)).sortBy(_.clusterParams.clusterId)
+  val clusterParams: Seq[ClusterParams] = clusterAttachParams.map(_.clusterParams)
+  val clusterCrossingTypes: Seq[ClockCrossingType] = clusterAttachParams.map(_.crossingParams.crossingType)
+  val cluster_prci_domains: Seq[ClusterPRCIDomain] = clusterAttachParams.foldLeft(Seq[ClusterPRCIDomain]()) {
+    case (instantiated, params) => instantiated :+ params.instantiate(clusterParams, instantiated)(p)
+  }
+
+  val element_prci_domains: Seq[ElementPRCIDomain[_]] = tile_prci_domains ++ cluster_prci_domains
 
-  val element_prci_domains: Seq[ElementPRCIDomain[_]] = tile_prci_domains
+  val leafTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile]).sortBy(_.hartId)
+  val totalTiles: Seq[BaseTile] = (leafTiles ++ cluster_prci_domains.map(_.element.asInstanceOf[Cluster].totalTiles).flatten).sortBy(_.hartId)
 
   // Helper functions for accessing certain parameters that are popular to refer to in subsystem code
-  def nLeafTiles: Int = tileAttachParams.size
-  def nTotalTiles: Int = tileAttachParams.size
-  def leafHartIdList: Seq[Int] = tileParams.map(_.hartId)
-  def totalHartIdList: Seq[Int] = tileParams.map(_.hartId)
-  def localIntCounts: Seq[Int] = tileParams.map(_.core.nLocalInterrupts)
+  def nLeafTiles: Int = leafTiles.size
+  def nTotalTiles: Int = totalTiles.size
+  def leafHartIdList: Seq[Int] = leafTiles.map(_.hartId)
+  def totalHartIdList: Seq[Int] = totalTiles.map(_.hartId)
+  def localIntCounts: Seq[Int] = totalTiles.map(_.tileParams.core.nLocalInterrupts)
 
   require(totalHartIdList.distinct.size == totalTiles.size, s"Every tile must be statically assigned a unique id, but got:\n${totalHartIdList}")
 }
@@ -94,6 +101,10 @@ trait HasElements extends DefaultElementContextType
   tileAttachParams.zip(tile_prci_domains).foreach { case (params, td) =>
     params.connect(td.asInstanceOf[TilePRCIDomain[params.TileType]], this.asInstanceOf[params.TileContextType])
   }
+
+  clusterAttachParams.zip(cluster_prci_domains).foreach { case (params, cd) =>
+    params.connect(cd.asInstanceOf[ClusterPRCIDomain], this.asInstanceOf[params.ClusterContextType])
+  }
 }
 
 /** Provides some Chisel connectivity to certain tile IOs
@@ -134,21 +145,25 @@ trait DefaultElementContextType
   val msipNodes: Map[Int, IntNode]
   val meipNodes: Map[Int, IntNode]
   val seipNodes: Map[Int, IntNode]
-  val plicNodes: Map[Int, IntNode]
+  val tileToPlicNodes: Map[Int, IntNode]
   val debugNodes: Map[Int, IntSyncNode]
   val nmiNodes: Map[Int, BundleBridgeNode[NMI]]
   val tileHartIdNodes: Map[Int, BundleBridgeNode[UInt]]
   val tileResetVectorNodes: Map[Int, BundleBridgeNode[UInt]]
+  val traceCoreNodes: Map[Int, BundleBridgeNode[TraceCoreInterface]]
+  val traceNodes: Map[Int, BundleBridgeNode[Vec[TracedInstruction]]]
 }
 
 /** This trait provides the tile attachment context for the root (outermost) subsystem */
 trait HasElementsRootContext
 { this: HasElements
-    with HasPeripheryDebug
-    with CanHavePeripheryCLINT
-    with CanHavePeripheryPLIC
     with HasTileNotificationSinks
     with InstantiatesElements =>
+
+  val clintOpt: Option[CLINT]
+  val plicOpt: Option[TLPLIC]
+  val debugOpt: Option[TLDebugModule]
+
   val msipNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode() := clintOpt.map(_.intnode).getOrElse(NullIntSource(sources = CLINTConsts.ints)))
   }.toMap
@@ -171,7 +186,7 @@ trait HasElementsRootContext
     (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(seipIONode.get))
   }.toMap
 
-  val plicNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
+  val tileToPlicNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
     plicOpt.map(o => (i, o.intnode :=* IntEphemeralNode()))
   }.flatten.toMap
 
@@ -183,9 +198,12 @@ trait HasElementsRootContext
     node := debugOpt.map(_.intnode).getOrElse(IntSyncCrossingSource() := NullIntSource())
   }
 
-  val nmiHarts = tileParams.filter(_.core.useNMI).map(_.hartId)
+  val nmiHarts = totalTiles.filter(_.tileParams.core.useNMI).map(_.hartId)
   val nmiIONodes = nmiHarts.map { i => (i, BundleBridgeSource[NMI]()) }.toMap
   val nmiNodes: Map[Int, BundleBridgeNode[NMI]] = nmiIONodes.map { case (i, n) =>
     (i, BundleBridgeEphemeralNode[NMI]() := n)
   }.toMap
+
+  val traceCoreNodes: Map[Int, BundleBridgeSink[TraceCoreInterface]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceCoreInterface]()) }.toMap
+  val traceNodes: Map[Int, BundleBridgeSink[Vec[TracedInstruction]]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[Vec[TracedInstruction]]()) }.toMap
 }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index cf8e0e80b18..b83579498b1 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -11,12 +11,16 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.prci.{ClockGroup, ResetCrossingType, ClockGroupNode}
+import freechips.rocketchip.prci._
 import freechips.rocketchip.util._
+import freechips.rocketchip.rocket.{TracedInstruction}
 
 /** Entry point for Config-uring the presence of Tiles */
 case class TilesLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachTile]](Nil)
 
+/** List of HierarchicalLocations which might contain a Tile */
+case object PossibleTileLocations extends Field[Seq[HierarchicalLocation]](Nil)
+
 /** Whether to add timing-closure registers along the path of the hart id
   * as it propagates through the subsystem and into the tile.
   *
@@ -163,6 +167,7 @@ trait CanAttachTile {
     connectPRC(domain, context)
     connectOutputNotifications(domain, context)
     connectInputConstants(domain, context)
+    connectTrace(domain, context)
   }
 
   /** Connect the port where the tile is the master to a TileLink interconnect. */
@@ -218,7 +223,7 @@ trait CanAttachTile {
 
     // 4. Interrupts coming out of the tile are sent to the PLIC,
     //    so might need to be synchronized depending on the Tile's crossing type.
-    context.plicNodes.get(domain.element.hartId).foreach { node =>
+    context.tileToPlicNodes.get(domain.element.hartId).foreach { node =>
       FlipRendering { implicit p => domain.element.intOutwardNode.foreach { out =>
         node :*= domain.crossIntOut(crossingParams.crossingType, out)
       }}
@@ -236,7 +241,6 @@ trait CanAttachTile {
     context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.element.ceaseNode)
     // TODO should context be forced to have a trace sink connected here?
     //      for now this just ensures domain.trace[Core]Node has been crossed without connecting it externally
-    domain.crossTracesOut()
   }
 
   /** Connect inputs to the tile that are assumed to be constant during normal operation, and so are not clock-crossed. */
@@ -271,6 +275,17 @@ trait CanAttachTile {
       domain.element_reset_domain.clockNode := crossingParams.resetCrossingType.injectClockNode := domain.clockNode
     }
   }
+
+  /** Function to handle all trace crossings when tile is instantiated inside domains */
+  def connectTrace(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
+    implicit val p = context.p
+    val traceNexusNode = BundleBridgeBlockDuringReset[Vec[TracedInstruction]](
+      resetCrossingType = crossingParams.resetCrossingType)
+    context.traceNodes(domain.element.hartId) := traceNexusNode := domain.element.traceNode
+    val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
+      resetCrossingType = crossingParams.resetCrossingType)
+    context.traceCoreNodes(domain.element.hartId) :*= traceCoreNexusNode := domain.element.traceCoreNode
+  }
 }
 
 case class CloneTileAttachParams(
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index bb418b9ec9d..36aea77bea6 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -37,6 +37,18 @@ class DualCoreConfig extends Config(new WithNBigCores(2) ++ new WithCoherentBusT
 class DualChannelConfig extends Config(new WithNMemoryChannels(2) ++ new DefaultConfig)
 class EightChannelConfig extends Config(new WithNMemoryChannels(8) ++ new DefaultConfig)
 
+class ClusterConfig extends Config(
+  new WithNBigCores(2, location=InCluster(0), overrideIdOffset=Some(1), crossing=RocketCrossingParams(
+    master=ElementMasterPortParams(where=CSBUS(0)),
+    slave=ElementSlavePortParams(blockerCtrlWhere=CCBUS(0), where=CCBUS(0)),
+    mmioBaseAddressPrefixWhere=CCBUS(0),
+  )) ++
+  new WithNClockGroups(3) ++ // 1 + nClusters TODO fix?
+  new WithCluster(1) ++
+  new WithCluster(0) ++
+  new DefaultConfig
+)
+
 class DualChannelDualBankConfig extends Config(
   new WithNMemoryChannels(2) ++
   new WithNBanks(4) ++ new DefaultConfig
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 4553a41d7fa..8a2795b7e63 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -25,7 +25,6 @@ trait TileParams extends ElementParams {
   val btb: Option[BTBParams]
   val hartId: Int
   val blockerCtrlAddr: Option[BigInt]
-  val clockSinkParams: ClockSinkParameters
 }
 
 abstract class InstantiableTileParams[TileType <: BaseTile]
@@ -277,13 +276,13 @@ abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
   /** Node for the core to drive legacy "raw" instruction trace. */
   val traceSourceNode = BundleBridgeSource(() => new TraceBundle)
   /** Node for external consumers to source a legacy instruction trace from the core. */
-  val traceNodes: Map[Int, BundleBridgeOutwardNode[TraceBundle]] = Map(hartId -> traceSourceNode)
+  val traceNode = traceSourceNode
 
   def traceCoreParams = new TraceCoreParams()
   /** Node for core to drive instruction trace conforming to RISC-V Processor Trace spec V1.0 */
   val traceCoreSourceNode = BundleBridgeSource(() => new TraceCoreInterface(traceCoreParams))
   /** Node for external consumers to source  a V1.0 instruction trace from the core. */
-  val traceCoreNodes: Map[Int, BundleBridgeOutwardNode[TraceCoreInterface]] = Map(hartId -> traceCoreSourceNode)
+  val traceCoreNode = traceCoreSourceNode
 
   /** Node to broadcast collected trace sideband signals into the tile. */
   val traceAuxNexusNode = BundleBridgeNexus[TraceAux](default = Some(() => {
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index 22cc14f1e77..66f2afc4d4c 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -27,12 +27,4 @@ abstract class TilePRCIDomain[T <: BaseTile](
   (implicit p: Parameters)
     extends ElementPRCIDomain[T](clockSinkParams, crossingParams)
 {
-  private val traceSignalName = "trace"
-  private val traceCoreSignalName = "tracecore"
-  /** Node to broadcast legacy "raw" instruction trace while surpressing it during (async) reset. */
-  val traceNode: BundleBridgeIdentityNode[TraceBundle] = BundleBridgeNameNode(traceSignalName)
-  lazy val traceNodes = Map(element.hartId -> traceNode)
-  /** Node to broadcast standardized instruction trace while surpressing it during (async) reset. */
-  val traceCoreNode: BundleBridgeIdentityNode[TraceCoreInterface] = BundleBridgeNameNode(traceCoreSignalName)
-  lazy val traceCoreNodes = Map(element.hartId -> traceCoreNode)
 }

From 8190f4b661c0a56c85d7e885b033ba65a32b8f55 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 30 Jan 2023 11:05:02 -0800
Subject: [PATCH 22/47] Don't print minLatency in TLMonitor to improve dedup

---
 src/main/scala/tilelink/Monitor.scala | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/tilelink/Monitor.scala b/src/main/scala/tilelink/Monitor.scala
index f34e388e54c..013912a562d 100644
--- a/src/main/scala/tilelink/Monitor.scala
+++ b/src/main/scala/tilelink/Monitor.scala
@@ -587,7 +587,7 @@ class TLMonitor(args: TLMonitorArgs, monitorDir: MonitorDirection = MonitorDirec
     }
 
     if (edge.manager.minLatency > 0) {
-      assume(a_set =/= d_clr || !a_set.orR, s"'A' and 'D' concurrent, despite minlatency ${edge.manager.minLatency}" + extra)
+      assume(a_set =/= d_clr || !a_set.orR, s"'A' and 'D' concurrent, despite minlatency > 0" + extra)
     }
 
     inflight := (inflight | a_set) & ~d_clr
@@ -696,7 +696,7 @@ class TLMonitor(args: TLMonitorArgs, monitorDir: MonitorDirection = MonitorDirec
     }
 
     if (edge.manager.minLatency > 0) {
-      assume(a_set_wo_ready =/= d_clr_wo_ready || !a_set_wo_ready.orR, s"'A' and 'D' concurrent, despite minlatency ${edge.manager.minLatency}" + extra)
+      assume(a_set_wo_ready =/= d_clr_wo_ready || !a_set_wo_ready.orR, s"'A' and 'D' concurrent, despite minlatency > 0" + extra)
     }
 
     inflight := (inflight | a_set) & ~d_clr
@@ -804,7 +804,7 @@ class TLMonitor(args: TLMonitorArgs, monitorDir: MonitorDirection = MonitorDirec
 
     if (edge.manager.minLatency > 0) {
       when (c_set_wo_ready.orR) {
-        assume(c_set_wo_ready =/= d_clr_wo_ready, s"'C' and 'D' concurrent, despite minlatency ${edge.manager.minLatency}" + extra)
+        assume(c_set_wo_ready =/= d_clr_wo_ready, s"'C' and 'D' concurrent, despite minlatency > 0" + extra)
       }
     }
 

From c8d29f641a272269cbf0595bd579dbe77163c677 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 30 Jan 2023 11:18:37 -0800
Subject: [PATCH 23/47] Add clustesr

---
 src/main/scala/subsystem/Cluster.scala        | 215 ++++++++++++++++++
 src/main/scala/subsystem/HasElements.scala    |   9 +-
 src/main/scala/subsystem/HasTiles.scala       |   2 +-
 .../scala/subsystem/LookupByClusterId.scala   |  19 ++
 4 files changed, 241 insertions(+), 4 deletions(-)
 create mode 100644 src/main/scala/subsystem/Cluster.scala
 create mode 100644 src/main/scala/subsystem/LookupByClusterId.scala

diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
new file mode 100644
index 00000000000..a5c8163dc63
--- /dev/null
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -0,0 +1,215 @@
+package freechips.rocketchip.subsystem
+
+import chisel3._
+import chisel3.util._
+
+import org.chipsalliance.cde.config.{Field, Parameters}
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.interrupts._
+import freechips.rocketchip.prci._
+import freechips.rocketchip.tile.{RocketTile, NMI, TraceBundle}
+import freechips.rocketchip.subsystem._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.devices.debug.{TLDebugModule}
+import freechips.rocketchip.devices.tilelink._
+import freechips.rocketchip.util._
+
+case class ClustersLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachCluster]](Nil)
+
+case class ClusterParams(
+  val clusterId: Int,
+  val clockSinkParams: ClockSinkParameters = ClockSinkParameters()
+) extends ElementParams {
+  val name = s"cluster_$clusterId"
+  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByClusterIdImpl)(implicit p: Parameters): Cluster = {
+    new Cluster(this, crossing.crossingType, lookup)
+  }
+}
+
+class Cluster(
+  val thisClusterParams: ClusterParams,
+  crossing: ClockCrossingType,
+  lookup: LookupByClusterIdImpl)(implicit p: Parameters) extends BaseElement(crossing)(p)
+    with Attachable
+    with HasConfigurableTLNetworkTopology
+    with InstantiatesElements
+    with HasElements
+{
+  lazy val clusterId = thisClusterParams.clusterId
+  lazy val location = InCluster(clusterId)
+
+  val clockGroupNode = ClockGroupAggregator()
+  println(p(TLNetworkTopologyLocated(InCluster(0))))
+  println(p(TLNetworkTopologyLocated(InCluster(1))))
+  println(thisClusterParams)
+  println(clusterId)
+  println(location)
+  println(tlBusWrapperLocationMap)
+  val csbus = tlBusWrapperLocationMap(CSBUS(clusterId)) // like the sbus in the base subsystem
+  val ccbus = tlBusWrapperLocationMap(CCBUS(clusterId)) // like the cbus in the base subsystem
+
+  csbus.clockGroupNode := clockGroupNode
+  ccbus.clockGroupNode := clockGroupNode
+
+  val slaveNode = ccbus.inwardNode
+  val masterNode = csbus.outwardNode
+
+
+
+  val ibus = LazyModule(new InterruptBusWrapper)
+  ibus.clockNode := csbus.fixedClockNode
+  implicit val asyncClockGroupsNode = p(AsyncClockGroupsKey)()
+
+  lazy val msipNodes = totalHartIdList.map { i => (i, IntIdentityNode()) }.toMap
+  lazy val meipNodes = totalHartIdList.map { i => (i, IntIdentityNode()) }.toMap
+  lazy val seipNodes = totalHartIdList.map { i => (i, IntIdentityNode()) }.toMap
+  lazy val tileToPlicNodes = totalHartIdList.map { i => (i, IntIdentityNode()) }.toMap
+  lazy val debugNodes = totalHartIdList.map { i => (i, IntSyncIdentityNode()) }.toMap
+  lazy val nmiNodes = totalTiles.filter(_.tileParams.core.useNMI).map { t => (t.hartId, BundleBridgeIdentityNode[NMI]()) }.toMap
+  lazy val tileHartIdNodes = totalHartIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.toMap
+  lazy val tileResetVectorNodes = totalHartIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.toMap
+  lazy val traceCoreNodes = totalHartIdList.map { i => (i, BundleBridgeIdentityNode[TraceCoreInterface]()) }.toMap
+  lazy val traceNodes = totalHartIdList.map { i => (i, BundleBridgeIdentityNode[TraceBundle]()) }.toMap
+
+  // TODO fix: shouldn't need to connect dummy notifications
+  tileHaltXbarNode := NullIntSource()
+  tileWFIXbarNode := NullIntSource()
+  tileCeaseXbarNode := NullIntSource()
+
+  override lazy val module = new ClusterModuleImp(this)
+}
+
+class ClusterModuleImp(outer: Cluster) extends BaseElementModuleImp[Cluster](outer)
+
+case class InCluster(id: Int) extends HierarchicalLocation(s"Cluster$id")
+
+class ClusterPRCIDomain(
+  clockSinkParams: ClockSinkParameters,
+  crossingParams: ElementCrossingParamsLike,
+  clusterParams: ClusterParams,
+  lookup: LookupByClusterIdImpl)
+  (implicit p: Parameters) extends ElementPRCIDomain[Cluster](clockSinkParams, crossingParams)
+{
+  val clockGroupNode = ClockGroupAggregator()
+  val element = element_reset_domain {
+    LazyModule(clusterParams.instantiate(crossingParams, lookup))
+  }
+
+  clockNode := ClockGroup() := clockGroupNode
+  element.clockGroupNode := clockGroupNode
+}
+
+
+trait CanAttachCluster {
+  type ClusterContextType <: DefaultElementContextType
+
+  def clusterParams: ClusterParams
+  def crossingParams: ElementCrossingParamsLike
+
+  def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: Seq[ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
+    val clockSinkParams = clusterParams.clockSinkParams.copy(name = Some(clusterParams.name))
+    val cluster_prci_domain = LazyModule(new ClusterPRCIDomain(
+      clockSinkParams, crossingParams, clusterParams, PriorityMuxClusterIdFromSeq(allClusterParams)))
+    cluster_prci_domain
+  }
+
+  def connect(domain: ClusterPRCIDomain, context: ClusterContextType): Unit = {
+    connectMasterPorts(domain, context)
+    connectSlavePorts(domain, context)
+    connectInterrupts(domain, context)
+    connectPRC(domain, context)
+    connectOutputNotifications(domain, context)
+    connectInputConstants(domain, context)
+    connectTrace(domain, context)
+  }
+
+  def connectMasterPorts(domain: ClusterPRCIDomain, context: Attachable): Unit = {
+    implicit val p = context.p
+    val dataBus = context.locateTLBusWrapper(crossingParams.master.where)
+    dataBus.coupleFrom(clusterParams.name) { bus =>
+      bus :=* crossingParams.master.injectNode(context) :=* domain.crossMasterPort(crossingParams.crossingType)
+    }
+  }
+  def connectSlavePorts(domain: ClusterPRCIDomain, context: Attachable): Unit = {
+    implicit val p = context.p
+    val controlBus = context.locateTLBusWrapper(crossingParams.slave.where)
+    controlBus.coupleTo(clusterParams.name) { bus =>
+      domain.crossSlavePort(crossingParams.crossingType) :*= crossingParams.slave.injectNode(context) :*= TLWidthWidget(controlBus.beatBytes) :*= bus
+    }
+  }
+  def connectInterrupts(domain: ClusterPRCIDomain, context: ClusterContextType): Unit = {
+    implicit val p = context.p
+
+    domain.element.debugNodes.foreach { case (hartid, node) =>
+      node := context.debugNodes(hartid)
+    }
+
+    domain.element.msipNodes.foreach { case (hartid, node) =>
+      domain.crossIntIn(crossingParams.crossingType, node) := context.msipNodes(hartid)
+    }
+
+    domain.element.meipNodes.foreach { case (hartid, node) =>
+      domain.crossIntIn(crossingParams.crossingType, node) := context.meipNodes(hartid)
+    }
+
+    domain.element.seipNodes.foreach { case (hartid, node) =>
+      domain.crossIntIn(crossingParams.crossingType, node) := context.seipNodes(hartid)
+    }
+
+    domain.element.tileToPlicNodes.foreach { case (hartid, node) =>
+      FlipRendering { implicit p =>
+        context.tileToPlicNodes(hartid) :=* domain.crossIntOut(crossingParams.crossingType, node) }
+    }
+    context.ibus.fromSync :=* domain.crossIntOut(crossingParams.crossingType, domain.element.ibus.toPLIC)
+
+    domain.element.nmiNodes.foreach { case (hartid, node) =>
+      node := context.nmiNodes(hartid)
+    }
+  }
+
+  def connectPRC(domain: ClusterPRCIDomain, context: ClusterContextType): Unit = {
+    implicit val p = context.p
+    domain.clockGroupNode := context.asyncClockGroupsNode
+    domain {
+      domain.element_reset_domain.clockNode := crossingParams.resetCrossingType.injectClockNode := domain.clockNode
+    }
+  }
+
+  def connectOutputNotifications(domain: ClusterPRCIDomain, context: ClusterContextType): Unit = {
+    implicit val p = context.p
+    context.tileHaltXbarNode  :=* domain.crossIntOut(NoCrossing, domain.element.tileHaltXbarNode)
+    context.tileWFIXbarNode   :=* domain.crossIntOut(NoCrossing, domain.element.tileWFIXbarNode)
+    context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.element.tileCeaseXbarNode)
+
+  }
+
+  def connectInputConstants(domain: ClusterPRCIDomain, context: ClusterContextType): Unit = {
+    implicit val p = context.p
+    val tlBusToGetPrefixFrom = context.locateTLBusWrapper(crossingParams.mmioBaseAddressPrefixWhere)
+    domain.element.tileHartIdNodes.foreach { case (hartid, node) =>
+      node := context.tileHartIdNodes(hartid)
+    }
+    domain.element.tileResetVectorNodes.foreach { case (hartid, node) =>
+      node := context.tileResetVectorNodes(hartid)
+    }
+  }
+
+  def connectTrace(domain: ClusterPRCIDomain, context: ClusterContextType): Unit = {
+    implicit val p = context.p
+    domain.element.traceNodes.foreach { case (hartid, node) =>
+      val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
+        resetCrossingType = crossingParams.resetCrossingType)
+      context.traceNodes(hartid) := traceNexusNode := node
+    }
+    domain.element.traceCoreNodes.foreach { case (hartid, node) =>
+      val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
+        resetCrossingType = crossingParams.resetCrossingType)
+      context.traceCoreNodes(hartid) :*= traceCoreNexusNode := node
+    }
+  }
+}
+
+case class ClusterAttachParams(
+  clusterParams: ClusterParams,
+  crossingParams: ElementCrossingParamsLike
+) extends CanAttachCluster
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
index 1c2a5759e7d..fb25ba5733c 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -151,7 +151,7 @@ trait DefaultElementContextType
   val tileHartIdNodes: Map[Int, BundleBridgeNode[UInt]]
   val tileResetVectorNodes: Map[Int, BundleBridgeNode[UInt]]
   val traceCoreNodes: Map[Int, BundleBridgeNode[TraceCoreInterface]]
-  val traceNodes: Map[Int, BundleBridgeNode[Vec[TracedInstruction]]]
+  val traceNodes: Map[Int, BundleBridgeNode[TraceBundle]]
 }
 
 /** This trait provides the tile attachment context for the root (outermost) subsystem */
@@ -165,8 +165,11 @@ trait HasElementsRootContext
   val debugOpt: Option[TLDebugModule]
 
   val msipNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
-    (i, IntEphemeralNode() := clintOpt.map(_.intnode).getOrElse(NullIntSource(sources = CLINTConsts.ints)))
+    (i, IntEphemeralNode())
   }.toMap
+  msipNodes.foreach {
+    _._2 := clintOpt.map(_.intnode).getOrElse(NullIntSource(sources = CLINTConsts.ints))
+  }
 
   val meipIONode = Option.when(plicOpt.isEmpty)(IntNexusNode(
     sourceFn = { _ => IntSourcePortParameters(Seq(IntSourceParameters(1))) },
@@ -205,5 +208,5 @@ trait HasElementsRootContext
   }.toMap
 
   val traceCoreNodes: Map[Int, BundleBridgeSink[TraceCoreInterface]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceCoreInterface]()) }.toMap
-  val traceNodes: Map[Int, BundleBridgeSink[Vec[TracedInstruction]]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[Vec[TracedInstruction]]()) }.toMap
+  val traceNodes: Map[Int, BundleBridgeSink[TraceBundle]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceBundle]()) }.toMap
 }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index b83579498b1..f9fe6f98605 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -279,7 +279,7 @@ trait CanAttachTile {
   /** Function to handle all trace crossings when tile is instantiated inside domains */
   def connectTrace(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
     implicit val p = context.p
-    val traceNexusNode = BundleBridgeBlockDuringReset[Vec[TracedInstruction]](
+    val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
       resetCrossingType = crossingParams.resetCrossingType)
     context.traceNodes(domain.element.hartId) := traceNexusNode := domain.element.traceNode
     val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
diff --git a/src/main/scala/subsystem/LookupByClusterId.scala b/src/main/scala/subsystem/LookupByClusterId.scala
new file mode 100644
index 00000000000..d0c5c21a5ab
--- /dev/null
+++ b/src/main/scala/subsystem/LookupByClusterId.scala
@@ -0,0 +1,19 @@
+// See LICENSE.SiFive for license details.
+
+package freechips.rocketchip.subsystem
+
+import chisel3._
+import chisel3.util._
+
+abstract class LookupByClusterIdImpl {
+  def apply[T <: Data](f: ClusterParams => Option[T], clusterId: UInt): T
+}
+
+case class ClustersWontDeduplicate(t: ClusterParams) extends LookupByClusterIdImpl {
+  def apply[T <: Data](f: ClusterParams => Option[T], clusterId: UInt): T = f(t).get
+}
+
+case class PriorityMuxClusterIdFromSeq(seq: Seq[ClusterParams]) extends LookupByClusterIdImpl {
+  def apply[T <: Data](f: ClusterParams => Option[T], clusterId: UInt): T =
+    PriorityMux(seq.collect { case t if f(t).isDefined => (t.clusterId.U === clusterId) -> f(t).get })
+}

From 91a14ee71f6e38ba5f4418d5ed5958ff667ec300 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 30 Jan 2023 13:00:56 -0800
Subject: [PATCH 24/47] Rename staticHartId to tileId

---
 src/main/scala/groundtest/Configs.scala    |  6 ++--
 src/main/scala/groundtest/TraceGen.scala   |  6 ++--
 src/main/scala/rocket/Frontend.scala       |  6 ++--
 src/main/scala/rocket/HellaCache.scala     | 10 +++----
 src/main/scala/subsystem/Cluster.scala     | 26 +++++++----------
 src/main/scala/subsystem/Configs.scala     | 34 ++++++++++++----------
 src/main/scala/subsystem/HasElements.scala | 14 ++++-----
 src/main/scala/subsystem/HasTiles.scala    | 25 +++++++++-------
 src/main/scala/system/Configs.scala        |  2 +-
 src/main/scala/tile/BaseTile.scala         | 12 ++------
 src/main/scala/tile/LookupByHartId.scala   |  6 ++--
 src/main/scala/tile/RocketTile.scala       |  6 ++--
 12 files changed, 73 insertions(+), 80 deletions(-)

diff --git a/src/main/scala/groundtest/Configs.scala b/src/main/scala/groundtest/Configs.scala
index 4a4b37d2bcd..cf7a283e732 100644
--- a/src/main/scala/groundtest/Configs.scala
+++ b/src/main/scala/groundtest/Configs.scala
@@ -36,19 +36,18 @@ class GroundTestBaseConfig extends Config(
 
 class WithTraceGen(
   n: Int = 2,
-  overrideIdOffset: Option[Int] = None,
   overrideMemOffset: Option[BigInt] = None)(
   params: Seq[DCacheParams] = List.fill(n){ DCacheParams(nSets = 16, nWays = 1) },
   nReqs: Int = 8192
 ) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     val memOffset: BigInt = overrideMemOffset.orElse(site(ExtMem).map(_.master.base)).getOrElse(0x0L)
     params.zipWithIndex.map { case (dcp, i) =>
       TraceGenTileAttachParams(
         tileParams = TraceGenParams(
-          hartId = i + idOffset,
+          tileId = i + idOffset,
           dcache = Some(dcp),
           wordBits = site(XLen),
           addrBits = 32,
@@ -68,4 +67,5 @@ class WithTraceGen(
       )
     } ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })
diff --git a/src/main/scala/groundtest/TraceGen.scala b/src/main/scala/groundtest/TraceGen.scala
index 1b4457f50b7..4abed974664 100644
--- a/src/main/scala/groundtest/TraceGen.scala
+++ b/src/main/scala/groundtest/TraceGen.scala
@@ -68,14 +68,14 @@ case class TraceGenParams(
     memStart: BigInt, //p(ExtMem).base
     numGens: Int,
     dcache: Option[DCacheParams] = Some(DCacheParams()),
-    hartId: Int = 0
+    tileId: Int = 0
 ) extends InstantiableTileParams[TraceGenTile] with GroundTestTileParams
 {
   def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
     new TraceGenTile(this, crossing, lookup)
   }
   val blockerCtrlAddr = None
-  val name = s"tracegen_$hartId"
+  val name = s"tracegen_$tileId"
   val clockSinkParams = ClockSinkParameters()
 }
 
@@ -643,5 +643,5 @@ class TraceGenTileModuleImp(outer: TraceGenTile) extends GroundTestTileModuleImp
   status.timeout.bits := 0.U
   status.error.valid := false.B
 
-  assert(!tracegen.io.timeout, s"TraceGen tile ${outer.tileParams.hartId}: request timed out")
+  assert(!tracegen.io.timeout, s"TraceGen tile ${outer.tileParams.tileId}: request timed out")
 }
diff --git a/src/main/scala/rocket/Frontend.scala b/src/main/scala/rocket/Frontend.scala
index f6c0ad52b68..30297c5033c 100644
--- a/src/main/scala/rocket/Frontend.scala
+++ b/src/main/scala/rocket/Frontend.scala
@@ -61,9 +61,9 @@ class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) {
   val progress = Output(Bool())
 }
 
-class Frontend(val icacheParams: ICacheParams, staticIdForMetadataUseOnly: Int)(implicit p: Parameters) extends LazyModule {
+class Frontend(val icacheParams: ICacheParams, tileId: Int)(implicit p: Parameters) extends LazyModule {
   lazy val module = new FrontendModule(this)
-  val icache = LazyModule(new ICache(icacheParams, staticIdForMetadataUseOnly))
+  val icache = LazyModule(new ICache(icacheParams, tileId))
   val masterNode = icache.masterNode
   val slaveNode = icache.slaveNode
   val resetVectorSinkNode = BundleBridgeSink[UInt](Some(() => UInt(masterNode.edges.out.head.bundle.addressBits.W)))
@@ -383,7 +383,7 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
 /** Mix-ins for constructing tiles that have an ICache-based pipeline frontend */
 trait HasICacheFrontend extends CanHavePTW { this: BaseTile =>
   val module: HasICacheFrontendModule
-  val frontend = LazyModule(new Frontend(tileParams.icache.get, staticIdForMetadataUseOnly))
+  val frontend = LazyModule(new Frontend(tileParams.icache.get, tileId))
   tlMasterXbar.node := TLWidthWidget(tileParams.icache.get.rowBits/8) := frontend.masterNode
   connectTLSlave(frontend.slaveNode, tileParams.core.fetchBytes)
   frontend.icache.hartIdSinkNodeOpt.foreach { _ := hartIdNexusNode }
diff --git a/src/main/scala/rocket/HellaCache.scala b/src/main/scala/rocket/HellaCache.scala
index 7360ab89fc0..41c8bdfdfe8 100644
--- a/src/main/scala/rocket/HellaCache.scala
+++ b/src/main/scala/rocket/HellaCache.scala
@@ -188,17 +188,17 @@ class HellaCacheIO(implicit p: Parameters) extends CoreBundle()(p) {
 
 /** Base classes for Diplomatic TL2 HellaCaches */
 
-abstract class HellaCache(staticIdForMetadataUseOnly: Int)(implicit p: Parameters) extends LazyModule
+abstract class HellaCache(tileId: Int)(implicit p: Parameters) extends LazyModule
     with HasNonDiplomaticTileParameters {
   protected val cfg = tileParams.dcache.get
 
   protected def cacheClientParameters = cfg.scratch.map(x => Seq()).getOrElse(Seq(TLMasterParameters.v1(
-    name          = s"Core ${staticIdForMetadataUseOnly} DCache",
+    name          = s"Core ${tileId} DCache",
     sourceId      = IdRange(0, 1 max cfg.nMSHRs),
     supportsProbe = TransferSizes(cfg.blockBytes, cfg.blockBytes))))
 
   protected def mmioClientParameters = Seq(TLMasterParameters.v1(
-    name          = s"Core ${staticIdForMetadataUseOnly} DCache MMIO",
+    name          = s"Core ${tileId} DCache MMIO",
     sourceId      = IdRange(firstMMIO, firstMMIO + cfg.nMMIOs),
     requestFifo   = true))
 
@@ -254,9 +254,9 @@ case object BuildHellaCache extends Field[BaseTile => Parameters => HellaCache](
 object HellaCacheFactory {
   def apply(tile: BaseTile)(p: Parameters): HellaCache = {
     if (tile.tileParams.dcache.get.nMSHRs == 0)
-      new DCache(tile.staticIdForMetadataUseOnly, tile.crossing)(p)
+      new DCache(tile.tileId, tile.crossing)(p)
     else
-      new NonBlockingDCache(tile.staticIdForMetadataUseOnly)(p)
+      new NonBlockingDCache(tile.tileId)(p)
   }
 }
 
diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index a5c8163dc63..fda1ef3843c 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -39,12 +39,6 @@ class Cluster(
   lazy val location = InCluster(clusterId)
 
   val clockGroupNode = ClockGroupAggregator()
-  println(p(TLNetworkTopologyLocated(InCluster(0))))
-  println(p(TLNetworkTopologyLocated(InCluster(1))))
-  println(thisClusterParams)
-  println(clusterId)
-  println(location)
-  println(tlBusWrapperLocationMap)
   val csbus = tlBusWrapperLocationMap(CSBUS(clusterId)) // like the sbus in the base subsystem
   val ccbus = tlBusWrapperLocationMap(CCBUS(clusterId)) // like the cbus in the base subsystem
 
@@ -60,16 +54,16 @@ class Cluster(
   ibus.clockNode := csbus.fixedClockNode
   implicit val asyncClockGroupsNode = p(AsyncClockGroupsKey)()
 
-  lazy val msipNodes = totalHartIdList.map { i => (i, IntIdentityNode()) }.toMap
-  lazy val meipNodes = totalHartIdList.map { i => (i, IntIdentityNode()) }.toMap
-  lazy val seipNodes = totalHartIdList.map { i => (i, IntIdentityNode()) }.toMap
-  lazy val tileToPlicNodes = totalHartIdList.map { i => (i, IntIdentityNode()) }.toMap
-  lazy val debugNodes = totalHartIdList.map { i => (i, IntSyncIdentityNode()) }.toMap
-  lazy val nmiNodes = totalTiles.filter(_.tileParams.core.useNMI).map { t => (t.hartId, BundleBridgeIdentityNode[NMI]()) }.toMap
-  lazy val tileHartIdNodes = totalHartIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.toMap
-  lazy val tileResetVectorNodes = totalHartIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.toMap
-  lazy val traceCoreNodes = totalHartIdList.map { i => (i, BundleBridgeIdentityNode[TraceCoreInterface]()) }.toMap
-  lazy val traceNodes = totalHartIdList.map { i => (i, BundleBridgeIdentityNode[TraceBundle]()) }.toMap
+  lazy val msipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
+  lazy val meipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
+  lazy val seipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
+  lazy val tileToPlicNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
+  lazy val debugNodes = totalTileIdList.map { i => (i, IntSyncIdentityNode()) }.toMap
+  lazy val nmiNodes = totalTiles.filter(_.tileParams.core.useNMI).map { t => (t.tileId, BundleBridgeIdentityNode[NMI]()) }.toMap
+  lazy val tileHartIdNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.toMap
+  lazy val tileResetVectorNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.toMap
+  lazy val traceCoreNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceCoreInterface]()) }.toMap
+  lazy val traceNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceBundle]()) }.toMap
 
   // TODO fix: shouldn't need to connect dummy notifications
   tileHaltXbarNode := NullIntSource()
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index b70fc4cb64b..7804551f192 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -17,7 +17,7 @@ class BaseSubsystemConfig extends Config ((site, here, up) => {
   case PgLevels => if (site(XLen) == 64) 3 /* Sv39 */ else 2 /* Sv32 */
   case XLen => 64 // Applies to all cores
   case MaxHartIdBits => log2Up((site(PossibleTileLocations).flatMap(loc => site(TilesLocated(loc)))
-      .map(_.tileParams.hartId) :+ 0).max+1)
+      .map(_.tileParams.tileId) :+ 0).max+1)
   // Interconnect parameters
   case SystemBusKey => SystemBusParams(
     beatBytes = site(XLen)/8,
@@ -91,13 +91,12 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
 
 class WithNBigCores(
   n: Int,
-  overrideIdOffset: Option[Int] = None,
   crossing: RocketCrossingParams = RocketCrossingParams(),
   location: HierarchicalLocation = InSubsystem
 ) extends Config((site, here, up) => {
   case TilesLocated(`location`) => {
     val prev = up(TilesLocated(location), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     val big = RocketTileParams(
       core   = RocketCoreParams(mulDiv = Some(MulDivParams(
         mulUnroll = 8,
@@ -111,20 +110,21 @@ class WithNBigCores(
         rowBits = site(SystemBusKey).beatBits,
         blockBytes = site(CacheBlockBytes))))
     List.tabulate(n)(i => RocketTileAttachParams(
-      big.copy(hartId = i + idOffset),
+      big.copy(tileId = i + idOffset),
       crossing
     )) ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })
 
 class WithNMedCores(
   n: Int,
-  overrideIdOffset: Option[Int] = None,
-  crossing: RocketCrossingParams = RocketCrossingParams()
+  crossing: RocketCrossingParams = RocketCrossingParams(),
+  location: HierarchicalLocation = InSubsystem
 ) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => {
-    val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+  case TilesLocated(`location`) => {
+    val prev = up(TilesLocated(location), site)
+    val idOffset = up(NumTiles)
     val med = RocketTileParams(
       core = RocketCoreParams(fpu = None),
       btb = None,
@@ -144,20 +144,21 @@ class WithNMedCores(
         nTLBWays = 4,
         blockBytes = site(CacheBlockBytes))))
     List.tabulate(n)(i => RocketTileAttachParams(
-      med.copy(hartId = i + idOffset),
+      med.copy(tileId = i + idOffset),
       crossing
     )) ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })
 
 class WithNSmallCores(
   n: Int,
-  overrideIdOffset: Option[Int] = None,
-  crossing: RocketCrossingParams = RocketCrossingParams()
+  crossing: RocketCrossingParams = RocketCrossingParams(),
+  location: HierarchicalLocation = InSubsystem
 ) extends Config((site, here, up) => {
   case TilesLocated(InSubsystem) => {
     val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
+    val idOffset = up(NumTiles)
     val small = RocketTileParams(
       core = RocketCoreParams(useVM = false, fpu = None),
       btb = None,
@@ -177,10 +178,11 @@ class WithNSmallCores(
         nTLBWays = 4,
         blockBytes = site(CacheBlockBytes))))
     List.tabulate(n)(i => RocketTileAttachParams(
-      small.copy(hartId = i + idOffset),
+      small.copy(tileId = i + idOffset),
       crossing
     )) ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })
 
 class With1TinyCore extends Config((site, here, up) => {
@@ -216,6 +218,8 @@ class With1TinyCore extends Config((site, here, up) => {
         master = ElementMasterPortParams())
     ))
   }
+  case NumTiles => 1
+  case ClustersLocated(_) => Nil
 })
 
 class WithCluster(
@@ -679,7 +683,7 @@ class WithCloneRocketTiles(n: Int = 1, cloneHart: Int = 0, overrideIdOffset: Opt
     val tileAttachParams = prev(cloneHart).asInstanceOf[RocketTileAttachParams]
     (0 until n).map { i =>
       CloneTileAttachParams(cloneHart, tileAttachParams.copy(
-        tileParams = tileAttachParams.tileParams.copy(hartId = i + idOffset)
+        tileParams = tileAttachParams.tileParams.copy(tileId = i + idOffset)
       ))
     } ++ prev
   }
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
index fb25ba5733c..24525555014 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -61,7 +61,7 @@ trait InstantiatesElements { this: LazyModule with Attachable =>
     * Note that these ids, which are often used as the tiles' default hartid input,
     * may or may not be those actually reflected at runtime in e.g. the $mhartid CSR
     */
-  val tileAttachParams: Seq[CanAttachTile] = p(TilesLocated(location)).sortBy(_.tileParams.hartId)
+  val tileAttachParams: Seq[CanAttachTile] = p(TilesLocated(location)).sortBy(_.tileParams.tileId)
   val tileParams: Seq[TileParams] = tileAttachParams.map(_.tileParams)
   val tileCrossingTypes: Seq[ClockCrossingType] = tileAttachParams.map(_.crossingParams.crossingType)
 
@@ -79,17 +79,17 @@ trait InstantiatesElements { this: LazyModule with Attachable =>
 
   val element_prci_domains: Seq[ElementPRCIDomain[_]] = tile_prci_domains ++ cluster_prci_domains
 
-  val leafTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile]).sortBy(_.hartId)
-  val totalTiles: Seq[BaseTile] = (leafTiles ++ cluster_prci_domains.map(_.element.asInstanceOf[Cluster].totalTiles).flatten).sortBy(_.hartId)
+  val leafTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile]).sortBy(_.tileId)
+  val totalTiles: Seq[BaseTile] = (leafTiles ++ cluster_prci_domains.map(_.element.asInstanceOf[Cluster].totalTiles).flatten).sortBy(_.tileId)
 
   // Helper functions for accessing certain parameters that are popular to refer to in subsystem code
   def nLeafTiles: Int = leafTiles.size
   def nTotalTiles: Int = totalTiles.size
-  def leafHartIdList: Seq[Int] = leafTiles.map(_.hartId)
-  def totalHartIdList: Seq[Int] = totalTiles.map(_.hartId)
+  def leafTileIdList: Seq[Int] = leafTiles.map(_.tileId)
+  def totalTileIdList: Seq[Int] = totalTiles.map(_.tileId)
   def localIntCounts: Seq[Int] = totalTiles.map(_.tileParams.core.nLocalInterrupts)
 
-  require(totalHartIdList.distinct.size == totalTiles.size, s"Every tile must be statically assigned a unique id, but got:\n${totalHartIdList}")
+  require(totalTileIdList.distinct.size == totalTiles.size, s"Every tile must be statically assigned a unique id, but got:\n${totalTileIdList}")
 }
 
 /** HasTiles instantiates and also connects a Config-urable sequence of tiles of any type to subsystem interconnect resources. */
@@ -201,7 +201,7 @@ trait HasElementsRootContext
     node := debugOpt.map(_.intnode).getOrElse(IntSyncCrossingSource() := NullIntSource())
   }
 
-  val nmiHarts = totalTiles.filter(_.tileParams.core.useNMI).map(_.hartId)
+  val nmiHarts = totalTiles.filter(_.tileParams.core.useNMI).map(_.tileId)
   val nmiIONodes = nmiHarts.map { i => (i, BundleBridgeSource[NMI]()) }.toMap
   val nmiNodes: Map[Int, BundleBridgeNode[NMI]] = nmiIONodes.map { case (i, n) =>
     (i, BundleBridgeEphemeralNode[NMI]() := n)
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index f9fe6f98605..39392b303ab 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -21,6 +21,9 @@ case class TilesLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachTi
 /** List of HierarchicalLocations which might contain a Tile */
 case object PossibleTileLocations extends Field[Seq[HierarchicalLocation]](Nil)
 
+/** For determining static tile id */
+case object NumTiles extends Field[Int](0)
+
 /** Whether to add timing-closure registers along the path of the hart id
   * as it propagates through the subsystem and into the tile.
   *
@@ -68,7 +71,7 @@ trait HasTileInputConstants { this: LazyModule with Attachable with Instantiates
   val tileHartIdNexusNode = LazyModule(new BundleBridgeNexus[UInt](
     inputFn = BundleBridgeNexus.orReduction[UInt](registered = p(InsertTimingClosureRegistersOnHartIds)) _,
     outputFn = (prefix: UInt, n: Int) =>  Seq.tabulate(n) { i =>
-      val y = dontTouch(prefix | totalHartIdList(i).U(p(MaxHartIdBits).W)) // dontTouch to keep constant prop from breaking tile dedup
+      val y = dontTouch(prefix | totalTileIdList(i).U(p(MaxHartIdBits).W)) // dontTouch to keep constant prop from breaking tile dedup
       if (p(InsertTimingClosureRegistersOnHartIds)) BundleBridgeNexus.safeRegNext(y) else y
     },
     default = Some(() => 0.U(p(MaxHartIdBits).W)),
@@ -199,23 +202,23 @@ trait CanAttachTile {
 
     // 1. Debug interrupt is definitely asynchronous in all cases.
     domain.element.intInwardNode := domain { IntSyncAsyncCrossingSink(3) } :=
-      context.debugNodes(domain.element.hartId)
+      context.debugNodes(domain.element.tileId)
 
     // 2. The CLINT and PLIC output interrupts are synchronous to the TileLink bus clock,
     //    so might need to be synchronized depending on the Tile's crossing type.
 
     //    From CLINT: "msip" and "mtip"
     domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
-      context.msipNodes(domain.element.hartId)
+      context.msipNodes(domain.element.tileId)
 
     //    From PLIC: "meip"
     domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
-      context.meipNodes(domain.element.hartId)
+      context.meipNodes(domain.element.tileId)
 
     //    From PLIC: "seip" (only if supervisor mode is enabled)
     if (domain.element.tileParams.core.hasSupervisorMode) {
       domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
-        context.seipNodes(domain.element.hartId)
+        context.seipNodes(domain.element.tileId)
     }
 
     // 3. Local Interrupts ("lip") are required to already be synchronous to the Tile's clock.
@@ -223,14 +226,14 @@ trait CanAttachTile {
 
     // 4. Interrupts coming out of the tile are sent to the PLIC,
     //    so might need to be synchronized depending on the Tile's crossing type.
-    context.tileToPlicNodes.get(domain.element.hartId).foreach { node =>
+    context.tileToPlicNodes.get(domain.element.tileId).foreach { node =>
       FlipRendering { implicit p => domain.element.intOutwardNode.foreach { out =>
         node :*= domain.crossIntOut(crossingParams.crossingType, out)
       }}
     }
 
     // 5. Connect NMI inputs to the tile. These inputs are synchronous to the respective core_clock.
-    domain.element.nmiNode.foreach(_ := context.nmiNodes(domain.element.hartId))
+    domain.element.nmiNode.foreach(_ := context.nmiNodes(domain.element.tileId))
   }
 
   /** Notifications of tile status are connected to be broadcast without needing to be clock-crossed. */
@@ -247,8 +250,8 @@ trait CanAttachTile {
   def connectInputConstants(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
     implicit val p = context.p
     val tlBusToGetPrefixFrom = context.locateTLBusWrapper(crossingParams.mmioBaseAddressPrefixWhere)
-    domain.element.hartIdNode := context.tileHartIdNodes(domain.element.hartId)
-    domain.element.resetVectorNode := context.tileResetVectorNodes(domain.element.hartId)
+    domain.element.hartIdNode := context.tileHartIdNodes(domain.element.tileId)
+    domain.element.resetVectorNode := context.tileResetVectorNodes(domain.element.tileId)
     tlBusToGetPrefixFrom.prefixNode.foreach { domain.element.mmioAddressPrefixNode := _ }
   }
 
@@ -281,10 +284,10 @@ trait CanAttachTile {
     implicit val p = context.p
     val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
       resetCrossingType = crossingParams.resetCrossingType)
-    context.traceNodes(domain.element.hartId) := traceNexusNode := domain.element.traceNode
+    context.traceNodes(domain.element.tileId) := traceNexusNode := domain.element.traceNode
     val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
       resetCrossingType = crossingParams.resetCrossingType)
-    context.traceCoreNodes(domain.element.hartId) :*= traceCoreNexusNode := domain.element.traceCoreNode
+    context.traceCoreNodes(domain.element.tileId) :*= traceCoreNexusNode := domain.element.traceCoreNode
   }
 }
 
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index 36aea77bea6..4202fe62c06 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -38,7 +38,7 @@ class DualChannelConfig extends Config(new WithNMemoryChannels(2) ++ new Default
 class EightChannelConfig extends Config(new WithNMemoryChannels(8) ++ new DefaultConfig)
 
 class ClusterConfig extends Config(
-  new WithNBigCores(2, location=InCluster(0), overrideIdOffset=Some(1), crossing=RocketCrossingParams(
+  new WithNBigCores(2, location=InCluster(0), crossing=RocketCrossingParams(
     master=ElementMasterPortParams(where=CSBUS(0)),
     slave=ElementSlavePortParams(blockerCtrlWhere=CCBUS(0), where=CCBUS(0)),
     mmioBaseAddressPrefixWhere=CCBUS(0),
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 8a2795b7e63..3801d3a410f 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -23,7 +23,7 @@ trait TileParams extends ElementParams {
   val icache: Option[ICacheParams]
   val dcache: Option[DCacheParams]
   val btb: Option[BTBParams]
-  val hartId: Int
+  val tileId: Int // may not be hartid
   val blockerCtrlAddr: Option[BigInt]
 }
 
@@ -76,15 +76,7 @@ trait HasNonDiplomaticTileParameters {
     xLen match { case 32 => 34; case 64 => 56 }
   }
 
-  /** Use staticIdForMetadataUseOnly to emit information during the build or identify a component to diplomacy.
-    *
-    *   Including it in a constructed Chisel circuit by converting it to a UInt will prevent
-    *   Chisel/FIRRTL from being able to deduplicate tiles that are otherwise homogeneous,
-    *   a property which is important for hierarchical place & route flows.
-    */
-  def staticIdForMetadataUseOnly: Int = tileParams.hartId
-  @deprecated("use hartIdSinkNodeOpt.map(_.bundle) or staticIdForMetadataUseOnly", "rocket-chip 1.3")
-  def hartId: Int = staticIdForMetadataUseOnly
+  def tileId: Int = tileParams.tileId
 
   def cacheBlockBytes = p(CacheBlockBytes)
   def lgCacheBlockBytes = log2Up(cacheBlockBytes)
diff --git a/src/main/scala/tile/LookupByHartId.scala b/src/main/scala/tile/LookupByHartId.scala
index 263dd43c66f..631bb8f651e 100644
--- a/src/main/scala/tile/LookupByHartId.scala
+++ b/src/main/scala/tile/LookupByHartId.scala
@@ -10,10 +10,10 @@ abstract class LookupByHartIdImpl {
 }
 
 case class HartsWontDeduplicate(t: TileParams) extends LookupByHartIdImpl {
-  def apply[T <: Data](f: TileParams => Option[T], hartId: UInt): T = f(t).get
+  def apply[T <: Data](f: TileParams => Option[T], tileId: UInt): T = f(t).get
 }
 
 case class PriorityMuxHartIdFromSeq(seq: Seq[TileParams]) extends LookupByHartIdImpl {
-  def apply[T <: Data](f: TileParams => Option[T], hartId: UInt): T =
-    PriorityMux(seq.collect { case t if f(t).isDefined => (t.hartId.U === hartId) -> f(t).get })
+  def apply[T <: Data](f: TileParams => Option[T], tileId: UInt): T =
+    PriorityMux(seq.collect { case t if f(t).isDefined => (t.tileId.U === tileId) -> f(t).get })
 }
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index bcfef753f15..d0bcd0234f7 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -22,7 +22,7 @@ case class RocketTileParams(
     dcache: Option[DCacheParams] = Some(DCacheParams()),
     btb: Option[BTBParams] = Some(BTBParams()),
     dataScratchpadBytes: Int = 0,
-    hartId: Int = 0,
+    tileId: Int = 0,
     beuAddr: Option[BigInt] = None,
     blockerCtrlAddr: Option[BigInt] = None,
     clockSinkParams: ClockSinkParameters = ClockSinkParameters(),
@@ -30,7 +30,7 @@ case class RocketTileParams(
     ) extends InstantiableTileParams[RocketTile] {
   require(icache.isDefined)
   require(dcache.isDefined)
-  val name = s"rockettile_$hartId"
+  val name = s"rockettile_$tileId"
   def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
     new RocketTile(this, crossing, lookup)
   }
@@ -100,7 +100,7 @@ class RocketTile private(
   }
 
   ResourceBinding {
-    Resource(cpuDevice, "reg").bind(ResourceAddress(staticIdForMetadataUseOnly))
+    Resource(cpuDevice, "reg").bind(ResourceAddress(tileId))
   }
 
   override lazy val module = new RocketTileModuleImp(this)

From d0ddc636bf97918db19581595f64dd687e1eb9b5 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 30 Jan 2023 13:31:43 -0800
Subject: [PATCH 25/47] Remove CCBUS TLBusWrapperLocations

---
 src/main/scala/subsystem/BusTopology.scala | 7 ++-----
 src/main/scala/subsystem/Cluster.scala     | 4 ++--
 src/main/scala/system/Configs.scala        | 7 ++-----
 3 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index 3dd152fd764..c3e4691d359 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -30,9 +30,6 @@ case object MBUS extends TLBusWrapperLocation("subsystem_mbus")
 case object CBUS extends TLBusWrapperLocation("subsystem_cbus")
 case object L2   extends TLBusWrapperLocation("subsystem_l2")
 
-case class CSBUS(val clusterId: Int) extends TLBusWrapperLocation(s"cluster_${clusterId}_sbus")
-case class CCBUS(val clusterId: Int) extends TLBusWrapperLocation(s"cluster_${clusterId}_cbus")
-
 /** Parameterizes the subsystem in terms of optional clock-crossings
   *   that are insertable between some of the five traditional tilelink bus wrappers.
   *   This class exists for backwards compatiblity reasons but could eventually be retired
@@ -116,7 +113,7 @@ case class ClusterBusTopologyParams(
   ccbus: PeripheryBusParams
 ) extends TLBusWrapperTopology(
   instantiations = List(
-    (CSBUS(clusterId), csbus),
-    (CCBUS(clusterId), ccbus)),
+    (SBUS, csbus),
+    (CBUS, ccbus)),
   connections = Nil
 )
diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index fda1ef3843c..b39e2a841c7 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -39,8 +39,8 @@ class Cluster(
   lazy val location = InCluster(clusterId)
 
   val clockGroupNode = ClockGroupAggregator()
-  val csbus = tlBusWrapperLocationMap(CSBUS(clusterId)) // like the sbus in the base subsystem
-  val ccbus = tlBusWrapperLocationMap(CCBUS(clusterId)) // like the cbus in the base subsystem
+  val csbus = tlBusWrapperLocationMap(SBUS) // like the sbus in the base subsystem
+  val ccbus = tlBusWrapperLocationMap(CBUS) // like the cbus in the base subsystem
 
   csbus.clockGroupNode := clockGroupNode
   ccbus.clockGroupNode := clockGroupNode
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index 4202fe62c06..7cb821e6341 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -38,11 +38,8 @@ class DualChannelConfig extends Config(new WithNMemoryChannels(2) ++ new Default
 class EightChannelConfig extends Config(new WithNMemoryChannels(8) ++ new DefaultConfig)
 
 class ClusterConfig extends Config(
-  new WithNBigCores(2, location=InCluster(0), crossing=RocketCrossingParams(
-    master=ElementMasterPortParams(where=CSBUS(0)),
-    slave=ElementSlavePortParams(blockerCtrlWhere=CCBUS(0), where=CCBUS(0)),
-    mmioBaseAddressPrefixWhere=CCBUS(0),
-  )) ++
+  new WithNBigCores(2, location=InCluster(1)) ++
+  new WithNBigCores(2, location=InCluster(0)) ++
   new WithNClockGroups(3) ++ // 1 + nClusters TODO fix?
   new WithCluster(1) ++
   new WithCluster(0) ++

From 9b22a90f10af20668c81c83dfe6a164e719f6582 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 30 Jan 2023 14:14:08 -0800
Subject: [PATCH 26/47] Improve deduplication of clustesr

---
 src/main/scala/groundtest/TraceGen.scala | 3 ++-
 src/main/scala/subsystem/Cluster.scala   | 9 +++++----
 src/main/scala/subsystem/Element.scala   | 3 ++-
 src/main/scala/subsystem/HasTiles.scala  | 8 ++++----
 src/main/scala/tile/BaseTile.scala       | 2 +-
 src/main/scala/tile/RocketTile.scala     | 3 ++-
 6 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/main/scala/groundtest/TraceGen.scala b/src/main/scala/groundtest/TraceGen.scala
index 4abed974664..9da61ad1fbe 100644
--- a/src/main/scala/groundtest/TraceGen.scala
+++ b/src/main/scala/groundtest/TraceGen.scala
@@ -75,7 +75,8 @@ case class TraceGenParams(
     new TraceGenTile(this, crossing, lookup)
   }
   val blockerCtrlAddr = None
-  val name = s"tracegen_$tileId"
+  val baseName = "tracegentile"
+  val uniqueName = s"${baseName}_$tileId"
   val clockSinkParams = ClockSinkParameters()
 }
 
diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index b39e2a841c7..53edf0663b6 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -20,7 +20,8 @@ case class ClusterParams(
   val clusterId: Int,
   val clockSinkParams: ClockSinkParameters = ClockSinkParameters()
 ) extends ElementParams {
-  val name = s"cluster_$clusterId"
+  val baseName = "cluster"
+  val uniqueName = s"${baseName}_$clusterId"
   def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByClusterIdImpl)(implicit p: Parameters): Cluster = {
     new Cluster(this, crossing.crossingType, lookup)
   }
@@ -101,7 +102,7 @@ trait CanAttachCluster {
   def crossingParams: ElementCrossingParamsLike
 
   def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: Seq[ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
-    val clockSinkParams = clusterParams.clockSinkParams.copy(name = Some(clusterParams.name))
+    val clockSinkParams = clusterParams.clockSinkParams.copy(name = Some(clusterParams.uniqueName))
     val cluster_prci_domain = LazyModule(new ClusterPRCIDomain(
       clockSinkParams, crossingParams, clusterParams, PriorityMuxClusterIdFromSeq(allClusterParams)))
     cluster_prci_domain
@@ -120,14 +121,14 @@ trait CanAttachCluster {
   def connectMasterPorts(domain: ClusterPRCIDomain, context: Attachable): Unit = {
     implicit val p = context.p
     val dataBus = context.locateTLBusWrapper(crossingParams.master.where)
-    dataBus.coupleFrom(clusterParams.name) { bus =>
+    dataBus.coupleFrom(clusterParams.baseName) { bus =>
       bus :=* crossingParams.master.injectNode(context) :=* domain.crossMasterPort(crossingParams.crossingType)
     }
   }
   def connectSlavePorts(domain: ClusterPRCIDomain, context: Attachable): Unit = {
     implicit val p = context.p
     val controlBus = context.locateTLBusWrapper(crossingParams.slave.where)
-    controlBus.coupleTo(clusterParams.name) { bus =>
+    controlBus.coupleTo(clusterParams.baseName) { bus =>
       domain.crossSlavePort(crossingParams.crossingType) :*= crossingParams.slave.injectNode(context) :*= TLWidthWidget(controlBus.beatBytes) :*= bus
     }
   }
diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/Element.scala
index d48f4fa2863..ce35e01032f 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/Element.scala
@@ -15,7 +15,8 @@ import freechips.rocketchip.devices.debug.{TLDebugModule}
 import freechips.rocketchip.devices.tilelink._
 
 trait ElementParams {
-  val name: String
+  val baseName: String // duplicated instances shouuld share a base name
+  val uniqueName: String
   val clockSinkParams: ClockSinkParameters
 }
 
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 39392b303ab..7c1a3c1d743 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -155,7 +155,7 @@ trait CanAttachTile {
 
   /** Narrow waist through which all tiles are intended to pass while being instantiated. */
   def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: Seq[TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
-    val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.name))
+    val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.uniqueName))
     val tile_prci_domain = LazyModule(new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
       val element = self.element_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
     })
@@ -177,7 +177,7 @@ trait CanAttachTile {
   def connectMasterPorts(domain: TilePRCIDomain[TileType], context: Attachable): Unit = {
     implicit val p = context.p
     val dataBus = context.locateTLBusWrapper(crossingParams.master.where)
-    dataBus.coupleFrom(tileParams.name) { bus =>
+    dataBus.coupleFrom(tileParams.baseName) { bus =>
       bus :=* crossingParams.master.injectNode(context) :=* domain.crossMasterPort(crossingParams.crossingType)
     }
   }
@@ -187,7 +187,7 @@ trait CanAttachTile {
     implicit val p = context.p
     DisableMonitors { implicit p =>
       val controlBus = context.locateTLBusWrapper(crossingParams.slave.where)
-      controlBus.coupleTo(tileParams.name) { bus =>
+      controlBus.coupleTo(tileParams.baseName) { bus =>
         domain.crossSlavePort(crossingParams.crossingType) :*= crossingParams.slave.injectNode(context) :*= TLWidthWidget(controlBus.beatBytes) :*= bus
       }
     }
@@ -303,7 +303,7 @@ case class CloneTileAttachParams(
 
   override def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: Seq[TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
     require(sourceHart < instantiatedTiles.size)
-    val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.name))
+    val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.uniqueName))
     val tile_prci_domain = CloneLazyModule(
       new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
         val element = self.element_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 3801d3a410f..060ca985938 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -344,7 +344,7 @@ abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
     new C
   }
 
-  this.suggestName(tileParams.name)
+  this.suggestName(tileParams.baseName)
 }
 
 abstract class BaseTileModuleImp[+L <: BaseTile](outer: L) extends BaseElementModuleImp[L](outer)
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index d0bcd0234f7..6f8025bb7aa 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -30,7 +30,8 @@ case class RocketTileParams(
     ) extends InstantiableTileParams[RocketTile] {
   require(icache.isDefined)
   require(dcache.isDefined)
-  val name = s"rockettile_$tileId"
+  val baseName = "rockettile"
+  val uniqueName = s"${baseName}_$tileId"
   def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
     new RocketTile(this, crossing, lookup)
   }

From 0d950dba7552f8e9193fdad49867a8785688ad25 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 30 Jan 2023 19:59:31 -0800
Subject: [PATCH 27/47] Deprecate basesubsystemn clocking backwards
 compatibility features

---
 src/main/scala/prci/ClockBundles.scala        |  2 +-
 src/main/scala/prci/ClockDomain.scala         |  3 +-
 src/main/scala/prci/ClockGroup.scala          |  6 +--
 src/main/scala/prci/ClockGroupDriver.scala    | 46 -------------------
 src/main/scala/prci/package.scala             |  3 ++
 src/main/scala/subsystem/Attachable.scala     |  4 +-
 src/main/scala/subsystem/BaseSubsystem.scala  | 30 ++++++++----
 src/main/scala/subsystem/Cluster.scala        | 19 ++++----
 src/main/scala/subsystem/Configs.scala        |  4 --
 .../scala/subsystem/ElementPRCIDomain.scala   | 10 ++--
 src/main/scala/subsystem/HasElements.scala    |  2 +-
 src/main/scala/subsystem/HasTiles.scala       |  2 +-
 src/main/scala/subsystem/InterruptBus.scala   |  2 +-
 src/main/scala/subsystem/RTC.scala            |  4 +-
 src/main/scala/system/Configs.scala           |  4 +-
 src/main/scala/system/TestHarness.scala       |  4 +-
 src/main/scala/tilelink/BusWrapper.scala      | 12 ++---
 17 files changed, 65 insertions(+), 92 deletions(-)
 delete mode 100644 src/main/scala/prci/ClockGroupDriver.scala

diff --git a/src/main/scala/prci/ClockBundles.scala b/src/main/scala/prci/ClockBundles.scala
index 900e437a5af..0cf06cca6e5 100644
--- a/src/main/scala/prci/ClockBundles.scala
+++ b/src/main/scala/prci/ClockBundles.scala
@@ -5,7 +5,7 @@ import chisel3._
 import freechips.rocketchip.util.RecordMap
 
 
-class ClockBundle(val params: ClockBundleParameters) extends Bundle
+class ClockBundle(val params: ClockBundleParameters = ClockBundleParameters()) extends Bundle
 {
   val clock = Output(Clock())
   val reset = Output(Reset())
diff --git a/src/main/scala/prci/ClockDomain.scala b/src/main/scala/prci/ClockDomain.scala
index 665372ca500..83255d71e2f 100644
--- a/src/main/scala/prci/ClockDomain.scala
+++ b/src/main/scala/prci/ClockDomain.scala
@@ -4,7 +4,8 @@ import chisel3._
 import org.chipsalliance.cde.config.Parameters
 import freechips.rocketchip.diplomacy._
 
-abstract class Domain(implicit p: Parameters) extends LazyModule with HasDomainCrossing {
+abstract class Domain(implicit p: Parameters) extends LazyModule with HasDomainCrossing
+{
   def clockBundle: ClockBundle
 
   lazy val module = new Impl
diff --git a/src/main/scala/prci/ClockGroup.scala b/src/main/scala/prci/ClockGroup.scala
index ce2b8f98c97..f4c648bafe2 100644
--- a/src/main/scala/prci/ClockGroup.scala
+++ b/src/main/scala/prci/ClockGroup.scala
@@ -4,7 +4,7 @@ package freechips.rocketchip.prci
 import org.chipsalliance.cde.config.Parameters
 import freechips.rocketchip.diplomacy._
 
-case class ClockGroupNode(groupName: String)(implicit valName: ValName)
+case class ClockGroupingNode(groupName: String)(implicit valName: ValName)
   extends MixedNexusNode(ClockGroupImp, ClockImp)(
     dFn = { _ => ClockSourceParameters() },
     uFn = { seq => ClockGroupSinkParameters(name = groupName, members = seq) })
@@ -14,7 +14,7 @@ case class ClockGroupNode(groupName: String)(implicit valName: ValName)
 
 class ClockGroup(groupName: String)(implicit p: Parameters) extends LazyModule
 {
-  val node = ClockGroupNode(groupName)
+  val node = ClockGroupingNode(groupName)
 
   lazy val module = new Impl
   class Impl extends LazyRawModuleImp(this) {
@@ -107,7 +107,7 @@ class FixedClockBroadcast(fixedClockOpt: Option[ClockParameters])(implicit p: Pa
 
 object FixedClockBroadcast
 {
-  def apply(fixedClockOpt: Option[ClockParameters])(implicit p: Parameters, valName: ValName) = LazyModule(new FixedClockBroadcast(fixedClockOpt)).node
+  def apply(fixedClockOpt: Option[ClockParameters] = None)(implicit p: Parameters, valName: ValName) = LazyModule(new FixedClockBroadcast(fixedClockOpt)).node
 }
 
 case class PRCIClockGroupNode()(implicit valName: ValName)
diff --git a/src/main/scala/prci/ClockGroupDriver.scala b/src/main/scala/prci/ClockGroupDriver.scala
deleted file mode 100644
index 63ff42f1e42..00000000000
--- a/src/main/scala/prci/ClockGroupDriver.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-// See LICENSE.SiFive for license details.
-package freechips.rocketchip.prci
-
-import chisel3._
-import org.chipsalliance.cde.config.Parameters
-import freechips.rocketchip.diplomacy.{InModuleBody, ModuleValue, ValName}
-import freechips.rocketchip.util.{RecordMap}
-
-/** Used to parameterize the creation of simple clock group drivers */
-case class ClockGroupDriverParameters(
-  num: Int = 1,
-  driveFn: ClockGroupDriver.DriveFn = ClockGroupDriver.driveFromImplicitClock
-) {
-  def drive(node: ClockGroupEphemeralNode)(implicit p: Parameters, vn: ValName): ModuleValue[RecordMap[ClockBundle]] = {
-    driveFn(node, num, p, vn)
-  }
-}
-
-object ClockGroupDriver {
-  type DriveFn = (ClockGroupEphemeralNode, Int, Parameters, ValName) => ModuleValue[RecordMap[ClockBundle]]
-
-  /** Drive all members of all groups from the Chisel implicit clock */
-  def driveFromImplicitClock: DriveFn = { (groups, num, p, vn) =>
-    implicit val pp = p
-    val dummyClockGroupSourceNode: ClockGroupSourceNode = SimpleClockGroupSource(num)
-    groups :*= dummyClockGroupSourceNode
-    InModuleBody { RecordMap[ClockBundle]() }
-  }
-
-  /** Drive all members of all groups from a flattened IO representation */
-  def driveFromIOs: DriveFn = { (groups, num, p, vn) =>
-    implicit val pp = p
-    val ioClockGroupSourceNode = ClockGroupSourceNode(List.fill(num) { ClockGroupSourceParameters() })
-    groups :*= ioClockGroupSourceNode
-    InModuleBody {
-      val bundles = ioClockGroupSourceNode.out.map(_._1)
-      val elements =  bundles.map(_.member.elements).flatten
-      val io = IO(Flipped(RecordMap(elements.map { case (name, data) =>
-        name -> data.cloneType
-      }:_*)))
-
-      elements.foreach { case (name, data) => io(name).foreach { data := _ } }
-      io.suggestName(vn.name)
-    }
-  }
-}
diff --git a/src/main/scala/prci/package.scala b/src/main/scala/prci/package.scala
index e427add8d43..864126a03f2 100644
--- a/src/main/scala/prci/package.scala
+++ b/src/main/scala/prci/package.scala
@@ -9,6 +9,9 @@ package object prci
   type ClockInwardNode = InwardNodeHandle[ClockSourceParameters, ClockSinkParameters, ClockEdgeParameters, ClockBundle]
   type ClockOutwardNode = OutwardNodeHandle[ClockSourceParameters, ClockSinkParameters, ClockEdgeParameters, ClockBundle]
   type ClockNode = NodeHandle[ClockSourceParameters, ClockSinkParameters, ClockEdgeParameters, ClockBundle, ClockSourceParameters, ClockSinkParameters, ClockEdgeParameters, ClockBundle]
+
+  type ClockGroupNode = NodeHandle[ClockGroupSourceParameters, ClockGroupSinkParameters, ClockGroupEdgeParameters, ClockGroupBundle, ClockGroupSourceParameters, ClockGroupSinkParameters, ClockGroupEdgeParameters, ClockGroupBundle]
+
   def asyncMux[T](xType: ClockCrossingType, async: T, notasync: T): T = xType match {
     case _: AsynchronousCrossing => async
     case _ => notasync
diff --git a/src/main/scala/subsystem/Attachable.scala b/src/main/scala/subsystem/Attachable.scala
index 7809280a110..6a374b4db2d 100644
--- a/src/main/scala/subsystem/Attachable.scala
+++ b/src/main/scala/subsystem/Attachable.scala
@@ -4,7 +4,7 @@ package freechips.rocketchip.subsystem
 
 import org.chipsalliance.cde.config.Parameters
 import freechips.rocketchip.diplomacy.{LazyModule, LazyScope}
-import freechips.rocketchip.prci.ClockGroupEphemeralNode
+import freechips.rocketchip.prci.ClockGroupNode
 import freechips.rocketchip.tilelink.TLBusWrapper
 import freechips.rocketchip.util.{Location, LocationMap}
 
@@ -23,7 +23,7 @@ trait LazyScopeWithParameters extends LazyScope { this: LazyModule =>
 
 /** Layers of hierarchy with this trait contain attachment points for neworks of power, clock, reset, and interrupt resources */
 trait HasPRCILocations extends LazyScopeWithParameters { this: LazyModule =>
-  implicit val asyncClockGroupsNode: ClockGroupEphemeralNode
+  val allClockGroupsNode: ClockGroupNode
   val ibus: InterruptBusWrapper
   val anyLocationMap = LocationMap.empty[Any]
 }
diff --git a/src/main/scala/subsystem/BaseSubsystem.scala b/src/main/scala/subsystem/BaseSubsystem.scala
index 926e1d22923..8d38ea64b57 100644
--- a/src/main/scala/subsystem/BaseSubsystem.scala
+++ b/src/main/scala/subsystem/BaseSubsystem.scala
@@ -2,6 +2,8 @@
 
 package freechips.rocketchip.subsystem
 
+import chisel3.Flipped
+import chisel3.experimental.IO
 import chisel3.util._
 import org.chipsalliance.cde.config.{Field, Parameters}
 import freechips.rocketchip.diplomacy._
@@ -9,8 +11,7 @@ import freechips.rocketchip.prci._
 import freechips.rocketchip.tilelink.TLBusWrapper
 import freechips.rocketchip.util._
 
-case object SubsystemDriveAsyncClockGroupsKey extends Field[Option[ClockGroupDriverParameters]](Some(ClockGroupDriverParameters(1)))
-case object AsyncClockGroupsKey extends Field[() => ClockGroupEphemeralNode](() => ClockGroupEphemeralNode()(ValName("clock_sources")))
+case object SubsystemDriveClockGroupsFromIO extends Field[Boolean](true)
 case class TLNetworkTopologyLocated(where: HierarchicalLocation) extends Field[Seq[CanInstantiateWithinContextThatHasTileLinkLocations with CanConnectWithinContextThatHasTileLinkLocations]]
 case class TLManagerViewpointLocated(where: HierarchicalLocation) extends Field[Location[TLBusWrapper]](SBUS)
 
@@ -26,7 +27,7 @@ abstract class BareSubsystem(implicit p: Parameters) extends LazyModule with Bin
   lazy val json = JSON(bindingTree)
 }
 
-abstract class BareSubsystemModuleImp[+L <: BareSubsystem](_outer: L) extends LazyModuleImp(_outer) {
+abstract class BareSubsystemModuleImp[+L <: BareSubsystem](_outer: L) extends LazyRawModuleImp(_outer) {
   val outer = _outer
   ElaborationArtefacts.add("graphml", outer.graphML)
   ElaborationArtefacts.add("dts", outer.dts)
@@ -47,11 +48,22 @@ case object SubsystemResetSchemeKey extends Field[SubsystemResetScheme](ResetSyn
   */
 trait HasConfigurablePRCILocations { this: HasPRCILocations =>
   val ibus = LazyModule(new InterruptBusWrapper)
-  implicit val asyncClockGroupsNode = p(AsyncClockGroupsKey)()
-  val clock_sources: ModuleValue[RecordMap[ClockBundle]] =
-    p(SubsystemDriveAsyncClockGroupsKey)
-      .map(_.drive(asyncClockGroupsNode))
-      .getOrElse(InModuleBody { RecordMap[ClockBundle]() })
+  val allClockGroupsNode = ClockGroupIdentityNode()
+  val io_clocks = if (p(SubsystemDriveClockGroupsFromIO)) {
+    val aggregator = ClockGroupAggregator()
+    val source = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+    allClockGroupsNode :*= aggregator := source
+    Some(InModuleBody {
+      val elements = source.out.map(_._1.member.elements).flatten
+      val io = IO(Flipped(RecordMap(elements.map { case (name, data) =>
+        name -> data.cloneType
+      }:_*)))
+      elements.foreach { case (name, data) => io(name).foreach { data := _ } }
+      io
+    })
+  } else {
+    None
+  }
 }
 
 /** Look up the topology configuration for the TL buses located within this layer of the hierarchy */
@@ -79,7 +91,7 @@ abstract class BaseSubsystem(val location: HierarchicalLocation = InSubsystem)
 
   // TODO must there really always be an "sbus"?
   val sbus = tlBusWrapperLocationMap(SBUS)
-  tlBusWrapperLocationMap.lift(SBUS).map { _.clockGroupNode := asyncClockGroupsNode }
+  tlBusWrapperLocationMap.lift(SBUS).map { _.clockGroupNode := allClockGroupsNode }
 
   // TODO: Preserve legacy implicit-clock behavior for IBUS for now. If binding
   // a PLIC to the CBUS, ensure it is synchronously coupled to the SBUS.
diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index 53edf0663b6..a040fb0dcfc 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -39,21 +39,22 @@ class Cluster(
   lazy val clusterId = thisClusterParams.clusterId
   lazy val location = InCluster(clusterId)
 
-  val clockGroupNode = ClockGroupAggregator()
+  lazy val allClockGroupsNode = ClockGroupIdentityNode()
+
   val csbus = tlBusWrapperLocationMap(SBUS) // like the sbus in the base subsystem
   val ccbus = tlBusWrapperLocationMap(CBUS) // like the cbus in the base subsystem
 
-  csbus.clockGroupNode := clockGroupNode
-  ccbus.clockGroupNode := clockGroupNode
+  csbus.clockGroupNode := allClockGroupsNode
+  ccbus.clockGroupNode := allClockGroupsNode
 
   val slaveNode = ccbus.inwardNode
   val masterNode = csbus.outwardNode
 
 
 
-  val ibus = LazyModule(new InterruptBusWrapper)
+  lazy val ibus = LazyModule(new InterruptBusWrapper)
   ibus.clockNode := csbus.fixedClockNode
-  implicit val asyncClockGroupsNode = p(AsyncClockGroupsKey)()
+
 
   lazy val msipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
   lazy val meipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
@@ -85,13 +86,11 @@ class ClusterPRCIDomain(
   lookup: LookupByClusterIdImpl)
   (implicit p: Parameters) extends ElementPRCIDomain[Cluster](clockSinkParams, crossingParams)
 {
-  val clockGroupNode = ClockGroupAggregator()
   val element = element_reset_domain {
     LazyModule(clusterParams.instantiate(crossingParams, lookup))
   }
-
-  clockNode := ClockGroup() := clockGroupNode
-  element.clockGroupNode := clockGroupNode
+  // Nothing should depend on the clocks coming from clockNode anyways
+  clockNode := element.csbus.fixedClockNode
 }
 
 
@@ -164,7 +163,7 @@ trait CanAttachCluster {
 
   def connectPRC(domain: ClusterPRCIDomain, context: ClusterContextType): Unit = {
     implicit val p = context.p
-    domain.clockGroupNode := context.asyncClockGroupsNode
+    domain.element.allClockGroupsNode :*= context.allClockGroupsNode
     domain {
       domain.element_reset_domain.clockNode := crossingParams.resetCrossingType.injectClockNode := domain.clockNode
     }
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 7804551f192..c1816f1d9df 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -689,7 +689,3 @@ class WithCloneRocketTiles(n: Int = 1, cloneHart: Int = 0, overrideIdOffset: Opt
   }
 })
 
-
-class WithNClockGroups(n: Int) extends Config((site, here, up) => {
-  case SubsystemDriveAsyncClockGroupsKey => up(SubsystemDriveAsyncClockGroupsKey).map(_.copy(num=n))
-})
diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/ElementPRCIDomain.scala
index f80eb29c0d1..71b1ded9ee7 100644
--- a/src/main/scala/subsystem/ElementPRCIDomain.scala
+++ b/src/main/scala/subsystem/ElementPRCIDomain.scala
@@ -46,9 +46,13 @@ abstract class ElementPRCIDomain[T <: BaseElement](
 {
   val element: T
   val element_reset_domain = LazyModule(new ElementResetDomain(clockSinkParams, crossingParams.resetCrossingType))
-  val tapClockNode = ClockIdentityNode()
-  val clockNode = FixedClockBroadcast(None) :=* tapClockNode
-  lazy val clockBundle = tapClockNode.in.head._1
+  def clockBundle: ClockBundle = {
+    val dummy = Wire(new ClockBundle) // nothing should every depend on this
+    dummy.clock := false.B.asClock
+    dummy.reset := false.B
+    dummy
+  }
+  val clockNode = FixedClockBroadcast()
 
   /** External code looking to connect and clock-cross the interrupts driven into this tile can call this. */
   def crossIntIn(crossingType: ClockCrossingType, tileNode: IntInwardNode): IntInwardNode = {
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
index 24525555014..d3493c1d1e6 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -110,7 +110,7 @@ trait HasElements extends DefaultElementContextType
 /** Provides some Chisel connectivity to certain tile IOs
   * This trait is intended for the root subsystem
   */
-trait HasElementsRootContextModuleImp extends LazyModuleImp {
+trait HasElementsRootContextModuleImp extends LazyRawModuleImp {
   val outer: InstantiatesElements with HasElements with HasElementsRootContext with HasTileInputConstants
 
   val reset_vector = outer.tileResetVectorIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"reset_vector_$i") }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 7c1a3c1d743..01907871e69 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -269,7 +269,7 @@ trait CanAttachTile {
       case _: RationalCrossing => domain.clockNode := tlBusToGetClockDriverFrom.clockNode
       case _: AsynchronousCrossing => {
         val tileClockGroup = ClockGroup()
-        tileClockGroup := context.asyncClockGroupsNode
+        tileClockGroup := context.allClockGroupsNode
         domain.clockNode := tileClockGroup
       }
     })
diff --git a/src/main/scala/subsystem/InterruptBus.scala b/src/main/scala/subsystem/InterruptBus.scala
index e50f82a9ad5..0d20c26a45a 100644
--- a/src/main/scala/subsystem/InterruptBus.scala
+++ b/src/main/scala/subsystem/InterruptBus.scala
@@ -70,7 +70,7 @@ trait HasExtInterruptsBundle {
 /** This trait performs the translation from a UInt IO into Diplomatic Interrupts.
   * The wiring must be done in the concrete LazyModuleImp. 
   */
-trait HasExtInterruptsModuleImp extends LazyModuleImp with HasExtInterruptsBundle {
+trait HasExtInterruptsModuleImp extends LazyRawModuleImp with HasExtInterruptsBundle {
   val outer: HasExtInterrupts
   val interrupts = IO(Input(UInt(outer.nExtInterrupts.W)))
 
diff --git a/src/main/scala/subsystem/RTC.scala b/src/main/scala/subsystem/RTC.scala
index 499633f679a..522d652037f 100644
--- a/src/main/scala/subsystem/RTC.scala
+++ b/src/main/scala/subsystem/RTC.scala
@@ -4,10 +4,10 @@ package freechips.rocketchip.subsystem
 
 import chisel3._
 import chisel3.util.Counter
-import freechips.rocketchip.diplomacy.{LazyModuleImp, DTSTimebase}
+import freechips.rocketchip.diplomacy.{LazyRawModuleImp, DTSTimebase}
 import freechips.rocketchip.devices.tilelink.CanHavePeripheryCLINT
 
-trait HasRTCModuleImp extends LazyModuleImp {
+trait HasRTCModuleImp extends LazyRawModuleImp {
   val outer: BaseSubsystem with CanHavePeripheryCLINT
   private val pbusFreq = outer.p(PeripheryBusKey).dtsFrequency.get
   private val rtcFreq = outer.p(DTSTimebase)
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index 7cb821e6341..043d1532650 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -38,9 +38,11 @@ class DualChannelConfig extends Config(new WithNMemoryChannels(2) ++ new Default
 class EightChannelConfig extends Config(new WithNMemoryChannels(8) ++ new DefaultConfig)
 
 class ClusterConfig extends Config(
+  new WithNBigCores(1, location=InCluster(3)) ++
   new WithNBigCores(2, location=InCluster(1)) ++
   new WithNBigCores(2, location=InCluster(0)) ++
-  new WithNClockGroups(3) ++ // 1 + nClusters TODO fix?
+  new WithCluster(3, location=InCluster(2)) ++
+  new WithCluster(2) ++
   new WithCluster(1) ++
   new WithCluster(0) ++
   new DefaultConfig
diff --git a/src/main/scala/system/TestHarness.scala b/src/main/scala/system/TestHarness.scala
index a1e37dea169..788716258f1 100644
--- a/src/main/scala/system/TestHarness.scala
+++ b/src/main/scala/system/TestHarness.scala
@@ -16,8 +16,10 @@ class TestHarness()(implicit p: Parameters) extends Module {
   val ldut = LazyModule(new ExampleRocketSystem)
   val dut = Module(ldut.module)
 
+  ldut.io_clocks.get.elements.values.foreach(_.clock := clock)
   // Allow the debug ndreset to reset the dut, but not until the initial reset has completed
-  dut.reset := (reset.asBool | ldut.debug.map { debug => AsyncResetReg(debug.ndreset) }.getOrElse(false.B)).asBool
+  val dut_reset = (reset.asBool | ldut.debug.map { debug => AsyncResetReg(debug.ndreset) }.getOrElse(false.B)).asBool
+  ldut.io_clocks.get.elements.values.foreach(_.reset := dut_reset)
 
   dut.dontTouchPorts()
   dut.tieOffInterrupts()
diff --git a/src/main/scala/tilelink/BusWrapper.scala b/src/main/scala/tilelink/BusWrapper.scala
index 657bba873f6..c57914e0eb0 100644
--- a/src/main/scala/tilelink/BusWrapper.scala
+++ b/src/main/scala/tilelink/BusWrapper.scala
@@ -89,15 +89,15 @@ abstract class TLBusWrapper(params: HasTLBusParams, val busName: String)(implici
   def coupleFrom[T](name: String)(gen: TLInwardNode => T): T =
     from(name) { gen(inwardNode :*=* TLNameNode("tl")) }
 
-  def crossToBus(bus: TLBusWrapper, xType: ClockCrossingType)(implicit asyncClockGroupNode: ClockGroupEphemeralNode): NoHandle = {
-    bus.clockGroupNode := asyncMux(xType, asyncClockGroupNode, this.clockGroupNode)
+  def crossToBus(bus: TLBusWrapper, xType: ClockCrossingType, allClockGroupNode: ClockGroupEphemeralNode): NoHandle = {
+    bus.clockGroupNode := asyncMux(xType, allClockGroupNode, this.clockGroupNode)
     coupleTo(s"bus_named_${bus.busName}") {
       bus.crossInHelper(xType) :*= TLWidthWidget(beatBytes) :*= _
     }
   }
 
-  def crossFromBus(bus: TLBusWrapper, xType: ClockCrossingType)(implicit asyncClockGroupNode: ClockGroupEphemeralNode): NoHandle = {
-    bus.clockGroupNode := asyncMux(xType, asyncClockGroupNode, this.clockGroupNode)
+  def crossFromBus(bus: TLBusWrapper, xType: ClockCrossingType, allClockGroupNode: ClockGroupEphemeralNode): NoHandle = {
+    bus.clockGroupNode := asyncMux(xType, allClockGroupNode, this.clockGroupNode)
     coupleFrom(s"bus_named_${bus.busName}") {
       _ :=* TLWidthWidget(bus.beatBytes) :=* bus.crossOutHelper(xType)
     }
@@ -178,8 +178,8 @@ class TLBusWrapperConnection
     val masterTLBus = context.locateTLBusWrapper(master)
     val slaveTLBus  = context.locateTLBusWrapper(slave)
     def bindClocks(implicit p: Parameters) = driveClockFromMaster match {
-      case Some(true)  => slaveTLBus.clockGroupNode  := asyncMux(xType, context.asyncClockGroupsNode, masterTLBus.clockGroupNode)
-      case Some(false) => masterTLBus.clockGroupNode := asyncMux(xType, context.asyncClockGroupsNode, slaveTLBus.clockGroupNode)
+      case Some(true)  => slaveTLBus.clockGroupNode  := asyncMux(xType, context.allClockGroupsNode, masterTLBus.clockGroupNode)
+      case Some(false) => masterTLBus.clockGroupNode := asyncMux(xType, context.allClockGroupsNode, slaveTLBus.clockGroupNode)
       case None =>
     }
     def bindTLNodes(implicit p: Parameters) = nodeBinding match {

From e8630420c036e3f0cf0161c1fc40e0fad19dc81f Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 30 Jan 2023 20:41:28 -0800
Subject: [PATCH 28/47] Fix no SEIP configs

---
 src/main/scala/subsystem/HasElements.scala | 7 ++++---
 src/main/scala/system/Configs.scala        | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
index d3493c1d1e6..d07fce53cac 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -185,9 +185,10 @@ trait HasElementsRootContext
     sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
     outputRequiresInput = false,
     inputRequiresOutput = false))
-  val seipNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
-    (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(seipIONode.get))
-  }.toMap
+  val seipNodes: Map[Int, IntNode] = totalTiles.filter(_.tileParams.core.hasSupervisorMode)
+    .map(_.tileId).map { i =>
+      (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(seipIONode.get))
+    }.toMap
 
   val tileToPlicNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
     plicOpt.map(o => (i, o.intnode :=* IntEphemeralNode()))
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index 043d1532650..68136f571c9 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -38,7 +38,7 @@ class DualChannelConfig extends Config(new WithNMemoryChannels(2) ++ new Default
 class EightChannelConfig extends Config(new WithNMemoryChannels(8) ++ new DefaultConfig)
 
 class ClusterConfig extends Config(
-  new WithNBigCores(1, location=InCluster(3)) ++
+  new WithNBigCores(2, location=InCluster(3)) ++
   new WithNBigCores(2, location=InCluster(1)) ++
   new WithNBigCores(2, location=InCluster(0)) ++
   new WithCluster(3, location=InCluster(2)) ++

From 85709cc5dbb1c691f02e307de3a462cc8d46440b Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 31 Jan 2023 11:15:51 -0800
Subject: [PATCH 29/47] Fix ElementPRCIDomain missing its implicit clock/reset

---
 src/main/scala/subsystem/ElementPRCIDomain.scala | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/ElementPRCIDomain.scala
index 71b1ded9ee7..b030ab21ba6 100644
--- a/src/main/scala/subsystem/ElementPRCIDomain.scala
+++ b/src/main/scala/subsystem/ElementPRCIDomain.scala
@@ -46,13 +46,9 @@ abstract class ElementPRCIDomain[T <: BaseElement](
 {
   val element: T
   val element_reset_domain = LazyModule(new ElementResetDomain(clockSinkParams, crossingParams.resetCrossingType))
-  def clockBundle: ClockBundle = {
-    val dummy = Wire(new ClockBundle) // nothing should every depend on this
-    dummy.clock := false.B.asClock
-    dummy.reset := false.B
-    dummy
-  }
-  val clockNode = FixedClockBroadcast()
+  val tapClockNode = ClockIdentityNode()
+  val clockNode = FixedClockBroadcast() :=* tapClockNode
+  lazy val clockBundle = tapClockNode.in.head._1
 
   /** External code looking to connect and clock-cross the interrupts driven into this tile can call this. */
   def crossIntIn(crossingType: ClockCrossingType, tileNode: IntInwardNode): IntInwardNode = {

From 9f3698386038c2235b9f167ed0e9b9093cf23929 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 31 Jan 2023 11:23:56 -0800
Subject: [PATCH 30/47] Move BankedL2Params.scala to
 BankedCoherenceParams.scala

---
 .../{BankedL2Params.scala => BankedCoherenceParams.scala}         | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/scala/subsystem/{BankedL2Params.scala => BankedCoherenceParams.scala} (100%)

diff --git a/src/main/scala/subsystem/BankedL2Params.scala b/src/main/scala/subsystem/BankedCoherenceParams.scala
similarity index 100%
rename from src/main/scala/subsystem/BankedL2Params.scala
rename to src/main/scala/subsystem/BankedCoherenceParams.scala

From f478b45f29e68693e189d77cf66db00911840d92 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 31 Jan 2023 11:27:45 -0800
Subject: [PATCH 31/47] Rename BankedL2Key to BankedCoherenceKey

---
 src/main/scala/subsystem/BankedCoherenceParams.scala | 6 +++---
 src/main/scala/subsystem/BusTopology.scala           | 2 +-
 src/main/scala/subsystem/Configs.scala               | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/main/scala/subsystem/BankedCoherenceParams.scala b/src/main/scala/subsystem/BankedCoherenceParams.scala
index 402dd537804..34b5f876d4a 100644
--- a/src/main/scala/subsystem/BankedCoherenceParams.scala
+++ b/src/main/scala/subsystem/BankedCoherenceParams.scala
@@ -23,10 +23,10 @@ case class BroadcastParams(
   controlAddress: Option[BigInt] = None,
   filterFactory:  TLBroadcast.ProbeFilterFactory = BroadcastFilter.factory)
 
-/** L2 memory subsystem configuration */
-case object BankedL2Key extends Field(BankedL2Params())
+/** Coherence manager configuration */
+case object BankedCoherenceKey extends Field(BankedCoherenceParams())
 
-case class BankedL2Params(
+case class BankedCoherenceParams(
   nBanks: Int = 1,
   coherenceManager: CoherenceManagerInstantiationFn = broadcastManager
 ) {
diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index c3e4691d359..2d162d21fa9 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -91,7 +91,7 @@ case class HierarchicalBusTopologyParams(
 case class CoherentBusTopologyParams(
   sbus: SystemBusParams, // TODO remove this after better width propagation
   mbus: MemoryBusParams,
-  l2: BankedL2Params,
+  l2: BankedCoherenceParams,
   sbusToMbusXType: ClockCrossingType = NoCrossing,
   driveMBusClockFromSBus: Boolean = true
 ) extends TLBusWrapperTopology(
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index c1816f1d9df..0e40d337a14 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -84,7 +84,7 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
     CoherentBusTopologyParams(
       sbus = site(SystemBusKey),
       mbus = site(MemoryBusKey),
-      l2 = site(BankedL2Key),
+      l2 = site(BankedCoherenceKey),
       sbusToMbusXType = site(SbusToMbusXTypeKey),
       driveMBusClockFromSBus = site(DriveClocksFromSBus)))
 })
@@ -241,7 +241,7 @@ class WithCluster(
 })
 
 class WithNBanks(n: Int) extends Config((site, here, up) => {
-  case BankedL2Key => up(BankedL2Key, site).copy(nBanks = n)
+  case BankedCoherenceKey => up(BankedCoherenceKey, site).copy(nBanks = n)
 })
 
 class WithNTrackersPerBank(n: Int) extends Config((site, here, up) => {
@@ -321,7 +321,7 @@ class WithIncoherentTiles extends Config((site, here, up) => {
       }))
     case t => t
   }
-  case BankedL2Key => up(BankedL2Key, site).copy(
+  case BankedCoherenceKey => up(BankedCoherenceKey, site).copy(
     coherenceManager = CoherenceManagerWrapper.incoherentManager
   )
 })

From 5b29fe97145bc35d602104cea109e67b4c247bf3 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 31 Jan 2023 11:31:53 -0800
Subject: [PATCH 32/47] Rename L2 to LLC

---
 .../scala/subsystem/BankedCoherenceParams.scala    |  2 +-
 src/main/scala/subsystem/BusTopology.scala         | 14 +++++++-------
 src/main/scala/subsystem/Configs.scala             |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/main/scala/subsystem/BankedCoherenceParams.scala b/src/main/scala/subsystem/BankedCoherenceParams.scala
index 34b5f876d4a..dff2c41d4da 100644
--- a/src/main/scala/subsystem/BankedCoherenceParams.scala
+++ b/src/main/scala/subsystem/BankedCoherenceParams.scala
@@ -14,7 +14,7 @@ import CoherenceManagerWrapper._
 /** Global cache coherence granularity, which applies to all caches, for now. */
 case object CacheBlockBytes extends Field[Int](64)
 
-/** L2 Broadcast Hub configuration */
+/** LLC Broadcast Hub configuration */
 case object BroadcastKey extends Field(BroadcastParams())
 
 case class BroadcastParams(
diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index 2d162d21fa9..817e82d8a45 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -28,7 +28,7 @@ case object PBUS extends TLBusWrapperLocation("subsystem_pbus")
 case object FBUS extends TLBusWrapperLocation("subsystem_fbus")
 case object MBUS extends TLBusWrapperLocation("subsystem_mbus")
 case object CBUS extends TLBusWrapperLocation("subsystem_cbus")
-case object L2   extends TLBusWrapperLocation("subsystem_l2")
+case object LLC  extends TLBusWrapperLocation("subsystem_llc")
 
 /** Parameterizes the subsystem in terms of optional clock-crossings
   *   that are insertable between some of the five traditional tilelink bus wrappers.
@@ -91,16 +91,16 @@ case class HierarchicalBusTopologyParams(
 case class CoherentBusTopologyParams(
   sbus: SystemBusParams, // TODO remove this after better width propagation
   mbus: MemoryBusParams,
-  l2: BankedCoherenceParams,
+  coherence: BankedCoherenceParams,
   sbusToMbusXType: ClockCrossingType = NoCrossing,
   driveMBusClockFromSBus: Boolean = true
 ) extends TLBusWrapperTopology(
-  instantiations = (if (l2.nBanks == 0) Nil else List(
+  instantiations = (if (coherence.nBanks == 0) Nil else List(
     (MBUS, mbus),
-    (L2, CoherenceManagerWrapperParams(mbus.blockBytes, mbus.beatBytes, l2.nBanks, L2.name, sbus.dtsFrequency)(l2.coherenceManager)))),
-  connections = if (l2.nBanks == 0) Nil else List(
-    (SBUS, L2,   TLBusWrapperConnection(driveClockFromMaster = Some(true), nodeBinding = BIND_STAR)()),
-    (L2,  MBUS,  TLBusWrapperConnection.crossTo(
+    (LLC, CoherenceManagerWrapperParams(mbus.blockBytes, mbus.beatBytes, coherence.nBanks, LLC.name, sbus.dtsFrequency)(coherence.coherenceManager)))),
+  connections = if (coherence.nBanks == 0) Nil else List(
+    (SBUS, LLC,   TLBusWrapperConnection(driveClockFromMaster = Some(true), nodeBinding = BIND_STAR)()),
+    (LLC,  MBUS,  TLBusWrapperConnection.crossTo(
       xType = sbusToMbusXType,
       driveClockFromMaster = if (driveMBusClockFromSBus) Some(true) else None,
       nodeBinding = BIND_QUERY))
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 0e40d337a14..83cb0a937c0 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -84,7 +84,7 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
     CoherentBusTopologyParams(
       sbus = site(SystemBusKey),
       mbus = site(MemoryBusKey),
-      l2 = site(BankedCoherenceKey),
+      coherence = site(BankedCoherenceKey),
       sbusToMbusXType = site(SbusToMbusXTypeKey),
       driveMBusClockFromSBus = site(DriveClocksFromSBus)))
 })

From 686d57a3c90aae44c5620f59c388834cba22afa7 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 31 Jan 2023 11:56:39 -0800
Subject: [PATCH 33/47] Rename LLC to COH

---
 src/main/scala/subsystem/BusTopology.scala | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index 817e82d8a45..ec1ee900cbb 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -28,7 +28,7 @@ case object PBUS extends TLBusWrapperLocation("subsystem_pbus")
 case object FBUS extends TLBusWrapperLocation("subsystem_fbus")
 case object MBUS extends TLBusWrapperLocation("subsystem_mbus")
 case object CBUS extends TLBusWrapperLocation("subsystem_cbus")
-case object LLC  extends TLBusWrapperLocation("subsystem_llc")
+case object COH  extends TLBusWrapperLocation("subsystem_coh")
 
 /** Parameterizes the subsystem in terms of optional clock-crossings
   *   that are insertable between some of the five traditional tilelink bus wrappers.
@@ -97,10 +97,10 @@ case class CoherentBusTopologyParams(
 ) extends TLBusWrapperTopology(
   instantiations = (if (coherence.nBanks == 0) Nil else List(
     (MBUS, mbus),
-    (LLC, CoherenceManagerWrapperParams(mbus.blockBytes, mbus.beatBytes, coherence.nBanks, LLC.name, sbus.dtsFrequency)(coherence.coherenceManager)))),
+    (COH, CoherenceManagerWrapperParams(mbus.blockBytes, mbus.beatBytes, coherence.nBanks, COH.name, sbus.dtsFrequency)(coherence.coherenceManager)))),
   connections = if (coherence.nBanks == 0) Nil else List(
-    (SBUS, LLC,   TLBusWrapperConnection(driveClockFromMaster = Some(true), nodeBinding = BIND_STAR)()),
-    (LLC,  MBUS,  TLBusWrapperConnection.crossTo(
+    (SBUS, COH,   TLBusWrapperConnection(driveClockFromMaster = Some(true), nodeBinding = BIND_STAR)()),
+    (COH,  MBUS,  TLBusWrapperConnection.crossTo(
       xType = sbusToMbusXType,
       driveClockFromMaster = if (driveMBusClockFromSBus) Some(true) else None,
       nodeBinding = BIND_QUERY))

From 35a6e20f738b3a523826c2d09e1d5afd41f040b1 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 31 Jan 2023 13:33:26 -0800
Subject: [PATCH 34/47] Add busContextName

---
 src/main/scala/subsystem/Attachable.scala    |  1 +
 src/main/scala/subsystem/BaseSubsystem.scala |  2 ++
 src/main/scala/subsystem/BusTopology.scala   | 12 ++++++------
 src/main/scala/subsystem/Cluster.scala       |  1 +
 src/main/scala/tilelink/BusWrapper.scala     |  8 ++++----
 5 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/main/scala/subsystem/Attachable.scala b/src/main/scala/subsystem/Attachable.scala
index 6a374b4db2d..06f2269421f 100644
--- a/src/main/scala/subsystem/Attachable.scala
+++ b/src/main/scala/subsystem/Attachable.scala
@@ -30,6 +30,7 @@ trait HasPRCILocations extends LazyScopeWithParameters { this: LazyModule =>
 
 /** Layers of hierarchy with this trait contain attachment points for TileLink interfaces */
 trait HasTileLinkLocations extends HasPRCILocations { this: LazyModule =>
+  val busContextName: String
   val tlBusWrapperLocationMap = LocationMap.empty[TLBusWrapper]
   def locateTLBusWrapper(location: Location[TLBusWrapper]): TLBusWrapper = locateTLBusWrapper(location.name)
   def locateTLBusWrapper(name: String): TLBusWrapper = tlBusWrapperLocationMap(Location[TLBusWrapper](name))
diff --git a/src/main/scala/subsystem/BaseSubsystem.scala b/src/main/scala/subsystem/BaseSubsystem.scala
index 8d38ea64b57..93070ad3bf6 100644
--- a/src/main/scala/subsystem/BaseSubsystem.scala
+++ b/src/main/scala/subsystem/BaseSubsystem.scala
@@ -89,6 +89,8 @@ abstract class BaseSubsystem(val location: HierarchicalLocation = InSubsystem)
 {
   override val module: BaseSubsystemModuleImp[BaseSubsystem]
 
+  val busContextName = "subsystem"
+
   // TODO must there really always be an "sbus"?
   val sbus = tlBusWrapperLocationMap(SBUS)
   tlBusWrapperLocationMap.lift(SBUS).map { _.clockGroupNode := allClockGroupsNode }
diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index ec1ee900cbb..e0a5cedba22 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -23,12 +23,12 @@ case object MemoryBusKey extends Field[MemoryBusParams]
 //   dynamically-configured topologies.
 
 class TLBusWrapperLocation(name: String) extends Location[TLBusWrapper](name)
-case object SBUS extends TLBusWrapperLocation("subsystem_sbus")
-case object PBUS extends TLBusWrapperLocation("subsystem_pbus")
-case object FBUS extends TLBusWrapperLocation("subsystem_fbus")
-case object MBUS extends TLBusWrapperLocation("subsystem_mbus")
-case object CBUS extends TLBusWrapperLocation("subsystem_cbus")
-case object COH  extends TLBusWrapperLocation("subsystem_coh")
+case object SBUS extends TLBusWrapperLocation("sbus")
+case object PBUS extends TLBusWrapperLocation("pbus")
+case object FBUS extends TLBusWrapperLocation("fbus")
+case object MBUS extends TLBusWrapperLocation("mbus")
+case object CBUS extends TLBusWrapperLocation("cbus")
+case object COH  extends TLBusWrapperLocation("coh")
 
 /** Parameterizes the subsystem in terms of optional clock-crossings
   *   that are insertable between some of the five traditional tilelink bus wrappers.
diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index a040fb0dcfc..db7429cb729 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -36,6 +36,7 @@ class Cluster(
     with InstantiatesElements
     with HasElements
 {
+  val busContextName = thisClusterParams.baseName
   lazy val clusterId = thisClusterParams.clusterId
   lazy val location = InCluster(clusterId)
 
diff --git a/src/main/scala/tilelink/BusWrapper.scala b/src/main/scala/tilelink/BusWrapper.scala
index c57914e0eb0..c1888e4e36d 100644
--- a/src/main/scala/tilelink/BusWrapper.scala
+++ b/src/main/scala/tilelink/BusWrapper.scala
@@ -240,8 +240,8 @@ case class AddressAdjusterWrapperParams(
 {
   val dtsFrequency = None
   def instantiate(context: HasTileLinkLocations, loc: Location[TLBusWrapper])(implicit p: Parameters): AddressAdjusterWrapper = {
-    val aaWrapper = LazyModule(new AddressAdjusterWrapper(this, loc.name))
-    aaWrapper.suggestName(loc.name + "_wrapper")
+    val aaWrapper = LazyModule(new AddressAdjusterWrapper(this, context.busContextName + "_" + loc.name))
+    aaWrapper.suggestName(context.busContextName + "_" + loc.name + "_wrapper")
     context.tlBusWrapperLocationMap += (loc -> aaWrapper)
     aaWrapper
   }
@@ -270,8 +270,8 @@ case class TLJBarWrapperParams(
 {
   val dtsFrequency = None
   def instantiate(context: HasTileLinkLocations, loc: Location[TLBusWrapper])(implicit p: Parameters): TLJBarWrapper = {
-    val jbarWrapper = LazyModule(new TLJBarWrapper(this, loc.name))
-    jbarWrapper.suggestName(loc.name + "_wrapper")
+    val jbarWrapper = LazyModule(new TLJBarWrapper(this, context.busContextName + "_" + loc.name))
+    jbarWrapper.suggestName(context.busContextName + "_" + loc.name + "_wrapper")
     context.tlBusWrapperLocationMap += (loc -> jbarWrapper)
     jbarWrapper
   }

From 51bfd75c37f7d7b2a4435dcb70145f15dc73a2ab Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 31 Jan 2023 14:18:14 -0800
Subject: [PATCH 35/47] Remove old sbus arg to CoherentBusTopoParams

---
 src/main/scala/subsystem/BusTopology.scala | 4 ++--
 src/main/scala/subsystem/Configs.scala     | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index e0a5cedba22..d15bd49460d 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -89,7 +89,6 @@ case class HierarchicalBusTopologyParams(
 
 /** Parameterization of a topology containing a banked coherence manager and a bus for attaching memory devices. */
 case class CoherentBusTopologyParams(
-  sbus: SystemBusParams, // TODO remove this after better width propagation
   mbus: MemoryBusParams,
   coherence: BankedCoherenceParams,
   sbusToMbusXType: ClockCrossingType = NoCrossing,
@@ -97,7 +96,7 @@ case class CoherentBusTopologyParams(
 ) extends TLBusWrapperTopology(
   instantiations = (if (coherence.nBanks == 0) Nil else List(
     (MBUS, mbus),
-    (COH, CoherenceManagerWrapperParams(mbus.blockBytes, mbus.beatBytes, coherence.nBanks, COH.name, sbus.dtsFrequency)(coherence.coherenceManager)))),
+    (COH, CoherenceManagerWrapperParams(mbus.blockBytes, mbus.beatBytes, coherence.nBanks, COH.name)(coherence.coherenceManager)))),
   connections = if (coherence.nBanks == 0) Nil else List(
     (SBUS, COH,   TLBusWrapperConnection(driveClockFromMaster = Some(true), nodeBinding = BIND_STAR)()),
     (COH,  MBUS,  TLBusWrapperConnection.crossTo(
@@ -117,3 +116,4 @@ case class ClusterBusTopologyParams(
     (CBUS, ccbus)),
   connections = Nil
 )
+
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 83cb0a937c0..0f72c5e7c19 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -82,7 +82,6 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
         fbusToSbusXType = site(FbusToSbusXTypeKey)),
       driveClocksFromSBus = site(DriveClocksFromSBus)),
     CoherentBusTopologyParams(
-      sbus = site(SystemBusKey),
       mbus = site(MemoryBusKey),
       coherence = site(BankedCoherenceKey),
       sbusToMbusXType = site(SbusToMbusXTypeKey),

From 0a0014675eb6e3de83a10796bc9f7ebfeea81487 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 31 Jan 2023 19:24:42 -0800
Subject: [PATCH 36/47] Support CloneCluster

---
 .../groundtest/GroundTestSubsystem.scala      |   4 +-
 .../subsystem/BankedCoherenceParams.scala     |   3 +-
 src/main/scala/subsystem/BusTopology.scala    |  15 ++-
 src/main/scala/subsystem/Cluster.scala        |  46 +++++---
 src/main/scala/subsystem/Configs.scala        | 101 +++++++++++-------
 src/main/scala/subsystem/HasElements.scala    |  84 +++++++--------
 src/main/scala/subsystem/HasTiles.scala       |  19 ++--
 .../scala/subsystem/RocketSubsystem.scala     |   2 +-
 8 files changed, 166 insertions(+), 108 deletions(-)

diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index 232c69204be..d435acce080 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -30,7 +30,7 @@ class GroundTestSubsystem(implicit p: Parameters)
   // No PLIC in ground test; so just sink the interrupts to nowhere
   IntSinkNode(IntSinkPortSimple()) :=* ibus.toPLIC
 
-  val tileStatusNodes = totalTiles.collect { case t: GroundTestTile => t.statusNode.makeSink() }
+  val tileStatusNodes = totalTiles.values.collect { case t: GroundTestTile => t.statusNode.makeSink() }
   val clintOpt = None
   val debugOpt = None
   val plicOpt = None
@@ -40,6 +40,6 @@ class GroundTestSubsystem(implicit p: Parameters)
 
 class GroundTestSubsystemModuleImp[+L <: GroundTestSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer) {
   val success = IO(Output(Bool()))
-  val status = dontTouch(DebugCombiner(outer.tileStatusNodes.map(_.bundle)))
+  val status = dontTouch(DebugCombiner(outer.tileStatusNodes.map(_.bundle).toSeq))
   success := outer.tileCeaseSinkNode.in.head._1.asUInt.andR
 }
diff --git a/src/main/scala/subsystem/BankedCoherenceParams.scala b/src/main/scala/subsystem/BankedCoherenceParams.scala
index dff2c41d4da..84a7d015c37 100644
--- a/src/main/scala/subsystem/BankedCoherenceParams.scala
+++ b/src/main/scala/subsystem/BankedCoherenceParams.scala
@@ -24,7 +24,8 @@ case class BroadcastParams(
   filterFactory:  TLBroadcast.ProbeFilterFactory = BroadcastFilter.factory)
 
 /** Coherence manager configuration */
-case object BankedCoherenceKey extends Field(BankedCoherenceParams())
+case object SubsystemBankedCoherenceKey extends Field(BankedCoherenceParams())
+case class ClusterBankedCoherenceKey(clusterId: Int) extends Field(BankedCoherenceParams(nBanks=0))
 
 case class BankedCoherenceParams(
   nBanks: Int = 1,
diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index d15bd49460d..dc8443ae220 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -109,11 +109,20 @@ case class CoherentBusTopologyParams(
 case class ClusterBusTopologyParams(
   clusterId: Int,
   csbus: SystemBusParams,
-  ccbus: PeripheryBusParams
+  ccbus: PeripheryBusParams,
+  coherence: BankedCoherenceParams
 ) extends TLBusWrapperTopology(
   instantiations = List(
     (SBUS, csbus),
-    (CBUS, ccbus)),
-  connections = Nil
+    (CBUS, ccbus)) ++ (if (coherence.nBanks == 0) Nil else List(
+    (MBUS, csbus),
+    (COH , CoherenceManagerWrapperParams(csbus.blockBytes, csbus.beatBytes, coherence.nBanks, COH.name)(coherence.coherenceManager)))),
+  connections = if (coherence.nBanks == 0) Nil else List(
+    (SBUS, COH , TLBusWrapperConnection(driveClockFromMaster = Some(true), nodeBinding = BIND_STAR)()),
+    (COH , MBUS, TLBusWrapperConnection.crossTo(
+      xType = NoCrossing,
+      driveClockFromMaster = Some(true),
+      nodeBinding = BIND_QUERY))
+  )
 )
 
diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index db7429cb729..8febbe58ee7 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -13,6 +13,7 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.devices.debug.{TLDebugModule}
 import freechips.rocketchip.devices.tilelink._
 import freechips.rocketchip.util._
+import scala.collection.immutable.ListMap
 
 case class ClustersLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachCluster]](Nil)
 
@@ -44,29 +45,30 @@ class Cluster(
 
   val csbus = tlBusWrapperLocationMap(SBUS) // like the sbus in the base subsystem
   val ccbus = tlBusWrapperLocationMap(CBUS) // like the cbus in the base subsystem
+  val cmbus = tlBusWrapperLocationMap.lift(MBUS).getOrElse(csbus)
 
   csbus.clockGroupNode := allClockGroupsNode
   ccbus.clockGroupNode := allClockGroupsNode
 
   val slaveNode = ccbus.inwardNode
-  val masterNode = csbus.outwardNode
+  val masterNode = cmbus.outwardNode
 
 
 
   lazy val ibus = LazyModule(new InterruptBusWrapper)
   ibus.clockNode := csbus.fixedClockNode
 
-
-  lazy val msipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
-  lazy val meipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
-  lazy val seipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
-  lazy val tileToPlicNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.toMap
-  lazy val debugNodes = totalTileIdList.map { i => (i, IntSyncIdentityNode()) }.toMap
-  lazy val nmiNodes = totalTiles.filter(_.tileParams.core.useNMI).map { t => (t.tileId, BundleBridgeIdentityNode[NMI]()) }.toMap
-  lazy val tileHartIdNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.toMap
-  lazy val tileResetVectorNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.toMap
-  lazy val traceCoreNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceCoreInterface]()) }.toMap
-  lazy val traceNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceBundle]()) }.toMap
+  lazy val msipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(ListMap)
+  lazy val meipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(ListMap)
+  lazy val seipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(ListMap)
+  lazy val tileToPlicNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(ListMap)
+  lazy val debugNodes = totalTileIdList.map { i => (i, IntSyncIdentityNode()) }.to(ListMap)
+  lazy val nmiNodes = totalTiles.filter { case (i,t) => t.tileParams.core.useNMI }
+    .mapValues(_ => BundleBridgeIdentityNode[NMI]()).to(ListMap)
+  lazy val tileHartIdNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.to(ListMap)
+  lazy val tileResetVectorNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.to(ListMap)
+  lazy val traceCoreNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceCoreInterface]()) }.to(ListMap)
+  lazy val traceNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceBundle]()) }.to(ListMap)
 
   // TODO fix: shouldn't need to connect dummy notifications
   tileHaltXbarNode := NullIntSource()
@@ -101,7 +103,7 @@ trait CanAttachCluster {
   def clusterParams: ClusterParams
   def crossingParams: ElementCrossingParamsLike
 
-  def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: Seq[ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
+  def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: ListMap[Int, ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
     val clockSinkParams = clusterParams.clockSinkParams.copy(name = Some(clusterParams.uniqueName))
     val cluster_prci_domain = LazyModule(new ClusterPRCIDomain(
       clockSinkParams, crossingParams, clusterParams, PriorityMuxClusterIdFromSeq(allClusterParams)))
@@ -208,3 +210,21 @@ case class ClusterAttachParams(
   clusterParams: ClusterParams,
   crossingParams: ElementCrossingParamsLike
 ) extends CanAttachCluster
+
+case class CloneClusterAttachParams(
+  sourceClusterId: Int,
+  cloneParams: CanAttachCluster
+) extends CanAttachCluster {
+  def clusterParams = cloneParams.clusterParams
+  def crossingParams = cloneParams.crossingParams
+
+  override def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: ListMap[Int, ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
+    require(instantiatedClusters.contains(sourceClusterId))
+    val clockSinkParams = clusterParams.clockSinkParams.copy(name = Some(clusterParams.uniqueName))
+    val cluster_prci_domain = CloneLazyModule(
+      new ClusterPRCIDomain(clockSinkParams, crossingParams, clusterParams, PriorityMuxClusterIdFromSeq(allClusterParams)),
+      instantiatedClusters(sourceClusterId)
+    )
+    cluster_prci_domain
+  }
+}
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 0f72c5e7c19..1f4c47aa7ba 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -83,7 +83,7 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
       driveClocksFromSBus = site(DriveClocksFromSBus)),
     CoherentBusTopologyParams(
       mbus = site(MemoryBusKey),
-      coherence = site(BankedCoherenceKey),
+      coherence = site(SubsystemBankedCoherenceKey),
       sbusToMbusXType = site(SbusToMbusXTypeKey),
       driveMBusClockFromSBus = site(DriveClocksFromSBus)))
 })
@@ -155,8 +155,8 @@ class WithNSmallCores(
   crossing: RocketCrossingParams = RocketCrossingParams(),
   location: HierarchicalLocation = InSubsystem
 ) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => {
-    val prev = up(TilesLocated(InSubsystem), site)
+  case TilesLocated(`location`) => {
+    val prev = up(TilesLocated(location), site)
     val idOffset = up(NumTiles)
     val small = RocketTileParams(
       core = RocketCoreParams(useVM = false, fpu = None),
@@ -233,14 +233,19 @@ class WithCluster(
     ClusterBusTopologyParams(
       clusterId = clusterId,
       csbus = site(SystemBusKey),
-      ccbus = site(ControlBusKey).copy(errorDevice = None)
+      ccbus = site(ControlBusKey).copy(errorDevice = None),
+      coherence = site(ClusterBankedCoherenceKey(clusterId))
     )
   )
   case PossibleTileLocations => up(PossibleTileLocations) :+ InCluster(clusterId)
 })
 
+class WithClusterBanks(clusterId: Int, nBanks: Int = 1) extends Config((site, here, up) => {
+  case ClusterBankedCoherenceKey(`clusterId`) => up(ClusterBankedCoherenceKey(clusterId)).copy(nBanks=nBanks)
+})
+
 class WithNBanks(n: Int) extends Config((site, here, up) => {
-  case BankedCoherenceKey => up(BankedCoherenceKey, site).copy(nBanks = n)
+  case SubsystemBankedCoherenceKey => up(SubsystemBankedCoherenceKey, site).copy(nBanks = n)
 })
 
 class WithNTrackersPerBank(n: Int) extends Config((site, here, up) => {
@@ -249,7 +254,7 @@ class WithNTrackersPerBank(n: Int) extends Config((site, here, up) => {
 
 // This is the number of icache sets for all Rocket tiles
 class WithL1ICacheSets(sets: Int) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       icache = tp.tileParams.icache.map(_.copy(nSets = sets))))
     case t => t
@@ -258,7 +263,7 @@ class WithL1ICacheSets(sets: Int) extends Config((site, here, up) => {
 
 // This is the number of icache sets for all Rocket tiles
 class WithL1DCacheSets(sets: Int) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       dcache = tp.tileParams.dcache.map(_.copy(nSets = sets))))
     case t => t
@@ -266,7 +271,7 @@ class WithL1DCacheSets(sets: Int) extends Config((site, here, up) => {
 })
 
 class WithL1ICacheWays(ways: Int) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       icache = tp.tileParams.icache.map(_.copy(nWays = ways))))
     case t => t
@@ -274,7 +279,7 @@ class WithL1ICacheWays(ways: Int) extends Config((site, here, up) => {
 })
 
 class WithL1DCacheWays(ways: Int) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       dcache = tp.tileParams.dcache.map(_.copy(nWays = ways))))
     case t => t
@@ -312,7 +317,7 @@ class WithBufferlessBroadcastHub extends Config((site, here, up) => {
  * DO NOT use this configuration.
  */
 class WithIncoherentTiles extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
       master = tp.crossingParams.master match {
         case x: ElementMasterPortParams => x.copy(cork = Some(true))
@@ -320,14 +325,14 @@ class WithIncoherentTiles extends Config((site, here, up) => {
       }))
     case t => t
   }
-  case BankedCoherenceKey => up(BankedCoherenceKey, site).copy(
+  case SubsystemBankedCoherenceKey => up(SubsystemBankedCoherenceKey, site).copy(
     coherenceManager = CoherenceManagerWrapper.incoherentManager
   )
 })
 
 class WithRV32 extends Config((site, here, up) => {
   case XLen => 32
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(
         fpu = tp.tileParams.core.fpu.map(_.copy(fLen = 32)),
@@ -337,7 +342,7 @@ class WithRV32 extends Config((site, here, up) => {
 })
 
 class WithFP16 extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(
         fpu = tp.tileParams.core.fpu.map(_.copy(minFLen = 16))
@@ -348,7 +353,7 @@ class WithFP16 extends Config((site, here, up) => {
 })
 
 class WithNonblockingL1(nMSHRs: Int) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       dcache = tp.tileParams.dcache.map(_.copy(nMSHRs = nMSHRs))))
     case t => t
@@ -356,7 +361,7 @@ class WithNonblockingL1(nMSHRs: Int) extends Config((site, here, up) => {
 })
 
 class WithNBreakpoints(hwbp: Int) extends Config ((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(nBreakpoints = hwbp)))
     case t => t
@@ -364,7 +369,7 @@ class WithNBreakpoints(hwbp: Int) extends Config ((site, here, up) => {
 })
 
 class WithHypervisor(hext: Boolean = true) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(useHypervisor = hext)))
     case t => t
@@ -392,7 +397,7 @@ class WithRoccExample extends Config((site, here, up) => {
 })
 
 class WithDefaultBtb extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       btb = Some(BTBParams())))
     case t => t
@@ -400,7 +405,7 @@ class WithDefaultBtb extends Config((site, here, up) => {
 })
 
 class WithFastMulDiv extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(mulDiv = Some(
         MulDivParams(mulUnroll = 8, mulEarlyOut = (site(XLen) > 32), divEarlyOut = true)))))
@@ -409,7 +414,7 @@ class WithFastMulDiv extends Config((site, here, up) => {
 })
 
 class WithoutMulDiv extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(mulDiv = None)))
     case t => t
@@ -417,7 +422,7 @@ class WithoutMulDiv extends Config((site, here, up) => {
 })
 
 class WithoutFPU extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(fpu = None)))
     case t => t
@@ -425,7 +430,7 @@ class WithoutFPU extends Config((site, here, up) => {
 })
 
 class WithFPUWithoutDivSqrt extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(fpu = tp.tileParams.core.fpu.map(_.copy(divSqrt = false)))))
     case t => t
@@ -433,7 +438,7 @@ class WithFPUWithoutDivSqrt extends Config((site, here, up) => {
 })
 
 class WithBitManip extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(useBitManip = true)))
     case t => t
@@ -441,7 +446,7 @@ class WithBitManip extends Config((site, here, up) => {
 })
 
 class WithBitManipCrypto extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(useBitManipCrypto = true)))
     case t => t
@@ -449,7 +454,7 @@ class WithBitManipCrypto extends Config((site, here, up) => {
 })
 
 class WithCryptoNIST extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(useCryptoNIST = true)))
     case t => t
@@ -457,7 +462,7 @@ class WithCryptoNIST extends Config((site, here, up) => {
 })
 
 class WithCryptoSM extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(useCryptoSM = true)))
     case t => t
@@ -497,7 +502,7 @@ class WithClockGateModel(file: String = "/vsrc/EICG_wrapper.v") extends Config((
 })
 
 class WithSynchronousRocketTiles extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
       crossingType = SynchronousCrossing()))
     case t => t
@@ -505,7 +510,7 @@ class WithSynchronousRocketTiles extends Config((site, here, up) => {
 })
 
 class WithAsynchronousRocketTiles(depth: Int, sync: Int) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
       crossingType = AsynchronousCrossing()))
     case t => t
@@ -513,7 +518,7 @@ class WithAsynchronousRocketTiles(depth: Int, sync: Int) extends Config((site, h
 })
 
 class WithRationalRocketTiles extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
       crossingType = RationalCrossing()))
     case t => t
@@ -523,7 +528,6 @@ class WithRationalRocketTiles extends Config((site, here, up) => {
 class WithEdgeDataBits(dataBits: Int) extends Config((site, here, up) => {
   case MemoryBusKey => up(MemoryBusKey, site).copy(beatBytes = dataBits/8)
   case ExtIn => up(ExtIn, site).map(_.copy(beatBytes = dataBits/8))
-  
 })
 
 class WithJtagDTM extends Config ((site, here, up) => {
@@ -616,7 +620,7 @@ class WithScratchpadsBaseAddress(address: BigInt) extends Config((site, here, up
 })
 
 class WithScratchpadsOnly extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem), site) map {
+  case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(useVM = false),
       dcache = tp.tileParams.dcache.map(_.copy(
@@ -675,16 +679,41 @@ class WithDontDriveBusClocksFromSBus extends Config((site, here, up) => {
   case DriveClocksFromSBus => false
 })
 
-class WithCloneRocketTiles(n: Int = 1, cloneHart: Int = 0, overrideIdOffset: Option[Int] = None) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => {
-    val prev = up(TilesLocated(InSubsystem), site)
-    val idOffset = overrideIdOffset.getOrElse(prev.size)
-    val tileAttachParams = prev(cloneHart).asInstanceOf[RocketTileAttachParams]
+class WithCloneRocketTiles(
+  n: Int = 1,
+  cloneTileId: Int = 0,
+  location: HierarchicalLocation = InSubsystem,
+  cloneLocation: HierarchicalLocation = InSubsystem
+) extends Config((site, here, up) => {
+  case TilesLocated(`location`) => {
+    val prev = up(TilesLocated(location), site)
+    val idOffset = up(NumTiles)
+    val tileAttachParams = up(TilesLocated(cloneLocation)).find(_.tileParams.tileId == cloneTileId)
+      .get.asInstanceOf[RocketTileAttachParams]
     (0 until n).map { i =>
-      CloneTileAttachParams(cloneHart, tileAttachParams.copy(
+      CloneTileAttachParams(cloneTileId, tileAttachParams.copy(
         tileParams = tileAttachParams.tileParams.copy(tileId = i + idOffset)
       ))
     } ++ prev
   }
+  case NumTiles => up(NumTiles) + n
 })
 
+class WithCloneCluster(
+  clusterId: Int,
+  cloneClusterId: Int = 0,
+  location: HierarchicalLocation = InSubsystem,
+  cloneLocation: HierarchicalLocation = InSubsystem
+) extends Config((site, here, up) => {
+  case ClustersLocated(`location`) => {
+    val prev = up(ClustersLocated(location))
+    val clusterAttachParams = up(ClustersLocated(cloneLocation)).find(_.clusterParams.clusterId == cloneClusterId)
+      .get.asInstanceOf[ClusterAttachParams]
+    prev :+ CloneClusterAttachParams(
+      cloneClusterId,
+      clusterAttachParams.copy(clusterParams = clusterAttachParams.clusterParams.copy(clusterId = clusterId))
+    )
+  }
+  case TLNetworkTopologyLocated(InCluster(`clusterId`)) => site(TLNetworkTopologyLocated(InCluster(cloneClusterId)))
+  case PossibleTileLocations => up(PossibleTileLocations) :+ InCluster(clusterId)
+})
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasElements.scala
index d07fce53cac..0c7330d6881 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasElements.scala
@@ -14,6 +14,7 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.prci.{ClockGroup, ResetCrossingType, ClockGroupNode}
 import freechips.rocketchip.util._
 import freechips.rocketchip.rocket.{TracedInstruction}
+import scala.collection.immutable.ListMap
 
 /** A default implementation of parameterizing the connectivity of the port where the tile is the master.
   *   Optional timing buffers and/or an optional CacheCork can be inserted in the interconnect's clock domain.
@@ -66,28 +67,28 @@ trait InstantiatesElements { this: LazyModule with Attachable =>
   val tileCrossingTypes: Seq[ClockCrossingType] = tileAttachParams.map(_.crossingParams.crossingType)
 
   /** The actual list of instantiated tiles in this block. */
-  val tile_prci_domains: Seq[TilePRCIDomain[_]] = tileAttachParams.foldLeft(Seq[TilePRCIDomain[_]]()) {
-    case (instantiated, params) => instantiated :+ params.instantiate(tileParams, instantiated)(p)
+  val tile_prci_domains: ListMap[Int, TilePRCIDomain[_]] = tileAttachParams.foldLeft(ListMap[Int, TilePRCIDomain[_]]()) {
+    case (instantiated, params) => instantiated + (params.tileParams.tileId -> params.instantiate(tileParams, instantiated)(p))
   }
 
   val clusterAttachParams: Seq[CanAttachCluster] = p(ClustersLocated(location)).sortBy(_.clusterParams.clusterId)
   val clusterParams: Seq[ClusterParams] = clusterAttachParams.map(_.clusterParams)
   val clusterCrossingTypes: Seq[ClockCrossingType] = clusterAttachParams.map(_.crossingParams.crossingType)
-  val cluster_prci_domains: Seq[ClusterPRCIDomain] = clusterAttachParams.foldLeft(Seq[ClusterPRCIDomain]()) {
-    case (instantiated, params) => instantiated :+ params.instantiate(clusterParams, instantiated)(p)
+  val cluster_prci_domains: ListMap[Int, ClusterPRCIDomain] = clusterAttachParams.foldLeft(ListMap[Int, ClusterPRCIDomain]()) {
+    case (instantiated, params) => instantiated + (params.clusterParams.clusterId -> params.instantiate(clusterParams, instantiated)(p))
   }
 
-  val element_prci_domains: Seq[ElementPRCIDomain[_]] = tile_prci_domains ++ cluster_prci_domains
+  val element_prci_domains: Seq[ElementPRCIDomain[_]] = tile_prci_domains.values.toSeq ++ cluster_prci_domains.values.toSeq
 
-  val leafTiles: Seq[BaseTile] = tile_prci_domains.map(_.element.asInstanceOf[BaseTile]).sortBy(_.tileId)
-  val totalTiles: Seq[BaseTile] = (leafTiles ++ cluster_prci_domains.map(_.element.asInstanceOf[Cluster].totalTiles).flatten).sortBy(_.tileId)
+  val leafTiles: ListMap[Int, BaseTile] = tile_prci_domains.mapValues(_.element.asInstanceOf[BaseTile]).to(ListMap)
+  val totalTiles: ListMap[Int, BaseTile] = (leafTiles ++ cluster_prci_domains.values.map(_.element.totalTiles).flatten)
 
   // Helper functions for accessing certain parameters that are popular to refer to in subsystem code
   def nLeafTiles: Int = leafTiles.size
   def nTotalTiles: Int = totalTiles.size
-  def leafTileIdList: Seq[Int] = leafTiles.map(_.tileId)
-  def totalTileIdList: Seq[Int] = totalTiles.map(_.tileId)
-  def localIntCounts: Seq[Int] = totalTiles.map(_.tileParams.core.nLocalInterrupts)
+  def leafTileIdList: Seq[Int] = leafTiles.keys.toSeq.sorted
+  def totalTileIdList: Seq[Int] = totalTiles.keys.toSeq.sorted
+  def localIntCounts: ListMap[Int, Int] = totalTiles.mapValues(_.tileParams.core.nLocalInterrupts).to(ListMap)
 
   require(totalTileIdList.distinct.size == totalTiles.size, s"Every tile must be statically assigned a unique id, but got:\n${totalTileIdList}")
 }
@@ -98,12 +99,11 @@ trait HasElements extends DefaultElementContextType
   implicit val p: Parameters
 
   // connect all the tiles to interconnect attachment points made available in this subsystem context
-  tileAttachParams.zip(tile_prci_domains).foreach { case (params, td) =>
-    params.connect(td.asInstanceOf[TilePRCIDomain[params.TileType]], this.asInstanceOf[params.TileContextType])
+  tileAttachParams.foreach { params =>
+    params.connect(tile_prci_domains(params.tileParams.tileId).asInstanceOf[TilePRCIDomain[params.TileType]], this.asInstanceOf[params.TileContextType])
   }
-
-  clusterAttachParams.zip(cluster_prci_domains).foreach { case (params, cd) =>
-    params.connect(cd.asInstanceOf[ClusterPRCIDomain], this.asInstanceOf[params.ClusterContextType])
+  clusterAttachParams.foreach { params =>
+    params.connect(cluster_prci_domains(params.clusterParams.clusterId).asInstanceOf[ClusterPRCIDomain], this.asInstanceOf[params.ClusterContextType])
   }
 }
 
@@ -142,16 +142,16 @@ trait DefaultElementContextType
     extends Attachable
     with HasTileNotificationSinks
 { this: LazyModule with Attachable =>
-  val msipNodes: Map[Int, IntNode]
-  val meipNodes: Map[Int, IntNode]
-  val seipNodes: Map[Int, IntNode]
-  val tileToPlicNodes: Map[Int, IntNode]
-  val debugNodes: Map[Int, IntSyncNode]
-  val nmiNodes: Map[Int, BundleBridgeNode[NMI]]
-  val tileHartIdNodes: Map[Int, BundleBridgeNode[UInt]]
-  val tileResetVectorNodes: Map[Int, BundleBridgeNode[UInt]]
-  val traceCoreNodes: Map[Int, BundleBridgeNode[TraceCoreInterface]]
-  val traceNodes: Map[Int, BundleBridgeNode[TraceBundle]]
+  val msipNodes: ListMap[Int, IntNode]
+  val meipNodes: ListMap[Int, IntNode]
+  val seipNodes: ListMap[Int, IntNode]
+  val tileToPlicNodes: ListMap[Int, IntNode]
+  val debugNodes: ListMap[Int, IntSyncNode]
+  val nmiNodes: ListMap[Int, BundleBridgeNode[NMI]]
+  val tileHartIdNodes: ListMap[Int, BundleBridgeNode[UInt]]
+  val tileResetVectorNodes: ListMap[Int, BundleBridgeNode[UInt]]
+  val traceCoreNodes: ListMap[Int, BundleBridgeNode[TraceCoreInterface]]
+  val traceNodes: ListMap[Int, BundleBridgeNode[TraceBundle]]
 }
 
 /** This trait provides the tile attachment context for the root (outermost) subsystem */
@@ -164,9 +164,9 @@ trait HasElementsRootContext
   val plicOpt: Option[TLPLIC]
   val debugOpt: Option[TLDebugModule]
 
-  val msipNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
+  val msipNodes: ListMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode())
-  }.toMap
+  }.to(ListMap)
   msipNodes.foreach {
     _._2 := clintOpt.map(_.intnode).getOrElse(NullIntSource(sources = CLINTConsts.ints))
   }
@@ -176,38 +176,36 @@ trait HasElementsRootContext
     sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
     outputRequiresInput = false,
     inputRequiresOutput = false))
-  val meipNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
+  val meipNodes: ListMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(meipIONode.get))
-  }.toMap
+  }.to(ListMap)
 
   val seipIONode = Option.when(plicOpt.isEmpty)(IntNexusNode(
     sourceFn = { _ => IntSourcePortParameters(Seq(IntSourceParameters(1))) },
     sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
     outputRequiresInput = false,
     inputRequiresOutput = false))
-  val seipNodes: Map[Int, IntNode] = totalTiles.filter(_.tileParams.core.hasSupervisorMode)
-    .map(_.tileId).map { i =>
-      (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(seipIONode.get))
-    }.toMap
+  val seipNodes: ListMap[Int, IntNode] = totalTiles.filter { case (_, t) => t.tileParams.core.hasSupervisorMode }
+    .mapValues( _ => IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(seipIONode.get)).to(ListMap)
 
-  val tileToPlicNodes: Map[Int, IntNode] = (0 until nTotalTiles).map { i =>
+  val tileToPlicNodes: ListMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
     plicOpt.map(o => (i, o.intnode :=* IntEphemeralNode()))
-  }.flatten.toMap
+  }.flatten.to(ListMap)
 
-  val debugNodes: Map[Int, IntSyncNode] = (0 until nTotalTiles).map { i =>
+  val debugNodes: ListMap[Int, IntSyncNode] = (0 until nTotalTiles).map { i =>
     (i, IntSyncIdentityNode())
-  }.toMap
+  }.to(ListMap)
 
   debugNodes.foreach { case (hartid, node) =>
     node := debugOpt.map(_.intnode).getOrElse(IntSyncCrossingSource() := NullIntSource())
   }
 
-  val nmiHarts = totalTiles.filter(_.tileParams.core.useNMI).map(_.tileId)
-  val nmiIONodes = nmiHarts.map { i => (i, BundleBridgeSource[NMI]()) }.toMap
-  val nmiNodes: Map[Int, BundleBridgeNode[NMI]] = nmiIONodes.map { case (i, n) =>
+  val nmiHarts = totalTiles.filter { case (_, t) => t.tileParams.core.useNMI }.keys
+  val nmiIONodes = nmiHarts.map { i => (i, BundleBridgeSource[NMI]()) }.to(ListMap)
+  val nmiNodes: ListMap[Int, BundleBridgeNode[NMI]] = nmiIONodes.map { case (i, n) =>
     (i, BundleBridgeEphemeralNode[NMI]() := n)
-  }.toMap
+  }.to(ListMap)
 
-  val traceCoreNodes: Map[Int, BundleBridgeSink[TraceCoreInterface]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceCoreInterface]()) }.toMap
-  val traceNodes: Map[Int, BundleBridgeSink[TraceBundle]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceBundle]()) }.toMap
+  val traceCoreNodes: ListMap[Int, BundleBridgeSink[TraceCoreInterface]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceCoreInterface]()) }.to(ListMap)
+  val traceNodes: ListMap[Int, BundleBridgeSink[TraceBundle]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceBundle]()) }.to(ListMap)
 }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 01907871e69..bc0e3351f8e 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -14,6 +14,7 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.prci._
 import freechips.rocketchip.util._
 import freechips.rocketchip.rocket.{TracedInstruction}
+import scala.collection.immutable.ListMap
 
 /** Entry point for Config-uring the presence of Tiles */
 case class TilesLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachTile]](Nil)
@@ -52,9 +53,9 @@ case object HasTilesExternalResetVectorKey extends Field[Boolean](true)
   */
 trait HasTileInputConstants { this: LazyModule with Attachable with InstantiatesElements =>
   /** tileHartIdNode is used to collect publishers and subscribers of hartids. */
-  val tileHartIdNodes: Map[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
+  val tileHartIdNodes: ListMap[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
     (i, BundleBridgeEphemeralNode[UInt]())
-  }.toMap
+  }.to(ListMap)
 
   /** tileHartIdNexusNode is a BundleBridgeNexus that collects dynamic hart prefixes.
     *
@@ -81,9 +82,9 @@ trait HasTileInputConstants { this: LazyModule with Attachable with Instantiates
   // TODO: Replace the DebugModuleHartSelFuncs config key with logic to consume the dynamic hart IDs
 
   /** tileResetVectorNode is used to collect publishers and subscribers of tile reset vector addresses. */
-  val tileResetVectorNodes: Map[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
+  val tileResetVectorNodes: ListMap[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
     (i, BundleBridgeEphemeralNode[UInt]())
-  }.toMap
+  }.to(ListMap)
 
   /** tileResetVectorNexusNode is a BundleBridgeNexus that accepts a single reset vector source, and broadcasts it to all tiles. */
   val tileResetVectorNexusNode = BundleBroadcast[UInt](
@@ -154,7 +155,7 @@ trait CanAttachTile {
   def crossingParams: ElementCrossingParamsLike
 
   /** Narrow waist through which all tiles are intended to pass while being instantiated. */
-  def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: Seq[TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
+  def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: ListMap[Int, TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
     val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.uniqueName))
     val tile_prci_domain = LazyModule(new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
       val element = self.element_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
@@ -292,7 +293,7 @@ trait CanAttachTile {
 }
 
 case class CloneTileAttachParams(
-  sourceHart: Int,
+  sourceTileId: Int,
   cloneParams: CanAttachTile
 ) extends CanAttachTile {
   type TileType = cloneParams.TileType
@@ -301,14 +302,14 @@ case class CloneTileAttachParams(
   def tileParams = cloneParams.tileParams
   def crossingParams = cloneParams.crossingParams
 
-  override def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: Seq[TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
-    require(sourceHart < instantiatedTiles.size)
+  override def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: ListMap[Int, TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
+    require(instantiatedTiles.contains(sourceTileId))
     val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.uniqueName))
     val tile_prci_domain = CloneLazyModule(
       new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
         val element = self.element_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
       },
-      instantiatedTiles(sourceHart).asInstanceOf[TilePRCIDomain[TileType]]
+      instantiatedTiles(sourceTileId).asInstanceOf[TilePRCIDomain[TileType]]
     )
     tile_prci_domain
   }
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index 7d9ee26e562..c6bc28321f6 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -26,7 +26,7 @@ case class RocketTileAttachParams(
 
 trait HasRocketTiles {
   this: BaseSubsystem with InstantiatesElements =>
-  val rocketTiles = totalTiles.collect { case r: RocketTile => r }
+  val rocketTiles = totalTiles.values.collect { case r: RocketTile => r }
 
   def coreMonitorBundles = (rocketTiles map { t =>
     t.module.core.rocketImpl.coreMonitorBundle

From 8f220f10ab27122592b317c39679cc8ca663f0ab Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 19 Sep 2023 16:29:18 -0700
Subject: [PATCH 37/47] Rename Element to HierarchicalElement

---
 src/main/scala/devices/tilelink/BootROM.scala |  2 +-
 .../groundtest/GroundTestSubsystem.scala      |  6 ++--
 src/main/scala/groundtest/TraceGen.scala      |  8 ++---
 src/main/scala/subsystem/Cluster.scala        | 22 +++++++-------
 src/main/scala/subsystem/Configs.scala        |  4 +--
 ...ts.scala => HasHierarchicalElements.scala} | 30 +++++++++----------
 src/main/scala/subsystem/HasTiles.scala       |  6 ++--
 ...lement.scala => HierarchicalElement.scala} | 20 ++++++-------
 ...la => HierarchicalElementPRCIDomain.scala} |  8 ++---
 .../scala/subsystem/RocketSubsystem.scala     | 16 +++++-----
 src/main/scala/tile/BaseTile.scala            | 10 +++----
 src/main/scala/tile/RocketTile.scala          |  6 ++--
 src/main/scala/tile/TilePRCIDomain.scala      |  4 +--
 13 files changed, 71 insertions(+), 71 deletions(-)
 rename src/main/scala/subsystem/{HasElements.scala => HasHierarchicalElements.scala} (90%)
 rename src/main/scala/subsystem/{Element.scala => HierarchicalElement.scala} (80%)
 rename src/main/scala/subsystem/{ElementPRCIDomain.scala => HierarchicalElementPRCIDomain.scala} (91%)

diff --git a/src/main/scala/devices/tilelink/BootROM.scala b/src/main/scala/devices/tilelink/BootROM.scala
index 9431943072b..22ab348d752 100644
--- a/src/main/scala/devices/tilelink/BootROM.scala
+++ b/src/main/scala/devices/tilelink/BootROM.scala
@@ -66,7 +66,7 @@ object BootROM {
     *    at a configurable location, but also drives the tiles' reset vectors to point
     *    at its 'hang' address parameter value.
     */
-  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasElements with HasTileInputConstants, where: TLBusWrapperLocation)
+  def attach(params: BootROMParams, subsystem: BaseSubsystem with HasHierarchicalElements with HasTileInputConstants, where: TLBusWrapperLocation)
             (implicit p: Parameters): TLROM = {
     val tlbus = subsystem.locateTLBusWrapper(where)
     val bootROMDomainWrapper = LazyModule(new ClockSinkDomain(take = None))
diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index d435acce080..0aece29c5bf 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -14,9 +14,9 @@ import freechips.rocketchip.interrupts.{NullIntSyncSource}
 
 class GroundTestSubsystem(implicit p: Parameters)
   extends BaseSubsystem
-  with InstantiatesElements
-  with HasElementsRootContext
-  with HasElements
+  with InstantiatesHierarchicalElements
+  with HasHierarchicalElementsRootContext
+  with HasHierarchicalElements
   with HasTileNotificationSinks
   with HasTileInputConstants
   with CanHaveMasterAXI4MemPort
diff --git a/src/main/scala/groundtest/TraceGen.scala b/src/main/scala/groundtest/TraceGen.scala
index 9da61ad1fbe..f60c41517cd 100644
--- a/src/main/scala/groundtest/TraceGen.scala
+++ b/src/main/scala/groundtest/TraceGen.scala
@@ -26,7 +26,7 @@ import freechips.rocketchip.diplomacy.{ClockCrossingType}
 import freechips.rocketchip.rocket._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.subsystem.{ElementCrossingParamsLike, CanAttachTile}
+import freechips.rocketchip.subsystem.{HierarchicalElementCrossingParamsLike, CanAttachTile}
 import freechips.rocketchip.util._
 import freechips.rocketchip.prci.{ClockSinkParameters}
 
@@ -71,7 +71,7 @@ case class TraceGenParams(
     tileId: Int = 0
 ) extends InstantiableTileParams[TraceGenTile] with GroundTestTileParams
 {
-  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
+  def instantiate(crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): TraceGenTile = {
     new TraceGenTile(this, crossing, lookup)
   }
   val blockerCtrlAddr = None
@@ -105,7 +105,7 @@ trait HasTraceGenParams {
 
 case class TraceGenTileAttachParams(
   tileParams: TraceGenParams,
-  crossingParams: ElementCrossingParamsLike
+  crossingParams: HierarchicalElementCrossingParamsLike
 ) extends CanAttachTile {
   type TileType = TraceGenTile
   val lookup: LookupByHartIdImpl = HartsWontDeduplicate(tileParams)
@@ -617,7 +617,7 @@ class TraceGenTile private(
   q: Parameters
 ) extends GroundTestTile(params, crossing, lookup, q)
 {
-  def this(params: TraceGenParams, crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: TraceGenParams, crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   val masterNode: TLOutwardNode = TLIdentityNode() := visibilityNode := dcacheOpt.map(_.node).getOrElse(TLTempNode())
diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index 8febbe58ee7..4a50d1923f6 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -20,10 +20,10 @@ case class ClustersLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttac
 case class ClusterParams(
   val clusterId: Int,
   val clockSinkParams: ClockSinkParameters = ClockSinkParameters()
-) extends ElementParams {
+) extends HierarchicalElementParams {
   val baseName = "cluster"
   val uniqueName = s"${baseName}_$clusterId"
-  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByClusterIdImpl)(implicit p: Parameters): Cluster = {
+  def instantiate(crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByClusterIdImpl)(implicit p: Parameters): Cluster = {
     new Cluster(this, crossing.crossingType, lookup)
   }
 }
@@ -31,11 +31,11 @@ case class ClusterParams(
 class Cluster(
   val thisClusterParams: ClusterParams,
   crossing: ClockCrossingType,
-  lookup: LookupByClusterIdImpl)(implicit p: Parameters) extends BaseElement(crossing)(p)
+  lookup: LookupByClusterIdImpl)(implicit p: Parameters) extends BaseHierarchicalElement(crossing)(p)
     with Attachable
     with HasConfigurableTLNetworkTopology
-    with InstantiatesElements
-    with HasElements
+    with InstantiatesHierarchicalElements
+    with HasHierarchicalElements
 {
   val busContextName = thisClusterParams.baseName
   lazy val clusterId = thisClusterParams.clusterId
@@ -78,16 +78,16 @@ class Cluster(
   override lazy val module = new ClusterModuleImp(this)
 }
 
-class ClusterModuleImp(outer: Cluster) extends BaseElementModuleImp[Cluster](outer)
+class ClusterModuleImp(outer: Cluster) extends BaseHierarchicalElementModuleImp[Cluster](outer)
 
 case class InCluster(id: Int) extends HierarchicalLocation(s"Cluster$id")
 
 class ClusterPRCIDomain(
   clockSinkParams: ClockSinkParameters,
-  crossingParams: ElementCrossingParamsLike,
+  crossingParams: HierarchicalElementCrossingParamsLike,
   clusterParams: ClusterParams,
   lookup: LookupByClusterIdImpl)
-  (implicit p: Parameters) extends ElementPRCIDomain[Cluster](clockSinkParams, crossingParams)
+  (implicit p: Parameters) extends HierarchicalElementPRCIDomain[Cluster](clockSinkParams, crossingParams)
 {
   val element = element_reset_domain {
     LazyModule(clusterParams.instantiate(crossingParams, lookup))
@@ -98,10 +98,10 @@ class ClusterPRCIDomain(
 
 
 trait CanAttachCluster {
-  type ClusterContextType <: DefaultElementContextType
+  type ClusterContextType <: DefaultHierarchicalElementContextType
 
   def clusterParams: ClusterParams
-  def crossingParams: ElementCrossingParamsLike
+  def crossingParams: HierarchicalElementCrossingParamsLike
 
   def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: ListMap[Int, ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
     val clockSinkParams = clusterParams.clockSinkParams.copy(name = Some(clusterParams.uniqueName))
@@ -208,7 +208,7 @@ trait CanAttachCluster {
 
 case class ClusterAttachParams(
   clusterParams: ClusterParams,
-  crossingParams: ElementCrossingParamsLike
+  crossingParams: HierarchicalElementCrossingParamsLike
 ) extends CanAttachCluster
 
 case class CloneClusterAttachParams(
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 1f4c47aa7ba..fac37a0dd28 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -214,7 +214,7 @@ class With1TinyCore extends Config((site, here, up) => {
       tiny,
       RocketCrossingParams(
         crossingType = SynchronousCrossing(),
-        master = ElementMasterPortParams())
+        master = HierarchicalElementMasterPortParams())
     ))
   }
   case NumTiles => 1
@@ -320,7 +320,7 @@ class WithIncoherentTiles extends Config((site, here, up) => {
   case TilesLocated(location) => up(TilesLocated(location), site) map {
     case tp: RocketTileAttachParams => tp.copy(crossingParams = tp.crossingParams.copy(
       master = tp.crossingParams.master match {
-        case x: ElementMasterPortParams => x.copy(cork = Some(true))
+        case x: HierarchicalElementMasterPortParams => x.copy(cork = Some(true))
         case _ => throw new Exception("Unrecognized type for RocketCrossingParams.master")
       }))
     case t => t
diff --git a/src/main/scala/subsystem/HasElements.scala b/src/main/scala/subsystem/HasHierarchicalElements.scala
similarity index 90%
rename from src/main/scala/subsystem/HasElements.scala
rename to src/main/scala/subsystem/HasHierarchicalElements.scala
index 0c7330d6881..bcc4382da8e 100644
--- a/src/main/scala/subsystem/HasElements.scala
+++ b/src/main/scala/subsystem/HasHierarchicalElements.scala
@@ -19,11 +19,11 @@ import scala.collection.immutable.ListMap
 /** A default implementation of parameterizing the connectivity of the port where the tile is the master.
   *   Optional timing buffers and/or an optional CacheCork can be inserted in the interconnect's clock domain.
   */
-case class ElementMasterPortParams(
+case class HierarchicalElementMasterPortParams(
   buffers: Int = 0,
   cork: Option[Boolean] = None,
   where: TLBusWrapperLocation = SBUS
-) extends ElementPortParamsLike {
+) extends HierarchicalElementPortParamsLike {
   def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
     (TLBuffer.chainNode(buffers) :=* cork.map { u => TLCacheCork(unsafe = u) } .getOrElse { TLTempNode() })
   }
@@ -32,12 +32,12 @@ case class ElementMasterPortParams(
 /** A default implementation of parameterizing the connectivity of the port giving access to slaves inside the tile.
   *   Optional timing buffers and/or an optional BusBlocker adapter can be inserted in the interconnect's clock domain.
   */
-case class ElementSlavePortParams(
+case class HierarchicalElementSlavePortParams(
   buffers: Int = 0,
   blockerCtrlAddr: Option[BigInt] = None,
   blockerCtrlWhere: TLBusWrapperLocation = CBUS,
   where: TLBusWrapperLocation = CBUS
-) extends ElementPortParamsLike {
+) extends HierarchicalElementPortParamsLike {
   def injectNode(context: Attachable)(implicit p: Parameters): TLNode = {
     val controlBus = context.locateTLBusWrapper(where)
     val blockerBus = context.locateTLBusWrapper(blockerCtrlWhere)
@@ -51,10 +51,10 @@ case class ElementSlavePortParams(
   }
 }
 
-/** InstantiatesTiles adds a Config-urable sequence of Elements of any type
+/** InstantiatesTiles adds a Config-urable sequence of HierarchicalElements of any type
   *   to the subsystem class into which it is mixed.
   */
-trait InstantiatesElements { this: LazyModule with Attachable =>
+trait InstantiatesHierarchicalElements { this: LazyModule with Attachable =>
   val location: HierarchicalLocation
 
   /** Record the order in which to instantiate all tiles, based on statically-assigned ids.
@@ -78,7 +78,7 @@ trait InstantiatesElements { this: LazyModule with Attachable =>
     case (instantiated, params) => instantiated + (params.clusterParams.clusterId -> params.instantiate(clusterParams, instantiated)(p))
   }
 
-  val element_prci_domains: Seq[ElementPRCIDomain[_]] = tile_prci_domains.values.toSeq ++ cluster_prci_domains.values.toSeq
+  val element_prci_domains: Seq[HierarchicalElementPRCIDomain[_]] = tile_prci_domains.values.toSeq ++ cluster_prci_domains.values.toSeq
 
   val leafTiles: ListMap[Int, BaseTile] = tile_prci_domains.mapValues(_.element.asInstanceOf[BaseTile]).to(ListMap)
   val totalTiles: ListMap[Int, BaseTile] = (leafTiles ++ cluster_prci_domains.values.map(_.element.totalTiles).flatten)
@@ -94,8 +94,8 @@ trait InstantiatesElements { this: LazyModule with Attachable =>
 }
 
 /** HasTiles instantiates and also connects a Config-urable sequence of tiles of any type to subsystem interconnect resources. */
-trait HasElements extends DefaultElementContextType
-{ this: LazyModule with Attachable with InstantiatesElements =>
+trait HasHierarchicalElements extends DefaultHierarchicalElementContextType
+{ this: LazyModule with Attachable with InstantiatesHierarchicalElements =>
   implicit val p: Parameters
 
   // connect all the tiles to interconnect attachment points made available in this subsystem context
@@ -110,8 +110,8 @@ trait HasElements extends DefaultElementContextType
 /** Provides some Chisel connectivity to certain tile IOs
   * This trait is intended for the root subsystem
   */
-trait HasElementsRootContextModuleImp extends LazyRawModuleImp {
-  val outer: InstantiatesElements with HasElements with HasElementsRootContext with HasTileInputConstants
+trait HasHierarchicalElementsRootContextModuleImp extends LazyRawModuleImp {
+  val outer: InstantiatesHierarchicalElements with HasHierarchicalElements with HasHierarchicalElementsRootContext with HasTileInputConstants
 
   val reset_vector = outer.tileResetVectorIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"reset_vector_$i") }
   val tile_hartids = outer.tileHartIdIONodes.zipWithIndex.map { case (n, i) => n.makeIO(s"tile_hartids_$i") }
@@ -138,7 +138,7 @@ trait HasElementsRootContextModuleImp extends LazyRawModuleImp {
   *    BaseTiles subtypes with different needs can extend this trait to provide themselves with
   *    additional external connection points.
   */
-trait DefaultElementContextType
+trait DefaultHierarchicalElementContextType
     extends Attachable
     with HasTileNotificationSinks
 { this: LazyModule with Attachable =>
@@ -155,10 +155,10 @@ trait DefaultElementContextType
 }
 
 /** This trait provides the tile attachment context for the root (outermost) subsystem */
-trait HasElementsRootContext
-{ this: HasElements
+trait HasHierarchicalElementsRootContext
+{ this: HasHierarchicalElements
     with HasTileNotificationSinks
-    with InstantiatesElements =>
+    with InstantiatesHierarchicalElements =>
 
   val clintOpt: Option[CLINT]
   val plicOpt: Option[TLPLIC]
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index bc0e3351f8e..7f1b04ddec4 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -51,7 +51,7 @@ case object HasTilesExternalResetVectorKey extends Field[Boolean](true)
   * they may be either tied to a contant value or programmed during boot or reset.
   * They need to be instantiated before tiles are attached within the subsystem containing them.
   */
-trait HasTileInputConstants { this: LazyModule with Attachable with InstantiatesElements =>
+trait HasTileInputConstants { this: LazyModule with Attachable with InstantiatesHierarchicalElements =>
   /** tileHartIdNode is used to collect publishers and subscribers of hartids. */
   val tileHartIdNodes: ListMap[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
     (i, BundleBridgeEphemeralNode[UInt]())
@@ -150,9 +150,9 @@ trait HasTileNotificationSinks { this: LazyModule =>
   */
 trait CanAttachTile {
   type TileType <: BaseTile
-  type TileContextType <: DefaultElementContextType
+  type TileContextType <: DefaultHierarchicalElementContextType
   def tileParams: InstantiableTileParams[TileType]
-  def crossingParams: ElementCrossingParamsLike
+  def crossingParams: HierarchicalElementCrossingParamsLike
 
   /** Narrow waist through which all tiles are intended to pass while being instantiated. */
   def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: ListMap[Int, TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
diff --git a/src/main/scala/subsystem/Element.scala b/src/main/scala/subsystem/HierarchicalElement.scala
similarity index 80%
rename from src/main/scala/subsystem/Element.scala
rename to src/main/scala/subsystem/HierarchicalElement.scala
index ce35e01032f..18a7c189f3e 100644
--- a/src/main/scala/subsystem/Element.scala
+++ b/src/main/scala/subsystem/HierarchicalElement.scala
@@ -14,22 +14,22 @@ import freechips.rocketchip.util._
 import freechips.rocketchip.devices.debug.{TLDebugModule}
 import freechips.rocketchip.devices.tilelink._
 
-trait ElementParams {
+trait HierarchicalElementParams {
   val baseName: String // duplicated instances shouuld share a base name
   val uniqueName: String
   val clockSinkParams: ClockSinkParameters
 }
 
-abstract class InstantiableElementParams[ElementType <: BaseElement] extends ElementParams
+abstract class InstantiableHierarchicalElementParams[ElementType <: BaseHierarchicalElement] extends HierarchicalElementParams
 
-/** An interface for describing the parameteization of how Elements are connected to interconnects */
-trait ElementCrossingParamsLike {
+/** An interface for describing the parameteization of how HierarchicalElements are connected to interconnects */
+trait HierarchicalElementCrossingParamsLike {
   /** The type of clock crossing that should be inserted at the element boundary. */
   def crossingType: ClockCrossingType
   /** Parameters describing the contents and behavior of the point where the element is attached as an interconnect master. */
-  def master: ElementPortParamsLike
+  def master: HierarchicalElementPortParamsLike
   /** Parameters describing the contents and behavior of the point where the element is attached as an interconnect slave. */
-  def slave: ElementPortParamsLike
+  def slave: HierarchicalElementPortParamsLike
   /** The subnetwork location of the device selecting the apparent base address of MMIO devices inside the element */
   def mmioBaseAddressPrefixWhere: TLBusWrapperLocation
   /** Inject a reset management subgraph that effects the element child reset only */
@@ -39,18 +39,18 @@ trait ElementCrossingParamsLike {
 }
 
 /** An interface for describing the parameterization of how a particular element port is connected to an interconnect */
-trait ElementPortParamsLike {
+trait HierarchicalElementPortParamsLike {
   /** The subnetwork location of the interconnect to which this element port should be connected. */
   def where: TLBusWrapperLocation
   /** Allows port-specific adapters to be injected into the interconnect side of the attachment point. */
   def injectNode(context: Attachable)(implicit p: Parameters): TLNode
 }
 
-abstract class BaseElement (val crossing: ClockCrossingType)(implicit p: Parameters)
+abstract class BaseHierarchicalElement (val crossing: ClockCrossingType)(implicit p: Parameters)
     extends LazyModule()(p)
     with CrossesToOnlyOneClockDomain
 {
-  def module: BaseElementModuleImp[BaseElement]
+  def module: BaseHierarchicalElementModuleImp[BaseHierarchicalElement]
 
   protected val tlOtherMastersNode = TLIdentityNode()
   protected val tlMasterXbar = LazyModule(new TLXbar)
@@ -79,5 +79,5 @@ abstract class BaseElement (val crossing: ClockCrossingType)(implicit p: Paramet
 
 }
 
-abstract class BaseElementModuleImp[+L <: BaseElement](val outer: L) extends LazyModuleImp(outer)
+abstract class BaseHierarchicalElementModuleImp[+L <: BaseHierarchicalElement](val outer: L) extends LazyModuleImp(outer)
 
diff --git a/src/main/scala/subsystem/ElementPRCIDomain.scala b/src/main/scala/subsystem/HierarchicalElementPRCIDomain.scala
similarity index 91%
rename from src/main/scala/subsystem/ElementPRCIDomain.scala
rename to src/main/scala/subsystem/HierarchicalElementPRCIDomain.scala
index b030ab21ba6..a76b7c43402 100644
--- a/src/main/scala/subsystem/ElementPRCIDomain.scala
+++ b/src/main/scala/subsystem/HierarchicalElementPRCIDomain.scala
@@ -19,7 +19,7 @@ import freechips.rocketchip.util.{TraceCoreInterface}
   *
   * This does not add a layer of the module hierarchy.
   */
-class ElementResetDomain(clockSinkParams: ClockSinkParameters, resetCrossingType: ResetCrossingType)
+class HierarchicalElementResetDomain(clockSinkParams: ClockSinkParameters, resetCrossingType: ResetCrossingType)
                         (implicit p: Parameters)
     extends ResetDomain
     with CrossesToOnlyOneResetDomain
@@ -38,14 +38,14 @@ class ElementResetDomain(clockSinkParams: ClockSinkParameters, resetCrossingType
   * hierarchical P&R boundary buffers, core-local interrupt handling,
   * and any other IOs related to PRCI control.
   */
-abstract class ElementPRCIDomain[T <: BaseElement](
+abstract class HierarchicalElementPRCIDomain[T <: BaseHierarchicalElement](
   clockSinkParams: ClockSinkParameters,
-  crossingParams: ElementCrossingParamsLike)
+  crossingParams: HierarchicalElementCrossingParamsLike)
   (implicit p: Parameters)
     extends ClockDomain
 {
   val element: T
-  val element_reset_domain = LazyModule(new ElementResetDomain(clockSinkParams, crossingParams.resetCrossingType))
+  val element_reset_domain = LazyModule(new HierarchicalElementResetDomain(clockSinkParams, crossingParams.resetCrossingType))
   val tapClockNode = ClockIdentityNode()
   val clockNode = FixedClockBroadcast() :=* tapClockNode
   lazy val clockBundle = tapClockNode.in.head._1
diff --git a/src/main/scala/subsystem/RocketSubsystem.scala b/src/main/scala/subsystem/RocketSubsystem.scala
index c6bc28321f6..ba70ab0c24a 100644
--- a/src/main/scala/subsystem/RocketSubsystem.scala
+++ b/src/main/scala/subsystem/RocketSubsystem.scala
@@ -12,12 +12,12 @@ import freechips.rocketchip.devices.tilelink.{CanHavePeripheryCLINT, CanHavePeri
 
 case class RocketCrossingParams(
   crossingType: ClockCrossingType = SynchronousCrossing(),
-  master: ElementPortParamsLike = ElementMasterPortParams(),
-  slave: ElementSlavePortParams = ElementSlavePortParams(),
+  master: HierarchicalElementPortParamsLike = HierarchicalElementMasterPortParams(),
+  slave: HierarchicalElementSlavePortParams = HierarchicalElementSlavePortParams(),
   mmioBaseAddressPrefixWhere: TLBusWrapperLocation = CBUS,
   resetCrossingType: ResetCrossingType = NoResetCrossing(),
   forceSeparateClockReset: Boolean = false
-) extends ElementCrossingParamsLike
+) extends HierarchicalElementCrossingParamsLike
 
 case class RocketTileAttachParams(
   tileParams: RocketTileParams,
@@ -25,7 +25,7 @@ case class RocketTileAttachParams(
 ) extends CanAttachTile { type TileType = RocketTile }
 
 trait HasRocketTiles {
-  this: BaseSubsystem with InstantiatesElements =>
+  this: BaseSubsystem with InstantiatesHierarchicalElements =>
   val rocketTiles = totalTiles.values.collect { case r: RocketTile => r }
 
   def coreMonitorBundles = (rocketTiles map { t =>
@@ -34,14 +34,14 @@ trait HasRocketTiles {
 }
 
 class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
-    with InstantiatesElements
+    with InstantiatesHierarchicalElements
     with HasTileNotificationSinks
     with HasTileInputConstants
     with CanHavePeripheryCLINT
     with CanHavePeripheryPLIC
     with HasPeripheryDebug
-    with HasElementsRootContext
-    with HasElements
+    with HasHierarchicalElementsRootContext
+    with HasHierarchicalElements
     with HasCoreMonitorBundles
     with HasRocketTiles
 {
@@ -49,5 +49,5 @@ class RocketSubsystem(implicit p: Parameters) extends BaseSubsystem
 }
 
 class RocketSubsystemModuleImp[+L <: RocketSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
-    with HasElementsRootContextModuleImp
+    with HasHierarchicalElementsRootContextModuleImp
 
diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala
index 060ca985938..b05a08c985a 100644
--- a/src/main/scala/tile/BaseTile.scala
+++ b/src/main/scala/tile/BaseTile.scala
@@ -18,7 +18,7 @@ case object TileVisibilityNodeKey extends Field[TLEphemeralNode]
 case object TileKey extends Field[TileParams]
 case object LookupByHartId extends Field[LookupByHartIdImpl]
 
-trait TileParams extends ElementParams {
+trait TileParams extends HierarchicalElementParams {
   val core: CoreParams
   val icache: Option[ICacheParams]
   val dcache: Option[DCacheParams]
@@ -28,9 +28,9 @@ trait TileParams extends ElementParams {
 }
 
 abstract class InstantiableTileParams[TileType <: BaseTile]
-    extends InstantiableElementParams[TileType]
+    extends InstantiableHierarchicalElementParams[TileType]
     with TileParams {
-  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)
+  def instantiate(crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)
                  (implicit p: Parameters): TileType
 }
 
@@ -188,7 +188,7 @@ trait HasTileParameters extends HasNonDiplomaticTileParameters {
 
 /** Base class for all Tiles that use TileLink */
 abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
-    extends BaseElement(crossing)(q)
+    extends BaseHierarchicalElement(crossing)(q)
     with HasNonDiplomaticTileParameters
 {
   // Public constructor alters Parameters to supply some legacy compatibility keys
@@ -347,4 +347,4 @@ abstract class BaseTile private (crossing: ClockCrossingType, q: Parameters)
   this.suggestName(tileParams.baseName)
 }
 
-abstract class BaseTileModuleImp[+L <: BaseTile](outer: L) extends BaseElementModuleImp[L](outer)
+abstract class BaseTileModuleImp[+L <: BaseTile](outer: L) extends BaseHierarchicalElementModuleImp[L](outer)
diff --git a/src/main/scala/tile/RocketTile.scala b/src/main/scala/tile/RocketTile.scala
index 6f8025bb7aa..a22a44c7192 100644
--- a/src/main/scala/tile/RocketTile.scala
+++ b/src/main/scala/tile/RocketTile.scala
@@ -10,7 +10,7 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.rocket._
-import freechips.rocketchip.subsystem.ElementCrossingParamsLike
+import freechips.rocketchip.subsystem.HierarchicalElementCrossingParamsLike
 import freechips.rocketchip.util._
 import freechips.rocketchip.prci.{ClockSinkParameters}
 
@@ -32,7 +32,7 @@ case class RocketTileParams(
   require(dcache.isDefined)
   val baseName = "rockettile"
   val uniqueName = s"${baseName}_$tileId"
-  def instantiate(crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
+  def instantiate(crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters): RocketTile = {
     new RocketTile(this, crossing, lookup)
   }
 }
@@ -50,7 +50,7 @@ class RocketTile private(
     with HasICacheFrontend
 {
   // Private constructor ensures altered LazyModule.p is used implicitly
-  def this(params: RocketTileParams, crossing: ElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
+  def this(params: RocketTileParams, crossing: HierarchicalElementCrossingParamsLike, lookup: LookupByHartIdImpl)(implicit p: Parameters) =
     this(params, crossing.crossingType, lookup, p)
 
   val intOutwardNode = rocketParams.beuAddr map { _ => IntIdentityNode() }
diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index 66f2afc4d4c..f2996e81fa9 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -23,8 +23,8 @@ import freechips.rocketchip.util.{TraceCoreInterface}
   */
 abstract class TilePRCIDomain[T <: BaseTile](
   clockSinkParams: ClockSinkParameters,
-  crossingParams: ElementCrossingParamsLike)
+  crossingParams: HierarchicalElementCrossingParamsLike)
   (implicit p: Parameters)
-    extends ElementPRCIDomain[T](clockSinkParams, crossingParams)
+    extends HierarchicalElementPRCIDomain[T](clockSinkParams, crossingParams)
 {
 }

From 516288f88cc8e76ab436f2352099e169c748db43 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 16 Oct 2023 11:10:30 -0700
Subject: [PATCH 38/47] Switch to SortedMap for tile node maps

---
 src/main/scala/subsystem/Cluster.scala        | 26 ++++----
 .../subsystem/HasHierarchicalElements.scala   | 62 +++++++++----------
 src/main/scala/subsystem/HasTiles.scala       | 14 ++---
 3 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index 4a50d1923f6..3c686340bce 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -13,7 +13,7 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.devices.debug.{TLDebugModule}
 import freechips.rocketchip.devices.tilelink._
 import freechips.rocketchip.util._
-import scala.collection.immutable.ListMap
+import scala.collection.immutable.SortedMap
 
 case class ClustersLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachCluster]](Nil)
 
@@ -58,17 +58,17 @@ class Cluster(
   lazy val ibus = LazyModule(new InterruptBusWrapper)
   ibus.clockNode := csbus.fixedClockNode
 
-  lazy val msipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(ListMap)
-  lazy val meipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(ListMap)
-  lazy val seipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(ListMap)
-  lazy val tileToPlicNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(ListMap)
-  lazy val debugNodes = totalTileIdList.map { i => (i, IntSyncIdentityNode()) }.to(ListMap)
+  lazy val msipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(SortedMap)
+  lazy val meipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(SortedMap)
+  lazy val seipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(SortedMap)
+  lazy val tileToPlicNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(SortedMap)
+  lazy val debugNodes = totalTileIdList.map { i => (i, IntSyncIdentityNode()) }.to(SortedMap)
   lazy val nmiNodes = totalTiles.filter { case (i,t) => t.tileParams.core.useNMI }
-    .mapValues(_ => BundleBridgeIdentityNode[NMI]()).to(ListMap)
-  lazy val tileHartIdNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.to(ListMap)
-  lazy val tileResetVectorNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.to(ListMap)
-  lazy val traceCoreNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceCoreInterface]()) }.to(ListMap)
-  lazy val traceNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceBundle]()) }.to(ListMap)
+    .mapValues(_ => BundleBridgeIdentityNode[NMI]()).to(SortedMap)
+  lazy val tileHartIdNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.to(SortedMap)
+  lazy val tileResetVectorNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[UInt]()) }.to(SortedMap)
+  lazy val traceCoreNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceCoreInterface]()) }.to(SortedMap)
+  lazy val traceNodes = totalTileIdList.map { i => (i, BundleBridgeIdentityNode[TraceBundle]()) }.to(SortedMap)
 
   // TODO fix: shouldn't need to connect dummy notifications
   tileHaltXbarNode := NullIntSource()
@@ -103,7 +103,7 @@ trait CanAttachCluster {
   def clusterParams: ClusterParams
   def crossingParams: HierarchicalElementCrossingParamsLike
 
-  def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: ListMap[Int, ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
+  def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: SortedMap[Int, ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
     val clockSinkParams = clusterParams.clockSinkParams.copy(name = Some(clusterParams.uniqueName))
     val cluster_prci_domain = LazyModule(new ClusterPRCIDomain(
       clockSinkParams, crossingParams, clusterParams, PriorityMuxClusterIdFromSeq(allClusterParams)))
@@ -218,7 +218,7 @@ case class CloneClusterAttachParams(
   def clusterParams = cloneParams.clusterParams
   def crossingParams = cloneParams.crossingParams
 
-  override def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: ListMap[Int, ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
+  override def instantiate(allClusterParams: Seq[ClusterParams], instantiatedClusters: SortedMap[Int, ClusterPRCIDomain])(implicit p: Parameters): ClusterPRCIDomain = {
     require(instantiatedClusters.contains(sourceClusterId))
     val clockSinkParams = clusterParams.clockSinkParams.copy(name = Some(clusterParams.uniqueName))
     val cluster_prci_domain = CloneLazyModule(
diff --git a/src/main/scala/subsystem/HasHierarchicalElements.scala b/src/main/scala/subsystem/HasHierarchicalElements.scala
index bcc4382da8e..a299962bee6 100644
--- a/src/main/scala/subsystem/HasHierarchicalElements.scala
+++ b/src/main/scala/subsystem/HasHierarchicalElements.scala
@@ -14,7 +14,7 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.prci.{ClockGroup, ResetCrossingType, ClockGroupNode}
 import freechips.rocketchip.util._
 import freechips.rocketchip.rocket.{TracedInstruction}
-import scala.collection.immutable.ListMap
+import scala.collection.immutable.SortedMap
 
 /** A default implementation of parameterizing the connectivity of the port where the tile is the master.
   *   Optional timing buffers and/or an optional CacheCork can be inserted in the interconnect's clock domain.
@@ -67,28 +67,28 @@ trait InstantiatesHierarchicalElements { this: LazyModule with Attachable =>
   val tileCrossingTypes: Seq[ClockCrossingType] = tileAttachParams.map(_.crossingParams.crossingType)
 
   /** The actual list of instantiated tiles in this block. */
-  val tile_prci_domains: ListMap[Int, TilePRCIDomain[_]] = tileAttachParams.foldLeft(ListMap[Int, TilePRCIDomain[_]]()) {
+  val tile_prci_domains: SortedMap[Int, TilePRCIDomain[_]] = tileAttachParams.foldLeft(SortedMap[Int, TilePRCIDomain[_]]()) {
     case (instantiated, params) => instantiated + (params.tileParams.tileId -> params.instantiate(tileParams, instantiated)(p))
   }
 
   val clusterAttachParams: Seq[CanAttachCluster] = p(ClustersLocated(location)).sortBy(_.clusterParams.clusterId)
   val clusterParams: Seq[ClusterParams] = clusterAttachParams.map(_.clusterParams)
   val clusterCrossingTypes: Seq[ClockCrossingType] = clusterAttachParams.map(_.crossingParams.crossingType)
-  val cluster_prci_domains: ListMap[Int, ClusterPRCIDomain] = clusterAttachParams.foldLeft(ListMap[Int, ClusterPRCIDomain]()) {
+  val cluster_prci_domains: SortedMap[Int, ClusterPRCIDomain] = clusterAttachParams.foldLeft(SortedMap[Int, ClusterPRCIDomain]()) {
     case (instantiated, params) => instantiated + (params.clusterParams.clusterId -> params.instantiate(clusterParams, instantiated)(p))
   }
 
   val element_prci_domains: Seq[HierarchicalElementPRCIDomain[_]] = tile_prci_domains.values.toSeq ++ cluster_prci_domains.values.toSeq
 
-  val leafTiles: ListMap[Int, BaseTile] = tile_prci_domains.mapValues(_.element.asInstanceOf[BaseTile]).to(ListMap)
-  val totalTiles: ListMap[Int, BaseTile] = (leafTiles ++ cluster_prci_domains.values.map(_.element.totalTiles).flatten)
+  val leafTiles: SortedMap[Int, BaseTile] = SortedMap(tile_prci_domains.mapValues(_.element.asInstanceOf[BaseTile]).toSeq.sortBy(_._1):_*)
+  val totalTiles: SortedMap[Int, BaseTile] = (leafTiles ++ cluster_prci_domains.values.map(_.element.totalTiles).flatten)
 
   // Helper functions for accessing certain parameters that are popular to refer to in subsystem code
   def nLeafTiles: Int = leafTiles.size
   def nTotalTiles: Int = totalTiles.size
   def leafTileIdList: Seq[Int] = leafTiles.keys.toSeq.sorted
   def totalTileIdList: Seq[Int] = totalTiles.keys.toSeq.sorted
-  def localIntCounts: ListMap[Int, Int] = totalTiles.mapValues(_.tileParams.core.nLocalInterrupts).to(ListMap)
+  def localIntCounts: SortedMap[Int, Int] = totalTiles.mapValues(_.tileParams.core.nLocalInterrupts).to(SortedMap)
 
   require(totalTileIdList.distinct.size == totalTiles.size, s"Every tile must be statically assigned a unique id, but got:\n${totalTileIdList}")
 }
@@ -142,16 +142,16 @@ trait DefaultHierarchicalElementContextType
     extends Attachable
     with HasTileNotificationSinks
 { this: LazyModule with Attachable =>
-  val msipNodes: ListMap[Int, IntNode]
-  val meipNodes: ListMap[Int, IntNode]
-  val seipNodes: ListMap[Int, IntNode]
-  val tileToPlicNodes: ListMap[Int, IntNode]
-  val debugNodes: ListMap[Int, IntSyncNode]
-  val nmiNodes: ListMap[Int, BundleBridgeNode[NMI]]
-  val tileHartIdNodes: ListMap[Int, BundleBridgeNode[UInt]]
-  val tileResetVectorNodes: ListMap[Int, BundleBridgeNode[UInt]]
-  val traceCoreNodes: ListMap[Int, BundleBridgeNode[TraceCoreInterface]]
-  val traceNodes: ListMap[Int, BundleBridgeNode[TraceBundle]]
+  val msipNodes: SortedMap[Int, IntNode]
+  val meipNodes: SortedMap[Int, IntNode]
+  val seipNodes: SortedMap[Int, IntNode]
+  val tileToPlicNodes: SortedMap[Int, IntNode]
+  val debugNodes: SortedMap[Int, IntSyncNode]
+  val nmiNodes: SortedMap[Int, BundleBridgeNode[NMI]]
+  val tileHartIdNodes: SortedMap[Int, BundleBridgeNode[UInt]]
+  val tileResetVectorNodes: SortedMap[Int, BundleBridgeNode[UInt]]
+  val traceCoreNodes: SortedMap[Int, BundleBridgeNode[TraceCoreInterface]]
+  val traceNodes: SortedMap[Int, BundleBridgeNode[TraceBundle]]
 }
 
 /** This trait provides the tile attachment context for the root (outermost) subsystem */
@@ -164,9 +164,9 @@ trait HasHierarchicalElementsRootContext
   val plicOpt: Option[TLPLIC]
   val debugOpt: Option[TLDebugModule]
 
-  val msipNodes: ListMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
+  val msipNodes: SortedMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode())
-  }.to(ListMap)
+  }.to(SortedMap)
   msipNodes.foreach {
     _._2 := clintOpt.map(_.intnode).getOrElse(NullIntSource(sources = CLINTConsts.ints))
   }
@@ -176,36 +176,36 @@ trait HasHierarchicalElementsRootContext
     sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
     outputRequiresInput = false,
     inputRequiresOutput = false))
-  val meipNodes: ListMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
+  val meipNodes: SortedMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(meipIONode.get))
-  }.to(ListMap)
+  }.to(SortedMap)
 
   val seipIONode = Option.when(plicOpt.isEmpty)(IntNexusNode(
     sourceFn = { _ => IntSourcePortParameters(Seq(IntSourceParameters(1))) },
     sinkFn   = { _ => IntSinkPortParameters(Seq(IntSinkParameters())) },
     outputRequiresInput = false,
     inputRequiresOutput = false))
-  val seipNodes: ListMap[Int, IntNode] = totalTiles.filter { case (_, t) => t.tileParams.core.hasSupervisorMode }
-    .mapValues( _ => IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(seipIONode.get)).to(ListMap)
+  val seipNodes: SortedMap[Int, IntNode] = totalTiles.filter { case (_, t) => t.tileParams.core.hasSupervisorMode }
+    .mapValues( _ => IntEphemeralNode() := plicOpt.map(_.intnode).getOrElse(seipIONode.get)).to(SortedMap)
 
-  val tileToPlicNodes: ListMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
+  val tileToPlicNodes: SortedMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
     plicOpt.map(o => (i, o.intnode :=* IntEphemeralNode()))
-  }.flatten.to(ListMap)
+  }.flatten.to(SortedMap)
 
-  val debugNodes: ListMap[Int, IntSyncNode] = (0 until nTotalTiles).map { i =>
+  val debugNodes: SortedMap[Int, IntSyncNode] = (0 until nTotalTiles).map { i =>
     (i, IntSyncIdentityNode())
-  }.to(ListMap)
+  }.to(SortedMap)
 
   debugNodes.foreach { case (hartid, node) =>
     node := debugOpt.map(_.intnode).getOrElse(IntSyncCrossingSource() := NullIntSource())
   }
 
   val nmiHarts = totalTiles.filter { case (_, t) => t.tileParams.core.useNMI }.keys
-  val nmiIONodes = nmiHarts.map { i => (i, BundleBridgeSource[NMI]()) }.to(ListMap)
-  val nmiNodes: ListMap[Int, BundleBridgeNode[NMI]] = nmiIONodes.map { case (i, n) =>
+  val nmiIONodes = nmiHarts.map { i => (i, BundleBridgeSource[NMI]()) }.to(SortedMap)
+  val nmiNodes: SortedMap[Int, BundleBridgeNode[NMI]] = nmiIONodes.map { case (i, n) =>
     (i, BundleBridgeEphemeralNode[NMI]() := n)
-  }.to(ListMap)
+  }.to(SortedMap)
 
-  val traceCoreNodes: ListMap[Int, BundleBridgeSink[TraceCoreInterface]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceCoreInterface]()) }.to(ListMap)
-  val traceNodes: ListMap[Int, BundleBridgeSink[TraceBundle]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceBundle]()) }.to(ListMap)
+  val traceCoreNodes: SortedMap[Int, BundleBridgeSink[TraceCoreInterface]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceCoreInterface]()) }.to(SortedMap)
+  val traceNodes: SortedMap[Int, BundleBridgeSink[TraceBundle]] = (0 until nTotalTiles).map { i => (i, BundleBridgeSink[TraceBundle]()) }.to(SortedMap)
 }
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 7f1b04ddec4..8ab6e8d392c 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -14,7 +14,7 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.prci._
 import freechips.rocketchip.util._
 import freechips.rocketchip.rocket.{TracedInstruction}
-import scala.collection.immutable.ListMap
+import scala.collection.immutable.SortedMap
 
 /** Entry point for Config-uring the presence of Tiles */
 case class TilesLocated(loc: HierarchicalLocation) extends Field[Seq[CanAttachTile]](Nil)
@@ -53,9 +53,9 @@ case object HasTilesExternalResetVectorKey extends Field[Boolean](true)
   */
 trait HasTileInputConstants { this: LazyModule with Attachable with InstantiatesHierarchicalElements =>
   /** tileHartIdNode is used to collect publishers and subscribers of hartids. */
-  val tileHartIdNodes: ListMap[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
+  val tileHartIdNodes: SortedMap[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
     (i, BundleBridgeEphemeralNode[UInt]())
-  }.to(ListMap)
+  }.to(SortedMap)
 
   /** tileHartIdNexusNode is a BundleBridgeNexus that collects dynamic hart prefixes.
     *
@@ -82,9 +82,9 @@ trait HasTileInputConstants { this: LazyModule with Attachable with Instantiates
   // TODO: Replace the DebugModuleHartSelFuncs config key with logic to consume the dynamic hart IDs
 
   /** tileResetVectorNode is used to collect publishers and subscribers of tile reset vector addresses. */
-  val tileResetVectorNodes: ListMap[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
+  val tileResetVectorNodes: SortedMap[Int, BundleBridgeEphemeralNode[UInt]] = (0 until nTotalTiles).map { i =>
     (i, BundleBridgeEphemeralNode[UInt]())
-  }.to(ListMap)
+  }.to(SortedMap)
 
   /** tileResetVectorNexusNode is a BundleBridgeNexus that accepts a single reset vector source, and broadcasts it to all tiles. */
   val tileResetVectorNexusNode = BundleBroadcast[UInt](
@@ -155,7 +155,7 @@ trait CanAttachTile {
   def crossingParams: HierarchicalElementCrossingParamsLike
 
   /** Narrow waist through which all tiles are intended to pass while being instantiated. */
-  def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: ListMap[Int, TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
+  def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: SortedMap[Int, TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
     val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.uniqueName))
     val tile_prci_domain = LazyModule(new TilePRCIDomain[TileType](clockSinkParams, crossingParams) { self =>
       val element = self.element_reset_domain { LazyModule(tileParams.instantiate(crossingParams, PriorityMuxHartIdFromSeq(allTileParams))) }
@@ -302,7 +302,7 @@ case class CloneTileAttachParams(
   def tileParams = cloneParams.tileParams
   def crossingParams = cloneParams.crossingParams
 
-  override def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: ListMap[Int, TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
+  override def instantiate(allTileParams: Seq[TileParams], instantiatedTiles: SortedMap[Int, TilePRCIDomain[_]])(implicit p: Parameters): TilePRCIDomain[TileType] = {
     require(instantiatedTiles.contains(sourceTileId))
     val clockSinkParams = tileParams.clockSinkParams.copy(name = Some(tileParams.uniqueName))
     val tile_prci_domain = CloneLazyModule(

From fd51b74c4542944b56713566a148e9f71619e084 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 16 Oct 2023 11:10:52 -0700
Subject: [PATCH 39/47] Add tile_reset_domain alias to element_reset_domain in
 TilePRCIDomain

---
 src/main/scala/tile/TilePRCIDomain.scala | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/scala/tile/TilePRCIDomain.scala b/src/main/scala/tile/TilePRCIDomain.scala
index f2996e81fa9..d4066a63473 100644
--- a/src/main/scala/tile/TilePRCIDomain.scala
+++ b/src/main/scala/tile/TilePRCIDomain.scala
@@ -27,4 +27,5 @@ abstract class TilePRCIDomain[T <: BaseTile](
   (implicit p: Parameters)
     extends HierarchicalElementPRCIDomain[T](clockSinkParams, crossingParams)
 {
+  def tile_reset_domain = element_reset_domain
 }

From f745cff4358395c02a5ee3fd4075f2ff39a5ad15 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 17 Oct 2023 01:08:16 -0700
Subject: [PATCH 40/47] Switch to separate cluster-local bus locations

---
 src/main/scala/subsystem/BusTopology.scala | 16 ++++---
 src/main/scala/subsystem/Cluster.scala     |  6 +--
 src/main/scala/subsystem/Configs.scala     | 51 +++++++++++++++++-----
 src/main/scala/system/Configs.scala        |  6 +--
 4 files changed, 56 insertions(+), 23 deletions(-)

diff --git a/src/main/scala/subsystem/BusTopology.scala b/src/main/scala/subsystem/BusTopology.scala
index dc8443ae220..a9d77b5b127 100644
--- a/src/main/scala/subsystem/BusTopology.scala
+++ b/src/main/scala/subsystem/BusTopology.scala
@@ -29,6 +29,10 @@ case object FBUS extends TLBusWrapperLocation("fbus")
 case object MBUS extends TLBusWrapperLocation("mbus")
 case object CBUS extends TLBusWrapperLocation("cbus")
 case object COH  extends TLBusWrapperLocation("coh")
+case class CSBUS(clusterId: Int) extends TLBusWrapperLocation(s"csbus$clusterId")
+case class CMBUS(clusterId: Int) extends TLBusWrapperLocation(s"cmbus$clusterId")
+case class CCBUS(clusterId: Int) extends TLBusWrapperLocation(s"ccbus$clusterId")
+case class CCOH (clusterId: Int) extends TLBusWrapperLocation(s"ccoh$clusterId")
 
 /** Parameterizes the subsystem in terms of optional clock-crossings
   *   that are insertable between some of the five traditional tilelink bus wrappers.
@@ -113,13 +117,13 @@ case class ClusterBusTopologyParams(
   coherence: BankedCoherenceParams
 ) extends TLBusWrapperTopology(
   instantiations = List(
-    (SBUS, csbus),
-    (CBUS, ccbus)) ++ (if (coherence.nBanks == 0) Nil else List(
-    (MBUS, csbus),
-    (COH , CoherenceManagerWrapperParams(csbus.blockBytes, csbus.beatBytes, coherence.nBanks, COH.name)(coherence.coherenceManager)))),
+    (CSBUS(clusterId), csbus),
+    (CCBUS(clusterId), ccbus)) ++ (if (coherence.nBanks == 0) Nil else List(
+    (CMBUS(clusterId), csbus),
+    (CCOH (clusterId), CoherenceManagerWrapperParams(csbus.blockBytes, csbus.beatBytes, coherence.nBanks, CCOH(clusterId).name)(coherence.coherenceManager)))),
   connections = if (coherence.nBanks == 0) Nil else List(
-    (SBUS, COH , TLBusWrapperConnection(driveClockFromMaster = Some(true), nodeBinding = BIND_STAR)()),
-    (COH , MBUS, TLBusWrapperConnection.crossTo(
+    (CSBUS(clusterId), CCOH (clusterId), TLBusWrapperConnection(driveClockFromMaster = Some(true), nodeBinding = BIND_STAR)()),
+    (CCOH (clusterId), CMBUS(clusterId), TLBusWrapperConnection.crossTo(
       xType = NoCrossing,
       driveClockFromMaster = Some(true),
       nodeBinding = BIND_QUERY))
diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index 3c686340bce..6b827f00c0b 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -43,9 +43,9 @@ class Cluster(
 
   lazy val allClockGroupsNode = ClockGroupIdentityNode()
 
-  val csbus = tlBusWrapperLocationMap(SBUS) // like the sbus in the base subsystem
-  val ccbus = tlBusWrapperLocationMap(CBUS) // like the cbus in the base subsystem
-  val cmbus = tlBusWrapperLocationMap.lift(MBUS).getOrElse(csbus)
+  val csbus = tlBusWrapperLocationMap(CSBUS(clusterId)) // like the sbus in the base subsystem
+  val ccbus = tlBusWrapperLocationMap(CCBUS(clusterId)) // like the cbus in the base subsystem
+  val cmbus = tlBusWrapperLocationMap.lift(CMBUS(clusterId)).getOrElse(csbus)
 
   csbus.clockGroupNode := allClockGroupsNode
   ccbus.clockGroupNode := allClockGroupsNode
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index fac37a0dd28..b9e234b536f 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -90,11 +90,10 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
 
 class WithNBigCores(
   n: Int,
-  crossing: RocketCrossingParams = RocketCrossingParams(),
-  location: HierarchicalLocation = InSubsystem
+  crossing: RocketCrossingParams = RocketCrossingParams()
 ) extends Config((site, here, up) => {
-  case TilesLocated(`location`) => {
-    val prev = up(TilesLocated(location), site)
+  case TilesLocated(InSubsystem) => {
+    val prev = up(TilesLocated(InSubsystem), site)
     val idOffset = up(NumTiles)
     val big = RocketTileParams(
       core   = RocketCoreParams(mulDiv = Some(MulDivParams(
@@ -119,10 +118,9 @@ class WithNBigCores(
 class WithNMedCores(
   n: Int,
   crossing: RocketCrossingParams = RocketCrossingParams(),
-  location: HierarchicalLocation = InSubsystem
 ) extends Config((site, here, up) => {
-  case TilesLocated(`location`) => {
-    val prev = up(TilesLocated(location), site)
+  case TilesLocated(InSubsystem) => {
+    val prev = up(TilesLocated(InSubsystem), site)
     val idOffset = up(NumTiles)
     val med = RocketTileParams(
       core = RocketCoreParams(fpu = None),
@@ -152,11 +150,10 @@ class WithNMedCores(
 
 class WithNSmallCores(
   n: Int,
-  crossing: RocketCrossingParams = RocketCrossingParams(),
-  location: HierarchicalLocation = InSubsystem
+  crossing: RocketCrossingParams = RocketCrossingParams()
 ) extends Config((site, here, up) => {
-  case TilesLocated(`location`) => {
-    val prev = up(TilesLocated(location), site)
+  case TilesLocated(InSubsystem) => {
+    val prev = up(TilesLocated(InSubsystem), site)
     val idOffset = up(NumTiles)
     val small = RocketTileParams(
       core = RocketCoreParams(useVM = false, fpu = None),
@@ -221,6 +218,38 @@ class With1TinyCore extends Config((site, here, up) => {
   case ClustersLocated(_) => Nil
 })
 
+class WithNClusterCores(
+  n: Int,
+  clusterId: Int
+) extends Config((site, here, up) => {
+  case TilesLocated(InCluster(`clusterId`)) => {
+    val prev = up(TilesLocated(InCluster(clusterId)), site)
+    val idOffset = up(NumTiles)
+    val big = RocketTileParams(
+      core   = RocketCoreParams(mulDiv = Some(MulDivParams(
+        mulUnroll = 8,
+        mulEarlyOut = true,
+        divEarlyOut = true))),
+      dcache = Some(DCacheParams(
+        rowBits = site(SystemBusKey).beatBits,
+        nMSHRs = 0,
+        blockBytes = site(CacheBlockBytes))),
+      icache = Some(ICacheParams(
+        rowBits = site(SystemBusKey).beatBits,
+        blockBytes = site(CacheBlockBytes))))
+    List.tabulate(n)(i => RocketTileAttachParams(
+      big.copy(tileId = i + idOffset),
+      RocketCrossingParams(
+        master = HierarchicalElementMasterPortParams(where=CSBUS(clusterId)),
+        slave = HierarchicalElementSlavePortParams(where=CCBUS(clusterId), blockerCtrlWhere=CCBUS(clusterId)),
+        mmioBaseAddressPrefixWhere = CCBUS(clusterId)
+      )
+    )) ++ prev
+  }
+  case NumTiles => up(NumTiles) + n
+})
+
+
 class WithCluster(
   clusterId: Int,
   location: HierarchicalLocation = InSubsystem,
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index 68136f571c9..c4220a169b7 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -38,9 +38,9 @@ class DualChannelConfig extends Config(new WithNMemoryChannels(2) ++ new Default
 class EightChannelConfig extends Config(new WithNMemoryChannels(8) ++ new DefaultConfig)
 
 class ClusterConfig extends Config(
-  new WithNBigCores(2, location=InCluster(3)) ++
-  new WithNBigCores(2, location=InCluster(1)) ++
-  new WithNBigCores(2, location=InCluster(0)) ++
+  new WithNClusterCores(2, clusterId=3) ++
+  new WithNClusterCores(2, clusterId=1) ++
+  new WithNClusterCores(2, clusterId=0) ++
   new WithCluster(3, location=InCluster(2)) ++
   new WithCluster(2) ++
   new WithCluster(1) ++

From ef1d671966dab8199723c6eb8d3cc09c8a7f2320 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 17 Oct 2023 13:40:56 -0700
Subject: [PATCH 41/47] Generalize WithNBigCores to support clustered cores

---
 src/main/scala/subsystem/Configs.scala        | 52 ++++++-------------
 .../subsystem/HasHierarchicalElements.scala   | 14 +++++
 src/main/scala/system/Configs.scala           |  6 +--
 3 files changed, 33 insertions(+), 39 deletions(-)

diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index b9e234b536f..60fc67a4a07 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -90,10 +90,11 @@ class WithCoherentBusTopology extends Config((site, here, up) => {
 
 class WithNBigCores(
   n: Int,
-  crossing: RocketCrossingParams = RocketCrossingParams()
+  location: HierarchicalLocation,
+  crossing: RocketCrossingParams,
 ) extends Config((site, here, up) => {
-  case TilesLocated(InSubsystem) => {
-    val prev = up(TilesLocated(InSubsystem), site)
+  case TilesLocated(`location`) => {
+    val prev = up(TilesLocated(`location`), site)
     val idOffset = up(NumTiles)
     val big = RocketTileParams(
       core   = RocketCoreParams(mulDiv = Some(MulDivParams(
@@ -113,7 +114,18 @@ class WithNBigCores(
     )) ++ prev
   }
   case NumTiles => up(NumTiles) + n
-})
+}) {
+  def this(n: Int, location: HierarchicalLocation = InSubsystem) = this(n, location, RocketCrossingParams(
+    master = HierarchicalElementMasterPortParams.locationDefault(location),
+    slave = HierarchicalElementSlavePortParams.locationDefault(location),
+    mmioBaseAddressPrefixWhere = location match {
+      case InSubsystem => CBUS
+      case InCluster(clusterId) => CCBUS(clusterId)
+    }
+  ))
+}
+
+
 
 class WithNMedCores(
   n: Int,
@@ -218,38 +230,6 @@ class With1TinyCore extends Config((site, here, up) => {
   case ClustersLocated(_) => Nil
 })
 
-class WithNClusterCores(
-  n: Int,
-  clusterId: Int
-) extends Config((site, here, up) => {
-  case TilesLocated(InCluster(`clusterId`)) => {
-    val prev = up(TilesLocated(InCluster(clusterId)), site)
-    val idOffset = up(NumTiles)
-    val big = RocketTileParams(
-      core   = RocketCoreParams(mulDiv = Some(MulDivParams(
-        mulUnroll = 8,
-        mulEarlyOut = true,
-        divEarlyOut = true))),
-      dcache = Some(DCacheParams(
-        rowBits = site(SystemBusKey).beatBits,
-        nMSHRs = 0,
-        blockBytes = site(CacheBlockBytes))),
-      icache = Some(ICacheParams(
-        rowBits = site(SystemBusKey).beatBits,
-        blockBytes = site(CacheBlockBytes))))
-    List.tabulate(n)(i => RocketTileAttachParams(
-      big.copy(tileId = i + idOffset),
-      RocketCrossingParams(
-        master = HierarchicalElementMasterPortParams(where=CSBUS(clusterId)),
-        slave = HierarchicalElementSlavePortParams(where=CCBUS(clusterId), blockerCtrlWhere=CCBUS(clusterId)),
-        mmioBaseAddressPrefixWhere = CCBUS(clusterId)
-      )
-    )) ++ prev
-  }
-  case NumTiles => up(NumTiles) + n
-})
-
-
 class WithCluster(
   clusterId: Int,
   location: HierarchicalLocation = InSubsystem,
diff --git a/src/main/scala/subsystem/HasHierarchicalElements.scala b/src/main/scala/subsystem/HasHierarchicalElements.scala
index a299962bee6..8fb271bae3a 100644
--- a/src/main/scala/subsystem/HasHierarchicalElements.scala
+++ b/src/main/scala/subsystem/HasHierarchicalElements.scala
@@ -29,6 +29,13 @@ case class HierarchicalElementMasterPortParams(
   }
 }
 
+object HierarchicalElementMasterPortParams {
+  def locationDefault(loc: HierarchicalLocation) = loc match {
+    case InSubsystem => HierarchicalElementMasterPortParams()
+    case InCluster(clusterId) => HierarchicalElementMasterPortParams(where=CSBUS(clusterId))
+  }
+}
+
 /** A default implementation of parameterizing the connectivity of the port giving access to slaves inside the tile.
   *   Optional timing buffers and/or an optional BusBlocker adapter can be inserted in the interconnect's clock domain.
   */
@@ -51,6 +58,13 @@ case class HierarchicalElementSlavePortParams(
   }
 }
 
+object HierarchicalElementSlavePortParams {
+  def locationDefault(loc: HierarchicalLocation) = loc match {
+    case InSubsystem => HierarchicalElementSlavePortParams()
+    case InCluster(clusterId) => HierarchicalElementSlavePortParams(where=CCBUS(clusterId), blockerCtrlWhere=CCBUS(clusterId))
+  }
+}
+
 /** InstantiatesTiles adds a Config-urable sequence of HierarchicalElements of any type
   *   to the subsystem class into which it is mixed.
   */
diff --git a/src/main/scala/system/Configs.scala b/src/main/scala/system/Configs.scala
index c4220a169b7..96f9aba4e8f 100644
--- a/src/main/scala/system/Configs.scala
+++ b/src/main/scala/system/Configs.scala
@@ -38,9 +38,9 @@ class DualChannelConfig extends Config(new WithNMemoryChannels(2) ++ new Default
 class EightChannelConfig extends Config(new WithNMemoryChannels(8) ++ new DefaultConfig)
 
 class ClusterConfig extends Config(
-  new WithNClusterCores(2, clusterId=3) ++
-  new WithNClusterCores(2, clusterId=1) ++
-  new WithNClusterCores(2, clusterId=0) ++
+  new WithNBigCores(2, InCluster(3)) ++
+  new WithNBigCores(2, InCluster(1)) ++
+  new WithNBigCores(2, InCluster(0)) ++
   new WithCluster(3, location=InCluster(2)) ++
   new WithCluster(2) ++
   new WithCluster(1) ++

From e0ea90344e9edb6a4e24f84e7729d83c217c8859 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 17 Oct 2023 13:41:25 -0700
Subject: [PATCH 42/47] Rename trace crossing nodes to crossing nodes

---
 src/main/scala/subsystem/HasTiles.scala | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 8ab6e8d392c..216ef979a45 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -283,12 +283,12 @@ trait CanAttachTile {
   /** Function to handle all trace crossings when tile is instantiated inside domains */
   def connectTrace(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
     implicit val p = context.p
-    val traceNexusNode = BundleBridgeBlockDuringReset[TraceBundle](
+    val traceCrossingNode = BundleBridgeBlockDuringReset[TraceBundle](
       resetCrossingType = crossingParams.resetCrossingType)
-    context.traceNodes(domain.element.tileId) := traceNexusNode := domain.element.traceNode
-    val traceCoreNexusNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
+    context.traceNodes(domain.element.tileId) := traceCrossingNode := domain.element.traceNode
+    val traceCoreCrossingNode = BundleBridgeBlockDuringReset[TraceCoreInterface](
       resetCrossingType = crossingParams.resetCrossingType)
-    context.traceCoreNodes(domain.element.tileId) :*= traceCoreNexusNode := domain.element.traceCoreNode
+    context.traceCoreNodes(domain.element.tileId) :*= traceCoreCrossingNode := domain.element.traceCoreNode
   }
 }
 

From d48b45da568c0d370479325258018a8a5cf3369c Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 18 Oct 2023 00:26:13 -0700
Subject: [PATCH 43/47] Fix int clock crossings to use CLINT/PLIC domains

---
 src/main/scala/devices/tilelink/CLINT.scala   |  1 -
 src/main/scala/devices/tilelink/Plic.scala    |  1 -
 src/main/scala/subsystem/Cluster.scala        | 15 ++++++++------
 .../subsystem/HasHierarchicalElements.scala   |  7 +++++++
 src/main/scala/subsystem/HasTiles.scala       | 20 ++++++++++++-------
 5 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/src/main/scala/devices/tilelink/CLINT.scala b/src/main/scala/devices/tilelink/CLINT.scala
index 7ca4405f4a9..8156706ee8d 100644
--- a/src/main/scala/devices/tilelink/CLINT.scala
+++ b/src/main/scala/devices/tilelink/CLINT.scala
@@ -116,5 +116,4 @@ trait CanHavePeripheryCLINT { this: BaseSubsystem =>
     clint
 
   }
-  val clintNode = clintOpt.map(_.intnode)
 }
diff --git a/src/main/scala/devices/tilelink/Plic.scala b/src/main/scala/devices/tilelink/Plic.scala
index 539f076ea11..3e6de570423 100644
--- a/src/main/scala/devices/tilelink/Plic.scala
+++ b/src/main/scala/devices/tilelink/Plic.scala
@@ -366,5 +366,4 @@ trait CanHavePeripheryPLIC { this: BaseSubsystem =>
 
     plic
   }
-  val plicNode = plicOpt.map(_.intnode)
 }
diff --git a/src/main/scala/subsystem/Cluster.scala b/src/main/scala/subsystem/Cluster.scala
index 6b827f00c0b..7e2ca886a17 100644
--- a/src/main/scala/subsystem/Cluster.scala
+++ b/src/main/scala/subsystem/Cluster.scala
@@ -58,6 +58,9 @@ class Cluster(
   lazy val ibus = LazyModule(new InterruptBusWrapper)
   ibus.clockNode := csbus.fixedClockNode
 
+  def msipDomain = this
+  def meipDomain = this
+  def seipDomain = this
   lazy val msipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(SortedMap)
   lazy val meipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(SortedMap)
   lazy val seipNodes = totalTileIdList.map { i => (i, IntIdentityNode()) }.to(SortedMap)
@@ -141,17 +144,17 @@ trait CanAttachCluster {
       node := context.debugNodes(hartid)
     }
 
-    domain.element.msipNodes.foreach { case (hartid, node) =>
+    domain.element.msipNodes.foreach { case (hartid, node) => context.msipDomain {
       domain.crossIntIn(crossingParams.crossingType, node) := context.msipNodes(hartid)
-    }
+    }}
 
-    domain.element.meipNodes.foreach { case (hartid, node) =>
+    domain.element.meipNodes.foreach { case (hartid, node) => context.meipDomain {
       domain.crossIntIn(crossingParams.crossingType, node) := context.meipNodes(hartid)
-    }
+    }}
 
-    domain.element.seipNodes.foreach { case (hartid, node) =>
+    domain.element.seipNodes.foreach { case (hartid, node) => context.seipDomain {
       domain.crossIntIn(crossingParams.crossingType, node) := context.seipNodes(hartid)
-    }
+    }}
 
     domain.element.tileToPlicNodes.foreach { case (hartid, node) =>
       FlipRendering { implicit p =>
diff --git a/src/main/scala/subsystem/HasHierarchicalElements.scala b/src/main/scala/subsystem/HasHierarchicalElements.scala
index 8fb271bae3a..2d856ebb00e 100644
--- a/src/main/scala/subsystem/HasHierarchicalElements.scala
+++ b/src/main/scala/subsystem/HasHierarchicalElements.scala
@@ -156,8 +156,11 @@ trait DefaultHierarchicalElementContextType
     extends Attachable
     with HasTileNotificationSinks
 { this: LazyModule with Attachable =>
+  def msipDomain: LazyScope
   val msipNodes: SortedMap[Int, IntNode]
+  def meipDomain: LazyScope
   val meipNodes: SortedMap[Int, IntNode]
+  def seipDomain: LazyScope
   val seipNodes: SortedMap[Int, IntNode]
   val tileToPlicNodes: SortedMap[Int, IntNode]
   val debugNodes: SortedMap[Int, IntSyncNode]
@@ -178,6 +181,10 @@ trait HasHierarchicalElementsRootContext
   val plicOpt: Option[TLPLIC]
   val debugOpt: Option[TLDebugModule]
 
+  def msipDomain = locateTLBusWrapper(p(CLINTAttachKey).slaveWhere)
+  def meipDomain = locateTLBusWrapper(p(PLICAttachKey).slaveWhere)
+  def seipDomain = locateTLBusWrapper(p(PLICAttachKey).slaveWhere)
+
   val msipNodes: SortedMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode())
   }.to(SortedMap)
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 216ef979a45..467914b9c64 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -205,21 +205,27 @@ trait CanAttachTile {
     domain.element.intInwardNode := domain { IntSyncAsyncCrossingSink(3) } :=
       context.debugNodes(domain.element.tileId)
 
-    // 2. The CLINT and PLIC output interrupts are synchronous to the TileLink bus clock,
+    // 2. The CLINT and PLIC output interrupts are synchronous to the CLINT/PLIC respectively,
     //    so might need to be synchronized depending on the Tile's crossing type.
 
     //    From CLINT: "msip" and "mtip"
-    domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
-      context.msipNodes(domain.element.tileId)
+    context.msipDomain {
+      domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
+        context.msipNodes(domain.element.tileId)
+    }
 
     //    From PLIC: "meip"
-    domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
-      context.meipNodes(domain.element.tileId)
+    context.meipDomain {
+      domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
+        context.meipNodes(domain.element.tileId)
+    }
 
     //    From PLIC: "seip" (only if supervisor mode is enabled)
     if (domain.element.tileParams.core.hasSupervisorMode) {
-      domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
-        context.seipNodes(domain.element.tileId)
+      context.seipDomain {
+        domain.crossIntIn(crossingParams.crossingType, domain.element.intInwardNode) :=
+          context.seipNodes(domain.element.tileId)
+      }
     }
 
     // 3. Local Interrupts ("lip") are required to already be synchronous to the Tile's clock.

From 8881ccd1cab941ed0a0981c00361b1415027f8ce Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 18 Oct 2023 18:55:03 -0700
Subject: [PATCH 44/47] Fix any unassigned clocks due to removal of implicit
 clock from BaseSubsystem

---
 src/main/scala/devices/debug/Custom.scala     |  5 ++-
 src/main/scala/devices/debug/Debug.scala      |  3 ++
 src/main/scala/devices/tilelink/BootROM.scala |  4 +--
 src/main/scala/devices/tilelink/CLINT.scala   | 25 +++++++--------
 src/main/scala/devices/tilelink/Plic.scala    | 14 ++++----
 src/main/scala/diplomacy/BundleBridge.scala   |  2 +-
 src/main/scala/diplomacy/LazyModule.scala     |  6 +++-
 .../groundtest/GroundTestSubsystem.scala      |  2 ++
 src/main/scala/interrupts/Crossing.scala      |  2 +-
 src/main/scala/interrupts/Xbar.scala          |  2 +-
 src/main/scala/prci/ClockDomain.scala         |  1 +
 src/main/scala/subsystem/Configs.scala        |  1 +
 .../subsystem/HasHierarchicalElements.scala   | 10 +++---
 src/main/scala/subsystem/HasTiles.scala       |  8 +++--
 src/main/scala/subsystem/InterruptBus.scala   |  2 +-
 src/main/scala/subsystem/RTC.scala            | 32 ++++++++++---------
 src/main/scala/tilelink/BusWrapper.scala      |  5 +++
 17 files changed, 70 insertions(+), 54 deletions(-)

diff --git a/src/main/scala/devices/debug/Custom.scala b/src/main/scala/devices/debug/Custom.scala
index 22086988a28..64e3f944ca0 100644
--- a/src/main/scala/devices/debug/Custom.scala
+++ b/src/main/scala/devices/debug/Custom.scala
@@ -5,8 +5,7 @@ package freechips.rocketchip.devices.debug
 import chisel3._
 import chisel3.util._
 import chisel3.experimental.SourceInfo
-import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, NexusNode, RenderedEdge,
-  SimpleNodeImp, SinkNode, SourceNode, ValName}
+import freechips.rocketchip.diplomacy._
 import org.chipsalliance.cde.config.Parameters
 
 case class DebugCustomParams(
@@ -68,7 +67,7 @@ class DebugCustomXbar(
   )
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     // require only one sink
     require(node.out.size == 1, "Must have exactly one sink node, not ${node.out.size}")
     // send address to all sources
diff --git a/src/main/scala/devices/debug/Debug.scala b/src/main/scala/devices/debug/Debug.scala
index c1ea7c7f16e..d8f8371fd79 100755
--- a/src/main/scala/devices/debug/Debug.scala
+++ b/src/main/scala/devices/debug/Debug.scala
@@ -728,6 +728,7 @@ class TLDebugModuleOuterAsync(device: Device)(implicit p: Parameters) extends La
 
     childClock := io.dmi_clock
     childReset := io.dmi_reset
+    override def provideImplicitClockToLazyChildren = true
 
     withClockAndReset(childClock, childReset) {
       dmi2tlOpt.foreach { _.module.io.dmi <> io.dmi.get }
@@ -1899,6 +1900,7 @@ class TLDebugModuleInnerAsync(device: Device, getNComponents: () => Int, beatByt
 
     childClock := io.debug_clock
     childReset := io.debug_reset
+    override def provideImplicitClockToLazyChildren = true
 
     val dmactive_synced = withClockAndReset(childClock, childReset) {
       val dmactive_synced = AsyncResetSynchronizerShiftReg(in=io.dmactive, sync=3, name=Some("dmactiveSync"))
@@ -1987,6 +1989,7 @@ class TLDebugModule(beatBytes: Int)(implicit p: Parameters) extends LazyModule {
 
     childClock := io.tl_clock
     childReset := io.tl_reset
+    override def provideImplicitClockToLazyChildren = true
 
     dmOuter.module.io.dmi.foreach { dmOuterDMI =>
       dmOuterDMI <> io.dmi.get.dmi
diff --git a/src/main/scala/devices/tilelink/BootROM.scala b/src/main/scala/devices/tilelink/BootROM.scala
index 22ab348d752..2bbcb16be97 100644
--- a/src/main/scala/devices/tilelink/BootROM.scala
+++ b/src/main/scala/devices/tilelink/BootROM.scala
@@ -8,7 +8,6 @@ import org.chipsalliance.cde.config.{Field, Parameters}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.prci.{ClockSinkDomain}
 
 import java.nio.ByteBuffer
 import java.nio.file.{Files, Paths}
@@ -69,8 +68,7 @@ object BootROM {
   def attach(params: BootROMParams, subsystem: BaseSubsystem with HasHierarchicalElements with HasTileInputConstants, where: TLBusWrapperLocation)
             (implicit p: Parameters): TLROM = {
     val tlbus = subsystem.locateTLBusWrapper(where)
-    val bootROMDomainWrapper = LazyModule(new ClockSinkDomain(take = None))
-    bootROMDomainWrapper.clockNode := tlbus.fixedClockNode
+    val bootROMDomainWrapper = tlbus.generateSynchronousDomain.suggestName("bootrom_domain")
 
     val bootROMResetVectorSourceNode = BundleBridgeSource[UInt]()
     lazy val contents = {
diff --git a/src/main/scala/devices/tilelink/CLINT.scala b/src/main/scala/devices/tilelink/CLINT.scala
index 8156706ee8d..d83bde22899 100644
--- a/src/main/scala/devices/tilelink/CLINT.scala
+++ b/src/main/scala/devices/tilelink/CLINT.scala
@@ -102,18 +102,17 @@ class CLINT(params: CLINTParams, beatBytes: Int)(implicit p: Parameters) extends
 
 /** Trait that will connect a CLINT to a subsystem */
 trait CanHavePeripheryCLINT { this: BaseSubsystem =>
-  val clintOpt = p(CLINTKey).map { params =>
+  val (clintOpt, clintDomainOpt, clintTickOpt) = p(CLINTKey).map { params =>
     val tlbus = locateTLBusWrapper(p(CLINTAttachKey).slaveWhere)
-    val clint = LazyModule(new CLINT(params, cbus.beatBytes))
-    clint.node := tlbus.coupleTo("clint") { TLFragmenter(tlbus) := _ }
-
-    // Override the implicit clock and reset -- could instead include a clockNode in the clint, and make it a RawModuleImp?
-    InModuleBody {
-      clint.module.clock := tlbus.module.clock
-      clint.module.reset := tlbus.module.reset
-    }
-
-    clint
-
-  }
+    val clintDomainWrapper = tlbus.generateSynchronousDomain.suggestName("clint_domain")
+    val clint = clintDomainWrapper { LazyModule(new CLINT(params, cbus.beatBytes)) }
+    clintDomainWrapper { clint.node := tlbus.coupleTo("clint") { TLFragmenter(tlbus) := _ } }
+    val clintTick = clintDomainWrapper { InModuleBody {
+      val tick = IO(Input(Bool()))
+      clint.module.io.rtcTick := tick
+      tick
+    }}
+
+    (clint, clintDomainWrapper, clintTick)
+  }.unzip3
 }
diff --git a/src/main/scala/devices/tilelink/Plic.scala b/src/main/scala/devices/tilelink/Plic.scala
index 3e6de570423..8d87e74f37b 100644
--- a/src/main/scala/devices/tilelink/Plic.scala
+++ b/src/main/scala/devices/tilelink/Plic.scala
@@ -12,7 +12,6 @@ import freechips.rocketchip.tilelink._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.util._
 import freechips.rocketchip.util.property
-import freechips.rocketchip.prci.{ClockSinkDomain}
 import chisel3.experimental.SourceInfo
 
 import scala.math.min
@@ -355,15 +354,14 @@ class PLICFanIn(nDevices: Int, prioBits: Int) extends Module {
 
 /** Trait that will connect a PLIC to a subsystem */
 trait CanHavePeripheryPLIC { this: BaseSubsystem =>
-  val plicOpt  = p(PLICKey).map { params =>
+  val (plicOpt, plicDomainOpt) = p(PLICKey).map { params =>
     val tlbus = locateTLBusWrapper(p(PLICAttachKey).slaveWhere)
-    val plicDomainWrapper = LazyModule(new ClockSinkDomain(take = None))
-    plicDomainWrapper.clockNode := tlbus.fixedClockNode
+    val plicDomainWrapper = tlbus.generateSynchronousDomain
 
     val plic = plicDomainWrapper { LazyModule(new TLPLIC(params, tlbus.beatBytes)) }
-    plic.node := tlbus.coupleTo("plic") { TLFragmenter(tlbus) := _ }
-    plic.intnode :=* ibus.toPLIC
+    plicDomainWrapper { plic.node := tlbus.coupleTo("plic") { TLFragmenter(tlbus) := _ } }
+    plicDomainWrapper { plic.intnode :=* ibus.toPLIC }
 
-    plic
-  }
+    (plic, plicDomainWrapper)
+  }.unzip
 }
diff --git a/src/main/scala/diplomacy/BundleBridge.scala b/src/main/scala/diplomacy/BundleBridge.scala
index e02d6f4f966..f8a70371a6c 100644
--- a/src/main/scala/diplomacy/BundleBridge.scala
+++ b/src/main/scala/diplomacy/BundleBridge.scala
@@ -124,7 +124,7 @@ class BundleBridgeNexus[T <: Data](
   val node = BundleBridgeNexusNode[T](default, inputRequiresOutput)
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     val defaultWireOpt = default.map(_())
     val inputs: Seq[T] = node.in.map(_._1)
     inputs.foreach { i => require(DataMirror.checkTypeEquivalence(i, inputs.head),
diff --git a/src/main/scala/diplomacy/LazyModule.scala b/src/main/scala/diplomacy/LazyModule.scala
index a761a841eed..9cb6e0513cd 100644
--- a/src/main/scala/diplomacy/LazyModule.scala
+++ b/src/main/scala/diplomacy/LazyModule.scala
@@ -415,7 +415,11 @@ class LazyRawModuleImp(val wrapper: LazyModule) extends RawModule with LazyModul
   // the default is that these are disabled
   childClock := false.B.asClock
   childReset := chisel3.DontCare
-  val (auto, dangles) = withClockAndReset(childClock, childReset) {
+
+  def provideImplicitClockToLazyChildren: Boolean = false
+  val (auto, dangles) = if (provideImplicitClockToLazyChildren) {
+    withClockAndReset(childClock, childReset) { instantiate() }
+  } else {
     instantiate()
   }
 }
diff --git a/src/main/scala/groundtest/GroundTestSubsystem.scala b/src/main/scala/groundtest/GroundTestSubsystem.scala
index 0aece29c5bf..379ed58aaca 100644
--- a/src/main/scala/groundtest/GroundTestSubsystem.scala
+++ b/src/main/scala/groundtest/GroundTestSubsystem.scala
@@ -32,8 +32,10 @@ class GroundTestSubsystem(implicit p: Parameters)
 
   val tileStatusNodes = totalTiles.values.collect { case t: GroundTestTile => t.statusNode.makeSink() }
   val clintOpt = None
+  val clintDomainOpt = None
   val debugOpt = None
   val plicOpt = None
+  val plicDomainOpt = None
 
   override lazy val module = new GroundTestSubsystemModuleImp(this)
 }
diff --git a/src/main/scala/interrupts/Crossing.scala b/src/main/scala/interrupts/Crossing.scala
index 77a15480529..366efaaf532 100644
--- a/src/main/scala/interrupts/Crossing.scala
+++ b/src/main/scala/interrupts/Crossing.scala
@@ -85,7 +85,7 @@ class IntSyncSyncCrossingSink()(implicit p: Parameters) extends LazyModule
   val node = IntSyncSinkNode(0)
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     (node.in zip node.out) foreach { case ((in, edgeIn), (out, edgeOut)) =>
       out := in.sync
     }
diff --git a/src/main/scala/interrupts/Xbar.scala b/src/main/scala/interrupts/Xbar.scala
index 472fead6955..b2560106b61 100644
--- a/src/main/scala/interrupts/Xbar.scala
+++ b/src/main/scala/interrupts/Xbar.scala
@@ -19,7 +19,7 @@ class IntXbar()(implicit p: Parameters) extends LazyModule
   }
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     val cat = intnode.in.map { case (i, e) => i.take(e.source.num) }.flatten
     intnode.out.foreach { case (o, _) => o := cat }
   }
diff --git a/src/main/scala/prci/ClockDomain.scala b/src/main/scala/prci/ClockDomain.scala
index 83255d71e2f..7bc10c69470 100644
--- a/src/main/scala/prci/ClockDomain.scala
+++ b/src/main/scala/prci/ClockDomain.scala
@@ -12,6 +12,7 @@ abstract class Domain(implicit p: Parameters) extends LazyModule with HasDomainC
   class Impl extends LazyRawModuleImp(this) {
     childClock := clockBundle.clock
     childReset := clockBundle.reset
+    override def provideImplicitClockToLazyChildren = true
 
     // these are just for backwards compatibility with external devices
     // that were manually wiring themselves to the domain's clock/reset input:
diff --git a/src/main/scala/subsystem/Configs.scala b/src/main/scala/subsystem/Configs.scala
index 60fc67a4a07..dafbd88f268 100644
--- a/src/main/scala/subsystem/Configs.scala
+++ b/src/main/scala/subsystem/Configs.scala
@@ -25,6 +25,7 @@ class BaseSubsystemConfig extends Config ((site, here, up) => {
   case ControlBusKey => PeripheryBusParams(
     beatBytes = site(XLen)/8,
     blockBytes = site(CacheBlockBytes),
+    dtsFrequency = Some(100000000), // Default to 100 MHz cbus clock
     errorDevice = Some(BuiltInErrorDeviceParams(
       errorParams = DevNullParams(List(AddressSet(0x3000, 0xfff)), maxAtomic=site(XLen)/8, maxTransfer=4096))))
   case PeripheryBusKey => PeripheryBusParams(
diff --git a/src/main/scala/subsystem/HasHierarchicalElements.scala b/src/main/scala/subsystem/HasHierarchicalElements.scala
index 2d856ebb00e..fe6ce378463 100644
--- a/src/main/scala/subsystem/HasHierarchicalElements.scala
+++ b/src/main/scala/subsystem/HasHierarchicalElements.scala
@@ -11,7 +11,7 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.interrupts._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
-import freechips.rocketchip.prci.{ClockGroup, ResetCrossingType, ClockGroupNode}
+import freechips.rocketchip.prci.{ClockGroup, ResetCrossingType, ClockGroupNode, ClockDomain}
 import freechips.rocketchip.util._
 import freechips.rocketchip.rocket.{TracedInstruction}
 import scala.collection.immutable.SortedMap
@@ -178,12 +178,14 @@ trait HasHierarchicalElementsRootContext
     with InstantiatesHierarchicalElements =>
 
   val clintOpt: Option[CLINT]
+  val clintDomainOpt: Option[ClockDomain]
   val plicOpt: Option[TLPLIC]
+  val plicDomainOpt: Option[ClockDomain]
   val debugOpt: Option[TLDebugModule]
 
-  def msipDomain = locateTLBusWrapper(p(CLINTAttachKey).slaveWhere)
-  def meipDomain = locateTLBusWrapper(p(PLICAttachKey).slaveWhere)
-  def seipDomain = locateTLBusWrapper(p(PLICAttachKey).slaveWhere)
+  def msipDomain = clintDomainOpt.get
+  def meipDomain = plicDomainOpt.get
+  def seipDomain = plicDomainOpt.get
 
   val msipNodes: SortedMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode())
diff --git a/src/main/scala/subsystem/HasTiles.scala b/src/main/scala/subsystem/HasTiles.scala
index 467914b9c64..ff06ccff4a6 100644
--- a/src/main/scala/subsystem/HasTiles.scala
+++ b/src/main/scala/subsystem/HasTiles.scala
@@ -246,9 +246,11 @@ trait CanAttachTile {
   /** Notifications of tile status are connected to be broadcast without needing to be clock-crossed. */
   def connectOutputNotifications(domain: TilePRCIDomain[TileType], context: TileContextType): Unit = {
     implicit val p = context.p
-    context.tileHaltXbarNode  :=* domain.crossIntOut(NoCrossing, domain.element.haltNode)
-    context.tileWFIXbarNode   :=* domain.crossIntOut(NoCrossing, domain.element.wfiNode)
-    context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.element.ceaseNode)
+    domain {
+      context.tileHaltXbarNode  :=* domain.crossIntOut(NoCrossing, domain.element.haltNode)
+      context.tileWFIXbarNode   :=* domain.crossIntOut(NoCrossing, domain.element.wfiNode)
+      context.tileCeaseXbarNode :=* domain.crossIntOut(NoCrossing, domain.element.ceaseNode)
+    }
     // TODO should context be forced to have a trace sink connected here?
     //      for now this just ensures domain.trace[Core]Node has been crossed without connecting it externally
   }
diff --git a/src/main/scala/subsystem/InterruptBus.scala b/src/main/scala/subsystem/InterruptBus.scala
index 0d20c26a45a..b817b11d856 100644
--- a/src/main/scala/subsystem/InterruptBus.scala
+++ b/src/main/scala/subsystem/InterruptBus.scala
@@ -45,7 +45,7 @@ abstract trait HasExtInterrupts { this: BaseSubsystem =>
   */
 trait HasAsyncExtInterrupts extends HasExtInterrupts { this: BaseSubsystem =>
   if (nExtInterrupts > 0) {
-    ibus.fromAsync := extInterrupts
+    ibus { ibus.fromAsync := extInterrupts }
   }
 }
 
diff --git a/src/main/scala/subsystem/RTC.scala b/src/main/scala/subsystem/RTC.scala
index 522d652037f..2c67d453577 100644
--- a/src/main/scala/subsystem/RTC.scala
+++ b/src/main/scala/subsystem/RTC.scala
@@ -5,25 +5,27 @@ package freechips.rocketchip.subsystem
 import chisel3._
 import chisel3.util.Counter
 import freechips.rocketchip.diplomacy.{LazyRawModuleImp, DTSTimebase}
-import freechips.rocketchip.devices.tilelink.CanHavePeripheryCLINT
+import freechips.rocketchip.devices.tilelink.{CLINTAttachKey, CanHavePeripheryCLINT}
 
 trait HasRTCModuleImp extends LazyRawModuleImp {
   val outer: BaseSubsystem with CanHavePeripheryCLINT
-  private val pbusFreq = outer.p(PeripheryBusKey).dtsFrequency.get
-  private val rtcFreq = outer.p(DTSTimebase)
-  private val internalPeriod: BigInt = pbusFreq / rtcFreq
-
-  val pbus = outer.locateTLBusWrapper(PBUS)
-  // check whether pbusFreq >= rtcFreq
-  require(internalPeriod > 0)
-  // check wehther the integer division is within 5% of the real division
-  require((pbusFreq - rtcFreq * internalPeriod) * 100 / pbusFreq <= 5)
 
   // Use the static period to toggle the RTC
-  chisel3.withClockAndReset(pbus.module.clock, pbus.module.reset) {
-    val (_, int_rtc_tick) = Counter(true.B, internalPeriod.toInt)
-    outer.clintOpt.foreach { clint =>
-      clint.module.io.rtcTick := int_rtc_tick
+  outer.clintDomainOpt.map { domain => {
+    val bus = outer.locateTLBusWrapper(p(CLINTAttachKey).slaveWhere)
+    val busFreq = bus.dtsFrequency.get
+    val rtcFreq = outer.p(DTSTimebase)
+    val internalPeriod: BigInt = busFreq / rtcFreq
+
+
+    // check whether pbusFreq >= rtcFreq
+    require(internalPeriod > 0)
+    // check wehther the integer division is within 5% of the real division
+    require((busFreq - rtcFreq * internalPeriod) * 100 / busFreq <= 5)
+
+    withClockAndReset (domain.module.clock, domain.module.reset) {
+      val (_, int_rtc_tick) = Counter(true.B, internalPeriod.toInt)
+      outer.clintTickOpt.foreach { _ := int_rtc_tick }
     }
-  }
+  }}
 }
diff --git a/src/main/scala/tilelink/BusWrapper.scala b/src/main/scala/tilelink/BusWrapper.scala
index c1888e4e36d..246bba2f365 100644
--- a/src/main/scala/tilelink/BusWrapper.scala
+++ b/src/main/scala/tilelink/BusWrapper.scala
@@ -72,6 +72,11 @@ abstract class TLBusWrapper(params: HasTLBusParams, val busName: String)(implici
   def unifyManagers: List[TLManagerParameters] = ManagerUnification(busView.manager.managers)
   def crossOutHelper = this.crossOut(outwardNode)(ValName("bus_xing"))
   def crossInHelper = this.crossIn(inwardNode)(ValName("bus_xing"))
+  def generateSynchronousDomain = {
+    val domain = LazyModule(new ClockSinkDomain(take = fixedClockOpt))
+    domain.clockNode := fixedClockNode
+    domain
+  }
 
   protected val addressPrefixNexusNode = BundleBroadcast[UInt](registered = false, default = Some(() => 0.U(1.W)))
 

From 0e88fc066e293b0da45da7360afd4cd3e6399678 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 20 Oct 2023 13:58:37 -0700
Subject: [PATCH 45/47] Fix intsynccrossings not receiving a clock

---
 src/main/scala/interrupts/Crossing.scala           | 14 ++++++++------
 src/main/scala/interrupts/NullIntSource.scala      |  4 ++--
 .../scala/subsystem/HasHierarchicalElements.scala  |  8 ++++----
 src/main/scala/tilelink/BusWrapper.scala           |  2 +-
 4 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/main/scala/interrupts/Crossing.scala b/src/main/scala/interrupts/Crossing.scala
index 366efaaf532..b113dfdc538 100644
--- a/src/main/scala/interrupts/Crossing.scala
+++ b/src/main/scala/interrupts/Crossing.scala
@@ -35,14 +35,16 @@ class IntSyncCrossingSource(alreadyRegistered: Boolean = false)(implicit p: Para
 {
   val node = IntSyncSourceNode(alreadyRegistered)
 
-  lazy val module = new Impl
+  lazy val module = if (alreadyRegistered) (new ImplRegistered) else (new Impl)
+
   class Impl extends LazyModuleImp(this) {
     (node.in zip node.out) foreach { case ((in, edgeIn), (out, edgeOut)) =>
-      if (alreadyRegistered) {
-        out.sync := in
-      } else {
-        out.sync := AsyncResetReg(Cat(in.reverse)).asBools
-      }
+      out.sync := AsyncResetReg(Cat(in.reverse)).asBools
+    }
+  }
+  class ImplRegistered extends LazyRawModuleImp(this) {
+    (node.in zip node.out) foreach { case ((in, edgeIn), (out, edgeOut)) =>
+      out.sync := in
     }
   }
 }
diff --git a/src/main/scala/interrupts/NullIntSource.scala b/src/main/scala/interrupts/NullIntSource.scala
index d906ac36fd9..ec2528aa70b 100644
--- a/src/main/scala/interrupts/NullIntSource.scala
+++ b/src/main/scala/interrupts/NullIntSource.scala
@@ -12,7 +12,7 @@ class NullIntSource(num: Int = 1, ports: Int = 1, sources: Int = 1)(implicit p:
   val intnode = IntSourceNode(IntSourcePortSimple(num, ports, sources))
 
   lazy val module = new Impl
-  class Impl extends LazyModuleImp(this) {
+  class Impl extends LazyRawModuleImp(this) {
     intnode.out.foreach { case (o, _) => o.foreach { _ := false.B } }
   }
 }
@@ -26,6 +26,6 @@ object NullIntSource {
 
 object NullIntSyncSource {
   def apply(num: Int = 1, ports: Int = 1, sources: Int = 1)(implicit p: Parameters): IntSyncOutwardNode = {
-    IntSyncCrossingSource() := NullIntSource(num, ports, sources)
+    IntSyncCrossingSource(alreadyRegistered = true) := NullIntSource(num, ports, sources)
   }
 }
diff --git a/src/main/scala/subsystem/HasHierarchicalElements.scala b/src/main/scala/subsystem/HasHierarchicalElements.scala
index fe6ce378463..63f2ff14f18 100644
--- a/src/main/scala/subsystem/HasHierarchicalElements.scala
+++ b/src/main/scala/subsystem/HasHierarchicalElements.scala
@@ -183,9 +183,9 @@ trait HasHierarchicalElementsRootContext
   val plicDomainOpt: Option[ClockDomain]
   val debugOpt: Option[TLDebugModule]
 
-  def msipDomain = clintDomainOpt.get
-  def meipDomain = plicDomainOpt.get
-  def seipDomain = plicDomainOpt.get
+  def msipDomain = clintDomainOpt.getOrElse(this)
+  def meipDomain = plicDomainOpt.getOrElse(this)
+  def seipDomain = plicDomainOpt.getOrElse(this)
 
   val msipNodes: SortedMap[Int, IntNode] = (0 until nTotalTiles).map { i =>
     (i, IntEphemeralNode())
@@ -220,7 +220,7 @@ trait HasHierarchicalElementsRootContext
   }.to(SortedMap)
 
   debugNodes.foreach { case (hartid, node) =>
-    node := debugOpt.map(_.intnode).getOrElse(IntSyncCrossingSource() := NullIntSource())
+    node := debugOpt.map(_.intnode).getOrElse(NullIntSyncSource())
   }
 
   val nmiHarts = totalTiles.filter { case (_, t) => t.tileParams.core.useNMI }.keys
diff --git a/src/main/scala/tilelink/BusWrapper.scala b/src/main/scala/tilelink/BusWrapper.scala
index 246bba2f365..b07844cef6a 100644
--- a/src/main/scala/tilelink/BusWrapper.scala
+++ b/src/main/scala/tilelink/BusWrapper.scala
@@ -72,7 +72,7 @@ abstract class TLBusWrapper(params: HasTLBusParams, val busName: String)(implici
   def unifyManagers: List[TLManagerParameters] = ManagerUnification(busView.manager.managers)
   def crossOutHelper = this.crossOut(outwardNode)(ValName("bus_xing"))
   def crossInHelper = this.crossIn(inwardNode)(ValName("bus_xing"))
-  def generateSynchronousDomain = {
+  def generateSynchronousDomain: ClockSinkDomain = {
     val domain = LazyModule(new ClockSinkDomain(take = fixedClockOpt))
     domain.clockNode := fixedClockNode
     domain

From f85496e5993a46f01654b814c73b743c09a51b73 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 24 Oct 2023 20:38:05 -0700
Subject: [PATCH 46/47] Add SimpleLazyRawModule

---
 src/main/scala/diplomacy/LazyModule.scala | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/scala/diplomacy/LazyModule.scala b/src/main/scala/diplomacy/LazyModule.scala
index 9cb6e0513cd..a6b0791f231 100644
--- a/src/main/scala/diplomacy/LazyModule.scala
+++ b/src/main/scala/diplomacy/LazyModule.scala
@@ -431,6 +431,10 @@ class LazyRawModuleImp(val wrapper: LazyModule) extends RawModule with LazyModul
 class SimpleLazyModule(implicit p: Parameters) extends LazyModule {
   lazy val module = new LazyModuleImp(this)
 }
+class SimpleLazyRawModule(implicit p: Parameters) extends LazyModule {
+  lazy val module = new LazyRawModuleImp(this)
+}
+
 
 /** Allows dynamic creation of [[Module]] hierarchy and "shoving" logic into a [[LazyModule]]. */
 trait LazyScope {

From 59e86fd2ba98b02a763dad2bd51b6324671ca2b1 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 24 Oct 2023 20:38:34 -0700
Subject: [PATCH 47/47] Synchronize ext-ints within the ibus domain

---
 src/main/scala/subsystem/InterruptBus.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/subsystem/InterruptBus.scala b/src/main/scala/subsystem/InterruptBus.scala
index b817b11d856..2815e3f0866 100644
--- a/src/main/scala/subsystem/InterruptBus.scala
+++ b/src/main/scala/subsystem/InterruptBus.scala
@@ -54,7 +54,7 @@ trait HasAsyncExtInterrupts extends HasExtInterrupts { this: BaseSubsystem =>
   */
 trait HasSyncExtInterrupts extends HasExtInterrupts { this: BaseSubsystem =>
   if (nExtInterrupts > 0) {
-    ibus.fromSync := extInterrupts
+    ibus { ibus.fromSync := extInterrupts }
   }
 }