From 95a694a74661e7d7b006adf2126667c2565286b6 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Mon, 17 Aug 2026 13:40:13 +0200 Subject: [PATCH 01/17] [SRC] Change cluster base id to a paramemter instead of a logic input at cluster level --- hardware/src/cachepool_cluster.sv | 18 ++++++++---------- hardware/tb/cachepool_cluster_wrapper.sv | 4 ++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/hardware/src/cachepool_cluster.sv b/hardware/src/cachepool_cluster.sv index ee777efc..d6a0b6c6 100644 --- a/hardware/src/cachepool_cluster.sv +++ b/hardware/src/cachepool_cluster.sv @@ -91,11 +91,13 @@ module cachepool_cluster /*** ATTENTION: `NrSramCfg` should be changed if `L1NumDataBank` and `L1NumTagBank` is changed ***/ parameter int unsigned NrSramCfg = 1, /// Folded data bank configuration (0 = auto: min(4, L1AssoPerCtrl)). - parameter bit UseFoldedDataBanks = 1'b1, - parameter int unsigned FoldWayGroup = 0, - parameter bit UseHashWaySelect = 1'b1, + parameter bit UseFoldedDataBanks = 1'b1, + parameter int unsigned FoldWayGroup = 0, + parameter bit UseHashWaySelect = 1'b1, /// Enable the SRAM forwarding buffer (default on; requires UseHashWaySelect). - parameter bit UseForwardingBuffer = 1'b1 + parameter bit UseForwardingBuffer = 1'b1, + /// First hartid of the cluster; cores get hartids HartBaseId..HartBaseId+NrCores-1. + parameter logic [9:0] HartBaseId = 10'h0 ) ( /// System clock. input logic clk_i, @@ -117,10 +119,6 @@ module cachepool_cluster /// another core to facilitate inter-processor-interrupts. This signal is /// assumed to be _async_. input logic msip_i, - /// First hartid of the cluster. Cores of a cluster are monotonically - /// increasing without a gap, i.e., a cluster with 8 cores and a - /// `hart_base_id_i` of 5 get the hartids 5 - 12. - input logic [9:0] hart_base_id_i, /// Base address of cluster. TCDM and cluster peripheral location are derived from /// it. This signal is pseudo-static. input logic [AxiAddrWidth-1:0] cluster_base_addr_i, @@ -290,7 +288,7 @@ module cachepool_cluster .meip_i ( meip_i ), .mtip_i ( mtip_i ), .msip_i ( msip_i ), - .hart_base_id_i ( hart_base_id_i + 10'(g * NumCoreGroup) ), + .hart_base_id_i ( HartBaseId + 10'(g * NumCoreGroup) ), .tile_base_id_i ( TileIDWidth'(g * NumTilesPerGroup) ), .cluster_base_addr_i ( cluster_base_addr_i ), .private_start_addr_i ( private_start_addr ), @@ -1264,7 +1262,7 @@ module cachepool_cluster .tcdm_start_address_i ( tcdm_start_address ), .tcdm_end_address_i ( tcdm_end_address ), .icache_prefetch_enable_o ( icache_prefetch_enable ), - .cluster_hart_base_id_i ( hart_base_id_i ), + .cluster_hart_base_id_i ( HartBaseId ), .cluster_probe_o ( cluster_probe_o ), .dynamic_offset_o ( dynamic_offset ), .private_start_addr_o ( private_start_addr ), diff --git a/hardware/tb/cachepool_cluster_wrapper.sv b/hardware/tb/cachepool_cluster_wrapper.sv index 6c4d7913..b68fcc39 100644 --- a/hardware/tb/cachepool_cluster_wrapper.sv +++ b/hardware/tb/cachepool_cluster_wrapper.sv @@ -99,7 +99,8 @@ module cachepool_cluster_wrapper .UseFoldedDataBanks (UseFoldedDataBanks ), .FoldWayGroup (FoldWayGroup ), .UseHashWaySelect (UseHashWaySelect ), - .UseForwardingBuffer (UseForwardingBuffer ) + .UseForwardingBuffer (UseForwardingBuffer ), + .HartBaseId (10'h0 ) ) i_cluster ( .clk_i , .rst_ni , @@ -110,7 +111,6 @@ module cachepool_cluster_wrapper .meip_i (meip_i ), .mtip_i (mtip_i ), .msip_i (msip_i ), - .hart_base_id_i (10'h0 ), .cluster_base_addr_i (TCDMStartAddr ), .cluster_probe_o (cluster_probe_o ), // REQRSP peripheral in-port: passed straight through from wrapper port. From e5aa7e571ddf70e1f78e119ea37d18f899b073a3 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Mon, 17 Aug 2026 14:12:53 +0200 Subject: [PATCH 02/17] [Util] Add missing yaml package into venv, and fix a hardcoded path in peripheral geneartion. --- hardware/cachepool_peripheral/Makefile | 4 +++- requirements.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hardware/cachepool_peripheral/Makefile b/hardware/cachepool_peripheral/Makefile index dc79b69d..fdb2da45 100644 --- a/hardware/cachepool_peripheral/Makefile +++ b/hardware/cachepool_peripheral/Makefile @@ -5,7 +5,9 @@ # Noah Huetter ROOTDIR=../.. -REGTOOL=`$(ROOTDIR)/install/bender/bender path register_interface`/vendor/lowrisc_opentitan/util/regtool.py +BENDER ?= bender + +REGTOOL=`$(BENDER) path register_interface`/vendor/lowrisc_opentitan/util/regtool.py PYTHON3=$(shell which python) CLANG_FORMAT=$(shell which clang-format-10.0.1) diff --git a/requirements.txt b/requirements.txt index 3e978375..ef40567c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ numpy # CPU-only torch for data generation scripts (gen_data.py in software/tests/) --extra-index-url https://download.pytorch.org/whl/cpu torch +pyyaml From 357a6db913930c03ed1bca23b5e55b42456fd961 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Tue, 18 Aug 2026 09:44:41 +0200 Subject: [PATCH 03/17] [SRC] Rename paraemter NumCore to NumCC --- hardware/src/cachepool_cluster.sv | 10 +-- hardware/src/cachepool_group.sv | 16 ++--- hardware/src/cachepool_group_noc_wrapper.sv | 10 +-- hardware/src/cachepool_tile.sv | 74 ++++++++++++--------- hardware/tb/cachepool_cluster_wrapper.sv | 2 +- hardware/tb/cachepool_monitor.sv | 2 +- sim/scripts/vsim_wave.tcl | 2 +- 7 files changed, 62 insertions(+), 54 deletions(-) diff --git a/hardware/src/cachepool_cluster.sv b/hardware/src/cachepool_cluster.sv index d6a0b6c6..8fee4b61 100644 --- a/hardware/src/cachepool_cluster.sv +++ b/hardware/src/cachepool_cluster.sv @@ -31,14 +31,14 @@ module cachepool_cluster parameter logic [31:0] BootAddr = 32'h0, /// Address to indicate start of UART parameter logic [31:0] UartAddr = 32'h0, - /// The total amount of cores. - parameter int unsigned NrCores = 8, + /// Number of Core Complex (CC) slots in this cluster. + parameter int unsigned NumCC = 8, /// Data/TCDM memory depth per cut (in words). parameter int unsigned TCDMDepth = 1024, /// Cluster peripheral address region size (in kB). parameter int unsigned ClusterPeriphSize = 64, /// Number of TCDM Banks. - parameter int unsigned NrBanks = 2 * NrCores, + parameter int unsigned NrBanks = 2 * NumCC, /// Width of a single icache line. parameter unsigned ICacheLineWidth = 0, /// Number of icache lines per set. @@ -96,7 +96,7 @@ module cachepool_cluster parameter bit UseHashWaySelect = 1'b1, /// Enable the SRAM forwarding buffer (default on; requires UseHashWaySelect). parameter bit UseForwardingBuffer = 1'b1, - /// First hartid of the cluster; cores get hartids HartBaseId..HartBaseId+NrCores-1. + /// First hartid of the cluster; cores get hartids HartBaseId..HartBaseId+NumCC-1. parameter logic [9:0] HartBaseId = 10'h0 ) ( /// System clock. @@ -252,7 +252,7 @@ module cachepool_cluster .BootAddr ( BootAddr ), .UartAddr ( UartAddr ), .ClusterPeriphSize ( ClusterPeriphSize ), - .NrCores ( NumCoreGroup ), + .NumCC ( NumCoreGroup ), .TCDMDepth ( TCDMDepth ), .NrBanks ( NrBanks / NumGroups ), .ICacheLineWidth ( ICacheLineWidth ), diff --git a/hardware/src/cachepool_group.sv b/hardware/src/cachepool_group.sv index 9f9cdd47..1223ad1b 100644 --- a/hardware/src/cachepool_group.sv +++ b/hardware/src/cachepool_group.sv @@ -30,14 +30,14 @@ module cachepool_group parameter logic [31:0] BootAddr = 32'h0, /// Address to indicate start of UART parameter logic [31:0] UartAddr = 32'h0, - /// The total amount of cores. - parameter int unsigned NrCores = 8, + /// Number of Core Complex (CC) slots in this group. + parameter int unsigned NumCC = 8, /// Data/TCDM memory depth per cut (in words). parameter int unsigned TCDMDepth = 1024, /// Cluster peripheral address region size (in kB). parameter int unsigned ClusterPeriphSize = 64, /// Number of TCDM Banks. - parameter int unsigned NrBanks = 2 * NrCores, + parameter int unsigned NrBanks = 2 * NumCC, /// Width of a single icache line. parameter unsigned ICacheLineWidth = 0, /// Number of icache lines per set. @@ -144,9 +144,9 @@ module cachepool_group input floo_cachepool_noc_pkg::id_t l2_group_id_i, /// Peripheral signals - output icache_l1_events_t [NrCores-1:0] icache_events_o, + output icache_l1_events_t [NumCC-1:0] icache_events_o, input logic icache_prefetch_enable_i, - input logic [NrCores-1:0] cl_interrupt_i, + input logic [NumCC-1:0] cl_interrupt_i, input logic [$clog2(AxiAddrWidth)-1:0] dynamic_offset_i, input logic [$clog2(NumL1CtrlTile):0] l1d_private_i, input cache_insn_t l1d_insn_i, @@ -187,7 +187,7 @@ module cachepool_group // Constants // --------- // Per-group overrides of package-level constants that depend on NumTiles/NumCores. - localparam int unsigned NumL1CacheCtrlLocal = NrCores; + localparam int unsigned NumL1CacheCtrlLocal = NumCC; localparam int unsigned WideIdWidthIn = AxiIdWidthOut; @@ -562,7 +562,7 @@ module cachepool_group .BootAddr ( BootAddr ), .UartAddr ( UartAddr ), .ClusterPeriphSize ( ClusterPeriphSize ), - .NrCores ( NumCoresTile ), + .NumCC ( NumCoresTile ), .TCDMDepth ( TCDMDepth ), .NrBanks ( NrBanks ), .ICacheLineWidth ( ICacheLineWidth ), @@ -653,7 +653,7 @@ module cachepool_group .BootAddr ( BootAddr ), .UartAddr ( UartAddr ), .ClusterPeriphSize ( ClusterPeriphSize ), - .NrCores ( NumCoresTile ), + .NumCC ( NumCoresTile ), .TCDMDepth ( TCDMDepth ), .NrBanks ( NrBanks ), .ICacheLineWidth ( ICacheLineWidth ), diff --git a/hardware/src/cachepool_group_noc_wrapper.sv b/hardware/src/cachepool_group_noc_wrapper.sv index 0950eed9..227ae06c 100644 --- a/hardware/src/cachepool_group_noc_wrapper.sv +++ b/hardware/src/cachepool_group_noc_wrapper.sv @@ -27,10 +27,10 @@ module cachepool_group_noc_wrapper parameter int unsigned AxiUserWidth = 1, parameter logic [31:0] BootAddr = 32'h0, parameter logic [31:0] UartAddr = 32'h0, - parameter int unsigned NrCores = 0, + parameter int unsigned NumCC = 0, parameter int unsigned TCDMDepth = 1024, parameter int unsigned ClusterPeriphSize = 64, - parameter int unsigned NrBanks = 2 * NrCores, + parameter int unsigned NrBanks = 2 * NumCC, parameter int unsigned ICacheLineWidth = 0, parameter int unsigned ICacheLineCount = 0, parameter int unsigned ICacheSets = 0, @@ -90,9 +90,9 @@ module cachepool_group_noc_wrapper // L2 mesh: endpoint ID and source-routing table from floogen input floo_cachepool_noc_pkg::id_t l2_id_i, input floo_cachepool_noc_pkg::route_t [floo_cachepool_noc_pkg::RouteCfg.NumRoutes-1:0] l2_route_table_i, - output icache_l1_events_t [NrCores-1:0] icache_events_o, + output icache_l1_events_t [NumCC-1:0] icache_events_o, input logic icache_prefetch_enable_i, - input logic [NrCores-1:0] cl_interrupt_i, + input logic [NumCC-1:0] cl_interrupt_i, input logic [$clog2(AxiAddrWidth)-1:0] dynamic_offset_i, input logic [$clog2(NumL1CtrlTile):0] l1d_private_i, input cache_insn_t l1d_insn_i, @@ -742,7 +742,7 @@ module cachepool_group_noc_wrapper .BootAddr ( BootAddr ), .UartAddr ( UartAddr ), .ClusterPeriphSize ( ClusterPeriphSize ), - .NrCores ( NrCores ), + .NumCC ( NumCC ), .TCDMDepth ( TCDMDepth ), .NrBanks ( NrBanks ), .ICacheLineWidth ( ICacheLineWidth ), diff --git a/hardware/src/cachepool_tile.sv b/hardware/src/cachepool_tile.sv index 9defdd60..eda36f64 100644 --- a/hardware/src/cachepool_tile.sv +++ b/hardware/src/cachepool_tile.sv @@ -32,14 +32,17 @@ module cachepool_tile parameter logic [31:0] BootAddr = 32'h0, /// Address to indicate start of UART parameter logic [31:0] UartAddr = 32'h0, - /// The total amount of cores. - parameter int unsigned NrCores = 8, + /// Number of Core Complex (CC) slots in this tile. + parameter int unsigned NumCC = 8, + /// Number of Snitch scalar cores sharing one Spatz per CC (1 = today's + /// cachepool_cc, 2 = cachepool_cc_dual). Homogeneous per tile. + parameter int unsigned NumScalarPerCC = 1, /// Data/TCDM memory depth per cut (in words). parameter int unsigned TCDMDepth = 1024, /// Cluster peripheral address region size (in kB). parameter int unsigned ClusterPeriphSize = 64, /// Number of TCDM Banks. - parameter int unsigned NrBanks = 2 * NrCores, + parameter int unsigned NrBanks = 2 * NumCC, /// Width of a single icache line. parameter unsigned ICacheLineWidth = 0, /// Number of icache lines per set. @@ -110,7 +113,9 @@ module cachepool_tile parameter bit UseHashWaySelect = 1'b0, /// Enable the SRAM forwarding buffer (default on; requires UseHashWaySelect). parameter bit UseForwardingBuffer = 1'b1, - localparam int unsigned TotRGPorts = (NumRemoteGroupPortCore == 0) ? 0 : NumRemoteGroupPortCore*NrTCDMPortsPerCore-1 + localparam int unsigned TotRGPorts = (NumRemoteGroupPortCore == 0) ? 0 : NumRemoteGroupPortCore*NrTCDMPortsPerCore-1, + /// Derived parameter *Do not override*: true hart count (NumCC = CC-slot count). + localparam int unsigned NrHarts = NumCC * NumScalarPerCC ) ( /// System clock. input logic clk_i, @@ -170,9 +175,9 @@ module cachepool_tile input remote_group_req_t [TotRGPorts:0] remote_group_req_i, output remote_group_rsp_t [TotRGPorts:0] remote_group_rsp_o, /// Peripheral signals - output icache_l1_events_t [NrCores-1:0] icache_events_o, + output icache_l1_events_t [NrHarts-1:0] icache_events_o, input logic icache_prefetch_enable_i, - input logic [NrCores-1:0] cl_interrupt_i, + input logic [NrHarts-1:0] cl_interrupt_i, input logic [$clog2(AxiAddrWidth)-1:0] dynamic_offset_i, input cache_insn_t l1d_insn_i, input logic [$clog2(NumL1CtrlTile):0] l1d_private_i, @@ -204,7 +209,7 @@ module cachepool_tile assign num_private_cache = l1d_private_i [$clog2(NumL1CtrlTile):0]; /// Minimum width to hold the core number. - // localparam int unsigned CoreIDWidth = cf_math_pkg::idx_width(NrCores); + // localparam int unsigned CoreIDWidth = cf_math_pkg::idx_width(NumCC); localparam int unsigned TCDMMemAddrWidth = $clog2(TCDMDepth); // Enlarge the address width for Spatz due to cache @@ -222,7 +227,7 @@ module cachepool_tile return n; endfunction - localparam int unsigned NrTCDMPortsCores = get_tcdm_port_offs(NrCores); + localparam int unsigned NrTCDMPortsCores = get_tcdm_port_offs(NumCC); localparam int unsigned NumTCDMIn = NrTCDMPortsCores + 1; localparam logic [AxiAddrWidth-1:0] TCDMMask = ~(TCDMSize-1); @@ -350,13 +355,13 @@ module cachepool_tile tcdm_req_t [NrTCDMPortsCores-1:0] tcdm_req; tcdm_rsp_t [NrTCDMPortsCores-1:0] tcdm_rsp; - core_events_t [NrCores-1:0] core_events; + core_events_t [NumCC-1:0] core_events; - // snitch_icache_pkg::icache_events_t [NrCores-1:0] icache_events; + // snitch_icache_pkg::icache_events_t [NumCC-1:0] icache_events; // 4. Memory Subsystem (Core side). - reqrsp_req_t [NrCores-1:0] core_req, filtered_core_req; - reqrsp_rsp_t [NrCores-1:0] core_rsp, filtered_core_rsp; + reqrsp_req_t [NumCC-1:0] core_req, filtered_core_req; + reqrsp_rsp_t [NumCC-1:0] core_rsp, filtered_core_rsp; // 8. L1 D$ @@ -684,7 +689,7 @@ module cachepool_tile tcdm_cache_interco #( .NumTiles (NumTiles ), - .NumCores (NrCores ), + .NumCores (NumCC ), .NumCache (NumL1CtrlTile ), .NumTotCache (NumL1CacheCtrl ), .NumLGPort (NumLGPortCore ), @@ -716,7 +721,7 @@ module cachepool_tile // No inter-group remote ports: instantiate interco without inter-group remote ports (backward-compatible). tcdm_cache_interco #( .NumTiles (NumTiles ), - .NumCores (NrCores ), + .NumCores (NumCC ), .NumCache (NumL1CtrlTile ), .NumTotCache (NumL1CacheCtrl ), .NumLGPort (NumLGPortCore ), @@ -1448,12 +1453,15 @@ module cachepool_tile end end - hive_req_t [NrCores-1:0] hive_req; - hive_rsp_t [NrCores-1:0] hive_rsp; + hive_req_t [NrHarts-1:0] hive_req; + hive_rsp_t [NrHarts-1:0] hive_rsp; - for (genvar i = 0; i < NrCores; i++) begin : gen_core + for (genvar i = 0; i < NumCC; i++) begin : gen_cc localparam int unsigned TcdmPorts = get_tcdm_ports(i); localparam int unsigned TcdmPortsOffs = get_tcdm_port_offs(i); + // First hart index served by this CC slot. Only this hart is wired up + // for now (NumScalarPerCC>1's extra hart slots are added separately). + localparam int unsigned HartIdx = i * NumScalarPerCC; interrupts_t irq; @@ -1465,12 +1473,12 @@ module cachepool_tile i_sync_mtip (.clk_i, .rst_ni, .serial_i (mtip_i), .serial_o (irq.mtip)); sync #(.STAGES (2)) i_sync_msip (.clk_i, .rst_ni, .serial_i (msip_i), .serial_o (irq.msip)); - assign irq.mcip = cl_interrupt_i[i]; + assign irq.mcip = cl_interrupt_i[HartIdx]; tcdm_req_t [TcdmPorts-1:0] tcdm_req_wo_user; logic [31:0] hart_id; - assign hart_id = hart_base_id_i + i; + assign hart_id = hart_base_id_i + HartIdx; cachepool_cc #( .BootAddr (BootAddr ), @@ -1523,8 +1531,8 @@ module cachepool_tile .rst_ni (rst_ni ), .testmode_i (1'b0 ), .hart_id_i (hart_id ), - .hive_req_o (hive_req[i] ), - .hive_rsp_i (hive_rsp[i] ), + .hive_req_o (hive_req[HartIdx] ), + .hive_rsp_i (hive_rsp[HartIdx] ), .irq_i (irq ), .data_req_o (core_req[i] ), .data_rsp_i (core_rsp[i] ), @@ -1547,16 +1555,16 @@ module cachepool_tile // Instruction Cache // ---------------- - addr_t [NrCores-1:0] inst_addr; - logic [NrCores-1:0] inst_cacheable; - logic [NrCores-1:0][31:0] inst_data; - logic [NrCores-1:0] inst_valid; - logic [NrCores-1:0] inst_ready; - logic [NrCores-1:0] inst_error; - logic [NrCores-1:0] flush_valid; - logic [NrCores-1:0] flush_ready; + addr_t [NrHarts-1:0] inst_addr; + logic [NrHarts-1:0] inst_cacheable; + logic [NrHarts-1:0][31:0] inst_data; + logic [NrHarts-1:0] inst_valid; + logic [NrHarts-1:0] inst_ready; + logic [NrHarts-1:0] inst_error; + logic [NrHarts-1:0] flush_valid; + logic [NrHarts-1:0] flush_ready; - for (genvar i = 0; i < NrCores; i++) begin : gen_unpack_icache + for (genvar i = 0; i < NrHarts; i++) begin : gen_unpack_icache assign inst_addr[i] = hive_req[i].inst_addr; assign inst_cacheable[i] = hive_req[i].inst_cacheable; assign inst_valid[i] = hive_req[i].inst_valid; @@ -1571,7 +1579,7 @@ module cachepool_tile end snitch_icache #( - .NR_FETCH_PORTS ( NrCores ), + .NR_FETCH_PORTS ( NrHarts ), .L0_LINE_COUNT ( 8 ), .LINE_WIDTH ( ICacheLineWidth ), .LINE_COUNT ( ICacheLineCount ), @@ -1618,7 +1626,7 @@ module cachepool_tile // First-level barrier for CachePool system cachepool_tile_barrier #( .AddrWidth (AxiAddrWidth ), - .NrPorts (NrCores ), + .NrPorts (NumCC ), .dreq_t (reqrsp_req_t ), .drsp_t (reqrsp_rsp_t ), .user_t (tcdm_user_t ) @@ -1638,7 +1646,7 @@ module cachepool_tile reqrsp_rsp_t core_to_periph_rsp; reqrsp_mux #( - .NrPorts (NrCores ), + .NrPorts (NumCC ), .AddrWidth (AxiAddrWidth ), .DataWidth (NarrowDataWidth ), .UserWidth ($bits(tcdm_user_t)), diff --git a/hardware/tb/cachepool_cluster_wrapper.sv b/hardware/tb/cachepool_cluster_wrapper.sv index b68fcc39..d85d5b9f 100644 --- a/hardware/tb/cachepool_cluster_wrapper.sv +++ b/hardware/tb/cachepool_cluster_wrapper.sv @@ -73,7 +73,7 @@ module cachepool_cluster_wrapper .BootAddr (BootAddr ), .UartAddr (UartAddr ), .ClusterPeriphSize (64 ), - .NrCores (NumCores ), + .NumCC (NumCores ), .TCDMDepth (TCDMDepth ), .NrBanks (NumBank ), .ICacheLineWidth (ICacheLineWidth ), diff --git a/hardware/tb/cachepool_monitor.sv b/hardware/tb/cachepool_monitor.sv index 0ddd0d4e..bd7cf053 100644 --- a/hardware/tb/cachepool_monitor.sv +++ b/hardware/tb/cachepool_monitor.sv @@ -30,7 +30,7 @@ module cachepool_monitor i_cluster_wrapper.i_cluster.gen_group_y[gy].gen_group_x[gx].i_group.i_group.gen_tiles[t].gen_tile.i_tile `define CC_PATH(gy, gx, t, c) \ - `TILE_PATH(gy, gx, t).gen_core[c].i_cachepool_cc + `TILE_PATH(gy, gx, t).gen_cc[c].i_cachepool_cc `define CLUSTER_PATH i_cluster_wrapper.i_cluster diff --git a/sim/scripts/vsim_wave.tcl b/sim/scripts/vsim_wave.tcl index 8a1db670..74ee5ef5 100644 --- a/sim/scripts/vsim_wave.tcl +++ b/sim/scripts/vsim_wave.tcl @@ -61,7 +61,7 @@ for {set g 0} {$g < $NUM_GROUPS} {incr g} { # 4. Plot all cores grouped under their tile for {set core 0} {$core < $NUM_CORES} {incr core} { - quietly set core_path ${tile_path}/i_tile/gen_core[${core}] + quietly set core_path ${tile_path}/i_tile/gen_cc[${core}] do sim/scripts/vsim_core.tcl $g $tile $core ${core_path} "${gwp_name}" "tile[${tile}]" } } From 92a0e48b92a9851e967f197041b3772816834792 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Tue, 18 Aug 2026 10:48:47 +0200 Subject: [PATCH 04/17] [SRC] WIP: add dual-snitch basic hardware (not yet enabled) --- .../cachepool_peripheral_reg.hjson | 13 + .../cachepool_peripheral_reg_pkg.sv | 92 +-- .../cachepool_peripheral_reg_top.sv | 134 ++-- hardware/src/cachepool_cc_dual.sv | 657 ++++++++++++++++++ hardware/src/cachepool_pkg.sv | 9 +- hardware/src/cachepool_spatz_lock.sv | 329 +++++++++ hardware/src/cachepool_tile.sv | 303 +++++--- hardware/tb/cachepool_monitor.sv | 56 +- .../snRuntime/include/cachepool_peripheral.h | 39 +- 9 files changed, 1394 insertions(+), 238 deletions(-) create mode 100644 hardware/src/cachepool_cc_dual.sv create mode 100644 hardware/src/cachepool_spatz_lock.sv diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson index 02aadb65..bc3949bd 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson @@ -31,6 +31,19 @@ desc: "Hardware barrier register." }] }, + { + name: "SPATZ_LOCK", + desc: '''Spatz ownership lock for dual-Snitch core complexes. Write 1 to acquire + (blocks until granted), write 0 to release.''' + swaccess: "ro", + hwaccess: "hrw", + hwext: "true", + fields: [{ + bits: "31:0", + name: "SPATZ_LOCK", + desc: "Spatz ownership lock register." + }] + }, { name: "ICACHE_PREFETCH_ENABLE", desc: '''Controls prefetching of the instruction cache.''' diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv index 52b2d3ba..2fce913c 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv @@ -20,6 +20,10 @@ package cachepool_peripheral_reg_pkg; logic [31:0] q; } cachepool_peripheral_reg2hw_hw_barrier_reg_t; + typedef struct packed { + logic [31:0] q; + } cachepool_peripheral_reg2hw_spatz_lock_reg_t; + typedef struct packed { logic q; } cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t; @@ -84,6 +88,10 @@ package cachepool_peripheral_reg_pkg; logic [31:0] d; } cachepool_peripheral_hw2reg_hw_barrier_reg_t; + typedef struct packed { + logic [31:0] d; + } cachepool_peripheral_hw2reg_spatz_lock_reg_t; + typedef struct packed { logic d; logic de; @@ -105,7 +113,8 @@ package cachepool_peripheral_reg_pkg; // Register -> HW type typedef struct packed { - cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [285:254] + cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [317:286] + cachepool_peripheral_reg2hw_spatz_lock_reg_t spatz_lock; // [285:254] cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t icache_prefetch_enable; // [253:253] cachepool_peripheral_reg2hw_spatz_status_reg_t spatz_status; // [252:252] cachepool_peripheral_reg2hw_spatz_cycle_reg_t spatz_cycle; // [251:220] @@ -125,7 +134,8 @@ package cachepool_peripheral_reg_pkg; // HW -> register type typedef struct packed { - cachepool_peripheral_hw2reg_hw_barrier_reg_t hw_barrier; // [38:7] + cachepool_peripheral_hw2reg_hw_barrier_reg_t hw_barrier; // [70:39] + cachepool_peripheral_hw2reg_spatz_lock_reg_t spatz_lock; // [38:7] cachepool_peripheral_hw2reg_l1d_spm_commit_reg_t l1d_spm_commit; // [6:5] cachepool_peripheral_hw2reg_l1d_insn_commit_reg_t l1d_insn_commit; // [4:3] cachepool_peripheral_hw2reg_l1d_flush_status_reg_t l1d_flush_status; // [2:2] @@ -134,33 +144,36 @@ package cachepool_peripheral_reg_pkg; // Register offsets parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_OFFSET = 7'h 0; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET = 7'h 4; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET = 7'h 8; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET = 7'h c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET = 7'h 10; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET = 7'h 14; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET = 7'h 18; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET = 7'h 1c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET = 7'h 20; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET = 7'h 24; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET = 7'h 28; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET = 7'h 2c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET = 7'h 30; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET = 7'h 34; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET = 7'h 38; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET = 7'h 3c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET = 7'h 40; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET = 7'h 44; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET = 7'h 48; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_OFFSET = 7'h 4; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET = 7'h 8; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET = 7'h c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET = 7'h 10; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET = 7'h 14; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET = 7'h 18; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET = 7'h 1c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET = 7'h 20; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET = 7'h 24; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET = 7'h 28; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET = 7'h 2c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET = 7'h 30; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET = 7'h 34; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET = 7'h 38; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET = 7'h 3c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET = 7'h 40; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET = 7'h 44; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET = 7'h 48; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET = 7'h 4c; // Reset values for hwext registers and their fields parameter logic [31:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_RESVAL = 32'h 0; + parameter logic [31:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RESVAL = 32'h 0; parameter logic [0:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_RESVAL = 1'h 0; parameter logic [0:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_STATUS_RESVAL = 1'h 0; // Register index typedef enum int { CACHEPOOL_PERIPHERAL_HW_BARRIER, + CACHEPOOL_PERIPHERAL_SPATZ_LOCK, CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE, CACHEPOOL_PERIPHERAL_SPATZ_STATUS, CACHEPOOL_PERIPHERAL_SPATZ_CYCLE, @@ -182,26 +195,27 @@ package cachepool_peripheral_reg_pkg; } cachepool_peripheral_id_e; // Register width information to check illegal writes - parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [19] = '{ + parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [20] = '{ 4'b 1111, // index[ 0] CACHEPOOL_PERIPHERAL_HW_BARRIER - 4'b 0001, // index[ 1] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE - 4'b 0001, // index[ 2] CACHEPOOL_PERIPHERAL_SPATZ_STATUS - 4'b 1111, // index[ 3] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE - 4'b 1111, // index[ 4] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL - 4'b 0001, // index[ 5] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT - 4'b 0011, // index[ 6] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM - 4'b 0001, // index[ 7] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN - 4'b 1111, // index[ 8] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0 - 4'b 1111, // index[ 9] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1 - 4'b 1111, // index[10] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0 - 4'b 1111, // index[11] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1 - 4'b 0001, // index[12] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT - 4'b 0001, // index[13] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT - 4'b 0001, // index[14] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS - 4'b 0001, // index[15] CACHEPOOL_PERIPHERAL_L1D_PRIVATE - 4'b 1111, // index[16] CACHEPOOL_PERIPHERAL_L1D_ADDR - 4'b 0001, // index[17] CACHEPOOL_PERIPHERAL_XBAR_OFFSET - 4'b 0001 // index[18] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT + 4'b 1111, // index[ 1] CACHEPOOL_PERIPHERAL_SPATZ_LOCK + 4'b 0001, // index[ 2] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE + 4'b 0001, // index[ 3] CACHEPOOL_PERIPHERAL_SPATZ_STATUS + 4'b 1111, // index[ 4] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE + 4'b 1111, // index[ 5] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL + 4'b 0001, // index[ 6] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT + 4'b 0011, // index[ 7] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM + 4'b 0001, // index[ 8] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN + 4'b 1111, // index[ 9] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0 + 4'b 1111, // index[10] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1 + 4'b 1111, // index[11] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0 + 4'b 1111, // index[12] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1 + 4'b 0001, // index[13] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT + 4'b 0001, // index[14] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT + 4'b 0001, // index[15] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS + 4'b 0001, // index[16] CACHEPOOL_PERIPHERAL_L1D_PRIVATE + 4'b 1111, // index[17] CACHEPOOL_PERIPHERAL_L1D_ADDR + 4'b 0001, // index[18] CACHEPOOL_PERIPHERAL_XBAR_OFFSET + 4'b 0001 // index[19] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT }; endpackage diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv index 8552d9e0..efa820df 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv @@ -70,6 +70,8 @@ module cachepool_peripheral_reg_top #( // or _{wd|we|qs} if field == 1 or 0 logic [31:0] hw_barrier_qs; logic hw_barrier_re; + logic [31:0] spatz_lock_qs; + logic spatz_lock_re; logic icache_prefetch_enable_wd; logic icache_prefetch_enable_we; logic spatz_status_wd; @@ -139,6 +141,22 @@ module cachepool_peripheral_reg_top #( ); + // R[spatz_lock]: V(True) + + prim_subreg_ext #( + .DW (32) + ) u_spatz_lock ( + .re (spatz_lock_re), + .we (1'b0), + .wd ('0), + .d (hw2reg.spatz_lock.d), + .qre (), + .qe (), + .q (reg2hw.spatz_lock.q ), + .qs (spatz_lock_qs) + ); + + // R[icache_prefetch_enable]: V(False) prim_subreg #( @@ -618,28 +636,29 @@ module cachepool_peripheral_reg_top #( - logic [18:0] addr_hit; + logic [19:0] addr_hit; always_comb begin addr_hit = '0; addr_hit[ 0] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_OFFSET); - addr_hit[ 1] = (reg_addr == CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET); - addr_hit[ 2] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET); - addr_hit[ 3] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET); - addr_hit[ 4] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET); - addr_hit[ 5] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET); - addr_hit[ 6] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET); - addr_hit[ 7] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET); - addr_hit[ 8] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET); - addr_hit[ 9] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET); - addr_hit[10] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET); - addr_hit[11] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET); - addr_hit[12] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET); - addr_hit[13] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET); - addr_hit[14] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET); - addr_hit[15] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET); - addr_hit[16] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET); - addr_hit[17] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET); - addr_hit[18] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET); + addr_hit[ 1] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_LOCK_OFFSET); + addr_hit[ 2] = (reg_addr == CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET); + addr_hit[ 3] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET); + addr_hit[ 4] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET); + addr_hit[ 5] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET); + addr_hit[ 6] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET); + addr_hit[ 7] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET); + addr_hit[ 8] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET); + addr_hit[ 9] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET); + addr_hit[10] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET); + addr_hit[11] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET); + addr_hit[12] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET); + addr_hit[13] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET); + addr_hit[14] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET); + addr_hit[15] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET); + addr_hit[16] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET); + addr_hit[17] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET); + addr_hit[18] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET); + addr_hit[19] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET); end assign addrmiss = (reg_re || reg_we) ? ~|addr_hit : 1'b0 ; @@ -665,62 +684,65 @@ module cachepool_peripheral_reg_top #( (addr_hit[15] & (|(CACHEPOOL_PERIPHERAL_PERMIT[15] & ~reg_be))) | (addr_hit[16] & (|(CACHEPOOL_PERIPHERAL_PERMIT[16] & ~reg_be))) | (addr_hit[17] & (|(CACHEPOOL_PERIPHERAL_PERMIT[17] & ~reg_be))) | - (addr_hit[18] & (|(CACHEPOOL_PERIPHERAL_PERMIT[18] & ~reg_be))))); + (addr_hit[18] & (|(CACHEPOOL_PERIPHERAL_PERMIT[18] & ~reg_be))) | + (addr_hit[19] & (|(CACHEPOOL_PERIPHERAL_PERMIT[19] & ~reg_be))))); end assign hw_barrier_re = addr_hit[0] & reg_re & !reg_error; - assign icache_prefetch_enable_we = addr_hit[1] & reg_we & !reg_error; + assign spatz_lock_re = addr_hit[1] & reg_re & !reg_error; + + assign icache_prefetch_enable_we = addr_hit[2] & reg_we & !reg_error; assign icache_prefetch_enable_wd = reg_wdata[0]; - assign spatz_status_we = addr_hit[2] & reg_we & !reg_error; + assign spatz_status_we = addr_hit[3] & reg_we & !reg_error; assign spatz_status_wd = reg_wdata[0]; - assign spatz_cycle_we = addr_hit[3] & reg_we & !reg_error; + assign spatz_cycle_we = addr_hit[4] & reg_we & !reg_error; assign spatz_cycle_wd = reg_wdata[31:0]; - assign cluster_boot_control_we = addr_hit[4] & reg_we & !reg_error; + assign cluster_boot_control_we = addr_hit[5] & reg_we & !reg_error; assign cluster_boot_control_wd = reg_wdata[31:0]; - assign cluster_eoc_exit_we = addr_hit[5] & reg_we & !reg_error; + assign cluster_eoc_exit_we = addr_hit[6] & reg_we & !reg_error; assign cluster_eoc_exit_wd = reg_wdata[3:0]; - assign cfg_l1d_spm_we = addr_hit[6] & reg_we & !reg_error; + assign cfg_l1d_spm_we = addr_hit[7] & reg_we & !reg_error; assign cfg_l1d_spm_wd = reg_wdata[9:0]; - assign cfg_l1d_insn_we = addr_hit[7] & reg_we & !reg_error; + assign cfg_l1d_insn_we = addr_hit[8] & reg_we & !reg_error; assign cfg_l1d_insn_wd = reg_wdata[1:0]; - assign cfg_l1d_tile_sel_0_we = addr_hit[8] & reg_we & !reg_error; + assign cfg_l1d_tile_sel_0_we = addr_hit[9] & reg_we & !reg_error; assign cfg_l1d_tile_sel_0_wd = reg_wdata[31:0]; - assign cfg_l1d_tile_sel_1_we = addr_hit[9] & reg_we & !reg_error; + assign cfg_l1d_tile_sel_1_we = addr_hit[10] & reg_we & !reg_error; assign cfg_l1d_tile_sel_1_wd = reg_wdata[31:0]; - assign hw_barrier_participation_mask_0_we = addr_hit[10] & reg_we & !reg_error; + assign hw_barrier_participation_mask_0_we = addr_hit[11] & reg_we & !reg_error; assign hw_barrier_participation_mask_0_wd = reg_wdata[31:0]; - assign hw_barrier_participation_mask_1_we = addr_hit[11] & reg_we & !reg_error; + assign hw_barrier_participation_mask_1_we = addr_hit[12] & reg_we & !reg_error; assign hw_barrier_participation_mask_1_wd = reg_wdata[31:0]; - assign l1d_spm_commit_we = addr_hit[12] & reg_we & !reg_error; + assign l1d_spm_commit_we = addr_hit[13] & reg_we & !reg_error; assign l1d_spm_commit_wd = reg_wdata[0]; - assign l1d_insn_commit_we = addr_hit[13] & reg_we & !reg_error; + assign l1d_insn_commit_we = addr_hit[14] & reg_we & !reg_error; assign l1d_insn_commit_wd = reg_wdata[0]; - assign l1d_flush_status_re = addr_hit[14] & reg_re & !reg_error; + assign l1d_flush_status_re = addr_hit[15] & reg_re & !reg_error; - assign l1d_private_we = addr_hit[15] & reg_we & !reg_error; + assign l1d_private_we = addr_hit[16] & reg_we & !reg_error; assign l1d_private_wd = reg_wdata[3:0]; - assign l1d_addr_we = addr_hit[16] & reg_we & !reg_error; + assign l1d_addr_we = addr_hit[17] & reg_we & !reg_error; assign l1d_addr_wd = reg_wdata[31:0]; - assign xbar_offset_we = addr_hit[17] & reg_we & !reg_error; + assign xbar_offset_we = addr_hit[18] & reg_we & !reg_error; assign xbar_offset_wd = reg_wdata[4:0]; - assign xbar_offset_commit_we = addr_hit[18] & reg_we & !reg_error; + assign xbar_offset_commit_we = addr_hit[19] & reg_we & !reg_error; assign xbar_offset_commit_wd = reg_wdata[0]; // Read data return @@ -732,7 +754,7 @@ module cachepool_peripheral_reg_top #( end addr_hit[1]: begin - reg_rdata_next[0] = '0; + reg_rdata_next[31:0] = spatz_lock_qs; end addr_hit[2]: begin @@ -740,66 +762,70 @@ module cachepool_peripheral_reg_top #( end addr_hit[3]: begin - reg_rdata_next[31:0] = spatz_cycle_qs; + reg_rdata_next[0] = '0; end addr_hit[4]: begin - reg_rdata_next[31:0] = cluster_boot_control_qs; + reg_rdata_next[31:0] = spatz_cycle_qs; end addr_hit[5]: begin - reg_rdata_next[3:0] = cluster_eoc_exit_qs; + reg_rdata_next[31:0] = cluster_boot_control_qs; end addr_hit[6]: begin - reg_rdata_next[9:0] = cfg_l1d_spm_qs; + reg_rdata_next[3:0] = cluster_eoc_exit_qs; end addr_hit[7]: begin - reg_rdata_next[1:0] = cfg_l1d_insn_qs; + reg_rdata_next[9:0] = cfg_l1d_spm_qs; end addr_hit[8]: begin - reg_rdata_next[31:0] = cfg_l1d_tile_sel_0_qs; + reg_rdata_next[1:0] = cfg_l1d_insn_qs; end addr_hit[9]: begin - reg_rdata_next[31:0] = cfg_l1d_tile_sel_1_qs; + reg_rdata_next[31:0] = cfg_l1d_tile_sel_0_qs; end addr_hit[10]: begin - reg_rdata_next[31:0] = hw_barrier_participation_mask_0_qs; + reg_rdata_next[31:0] = cfg_l1d_tile_sel_1_qs; end addr_hit[11]: begin - reg_rdata_next[31:0] = hw_barrier_participation_mask_1_qs; + reg_rdata_next[31:0] = hw_barrier_participation_mask_0_qs; end addr_hit[12]: begin - reg_rdata_next[0] = l1d_spm_commit_qs; + reg_rdata_next[31:0] = hw_barrier_participation_mask_1_qs; end addr_hit[13]: begin - reg_rdata_next[0] = l1d_insn_commit_qs; + reg_rdata_next[0] = l1d_spm_commit_qs; end addr_hit[14]: begin - reg_rdata_next[0] = l1d_flush_status_qs; + reg_rdata_next[0] = l1d_insn_commit_qs; end addr_hit[15]: begin - reg_rdata_next[3:0] = l1d_private_qs; + reg_rdata_next[0] = l1d_flush_status_qs; end addr_hit[16]: begin - reg_rdata_next[31:0] = l1d_addr_qs; + reg_rdata_next[3:0] = l1d_private_qs; end addr_hit[17]: begin - reg_rdata_next[4:0] = xbar_offset_qs; + reg_rdata_next[31:0] = l1d_addr_qs; end addr_hit[18]: begin + reg_rdata_next[4:0] = xbar_offset_qs; + end + + addr_hit[19]: begin reg_rdata_next[0] = xbar_offset_commit_qs; end diff --git a/hardware/src/cachepool_cc_dual.sv b/hardware/src/cachepool_cc_dual.sv new file mode 100644 index 00000000..5426efef --- /dev/null +++ b/hardware/src/cachepool_cc_dual.sv @@ -0,0 +1,657 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Author: Diyou Shen + +`include "common_cells/assertions.svh" +`include "common_cells/registers.svh" +`include "snitch_vm/typedef.svh" +`include "reqrsp_interface/typedef.svh" + +/// CachePool Core Complex (dual flavor): 2 Snitch Integer Cores sharing +/// 1 Spatz Vector Unit, gated by cachepool_spatz_lock. cachepool_cc.sv +/// (single-hart) is untouched; this is a sibling module for tiles built +/// with num_scalar_per_core=2. +module cachepool_cc_dual + import snitch_pkg::interrupts_t; + import fpnew_pkg::fpu_implementation_t; #( + /// Address width of the buses + parameter int unsigned AddrWidth = 0, + /// Data width of the buses. + parameter int unsigned DataWidth = 0, + /// User width of the buses. + parameter int unsigned UserWidth = 0, + + parameter int unsigned SpmStackDepth = 512, + /// Data port request type. + parameter type dreq_t = logic, + /// Data port response type. + parameter type drsp_t = logic, + parameter type dreq_chan_t = logic, + parameter type drsp_chan_t = logic, + // TCDM port types + parameter type tcdm_req_t = logic, + parameter type tcdm_user_t = logic, + parameter type tcdm_req_chan_t = logic, + parameter type tcdm_rsp_t = logic, + parameter type tcdm_rsp_chan_t = logic, + /// TCDM Address Width + parameter int unsigned TCDMAddrWidth = 0, + parameter type hive_req_t = logic, + parameter type hive_rsp_t = logic, + parameter type acc_issue_req_t = logic, + parameter type acc_issue_rsp_t = logic, + parameter type acc_rsp_t = logic, + /// FPU configuration. + parameter fpu_implementation_t FPUImplementation = fpu_implementation_t'(0), + /// Boot address of core. + parameter logic [31:0] BootAddr = 32'h0000_1000, + + /// Address to indicate start of L2 + parameter logic [AddrWidth-1:0] UartAddr = 32'h0C00_0000, + /// Reduced-register extension + parameter bit RVE = 0, + /// Enable F and D Extension + parameter bit RVF = 1, + parameter bit RVD = 0, + parameter bit XDivSqrt = 0, + parameter bit XF8 = 0, + parameter bit XF16 = 0, + parameter bit XF16ALT = 0, + parameter bit XF8ALT = 0, + parameter int unsigned NumIntOutstandingLoads = 0, + parameter int unsigned NumIntOutstandingMem = 0, + parameter int unsigned NumSpatzOutstandingLoads = 0, + // Enable V Extension + parameter bit RVV = 1, + // Spatz paramaters + parameter int unsigned NumSpatzFPUs = 4, + parameter int unsigned NumSpatzIPUs = 1, + /// Add isochronous clock-domain crossings e.g., make it possible to operate + /// the core in a slower clock domain. + parameter bit IsoCrossing = 0, + /// Timing Parameters + /// Insert Pipeline registers into off-loading path (response) + parameter bit RegisterOffloadRsp = 0, + /// Insert Pipeline registers into data memory path (request) + parameter bit RegisterCoreReq = 0, + /// Insert Pipeline registers into data memory path (response) + parameter bit RegisterCoreRsp = 0, + parameter snitch_pma_pkg::snitch_pma_t SnitchPMACfg = '{default: 0}, + /// Derived parameter *Do not override* + parameter int unsigned NumSpatzFUs = (NumSpatzFPUs > NumSpatzIPUs) ? NumSpatzFPUs : NumSpatzIPUs, + parameter int unsigned NumMemPortsPerSpatz = NumSpatzFUs, + /// 2 dedicated scalar ports (one per host) instead of 1 merged one. + parameter int unsigned TCDMPorts = RVV ? NumMemPortsPerSpatz + 2 : 2, + parameter type addr_t = logic [AddrWidth-1:0], + parameter type req_id_t = logic [$clog2(NumSpatzOutstandingLoads)-1:0] + ) ( + input logic clk_i, + input logic rst_ni, + input logic testmode_i, + input logic [1:0][31:0] hart_id_i, + input interrupts_t [1:0] irq_i, + output hive_req_t [1:0] hive_req_o, + input hive_rsp_t [1:0] hive_rsp_i, + // Core data ports (one per host, bypasses cache/TCDM) + output dreq_t [1:0] data_req_o, + input drsp_t [1:0] data_rsp_i, + // TCDM Streamer Ports + output tcdm_req_t [TCDMPorts-1:0] tcdm_req_o, + input tcdm_rsp_t [TCDMPorts-1:0] tcdm_rsp_i, + // Response-side ready, reflecting real downstream capacity (Spatz + scalar ports). + output logic [TCDMPorts-1:0] tcdm_rsp_ready_o, + input addr_t tcdm_addr_base_i, + // Spatz lock/switch status (debug) + output logic owner_id_o, + output logic lock_error_o + ); + + // FMA architecture is "merged" -> mulexp and macexp instructions are supported + localparam bit FPEn = RVF | RVD | XF16 | XF8; + localparam int unsigned FLEN = + RVD ? 64 : // D ext. + RVF ? 32 : // F ext. + XF16 ? 16 : // Xf16 ext. + XF8 ? 8 : // Xf8 ext. + 0; // Unused in case of no FP + + `SNITCH_VM_TYPEDEF(AddrWidth) + + typedef logic [DataWidth-1:0] data_t; + typedef logic [DataWidth/8-1:0] strb_t; + + `REQRSP_TYPEDEF_ALL(reqrsp, addr_t, data_t, strb_t, tcdm_user_t) + + // Raw per-host Snitch data port, and its acc issue/response bundle. + dreq_t [1:0] snitch_dreq_d; + drsp_t [1:0] snitch_drsp_d; + + acc_issue_req_t [1:0] acc_snitch_req; + acc_issue_rsp_t [1:0] acc_snitch_rsp; + logic [1:0] acc_snitch_qvalid, acc_snitch_qready; + acc_rsp_t [1:0] acc_snitch_prsp; + logic [1:0] acc_snitch_pvalid, acc_snitch_pready; + + fpnew_pkg::roundmode_e [1:0] fpu_rnd_mode_h; + fpnew_pkg::fmt_mode_t [1:0] fpu_fmt_mode_h; + fpnew_pkg::status_t fpu_status; + + // Spatz memory-consistency signals (single Spatz instance). Forwarded to the + // owner only -- the non-owner never issued anything, so its own acc_mem_cnt_q + // never incremented; broadcasting these finish pulses to it too would + // decrement past 0 and roll over. The non-owner sees "nothing pending" instead. + logic [1:0] spatz_mem_finished; + logic [1:0] spatz_mem_str_finished; + logic spatz_st_rsp_done; + + logic [1:0][1:0] spatz_mem_finished_h; + logic [1:0][1:0] spatz_mem_str_finished_h; + logic [1:0] spatz_st_rsp_done_h; + + for (genvar h = 0; h < 2; h++) begin : gen_spatz_mem_consistency_gate + assign spatz_mem_finished_h[h] = (owner_id == h[0]) ? spatz_mem_finished : '0; + assign spatz_mem_str_finished_h[h] = (owner_id == h[0]) ? spatz_mem_str_finished : '0; + assign spatz_st_rsp_done_h[h] = (owner_id == h[0]) ? spatz_st_rsp_done : 1'b1; + end + + for (genvar h = 0; h < 2; h++) begin : gen_snitch + snitch #( + .AddrWidth (AddrWidth ), + .DataWidth (DataWidth ), + .acc_issue_req_t (acc_issue_req_t ), + .acc_issue_rsp_t (acc_issue_rsp_t ), + .acc_rsp_t (acc_rsp_t ), + .dreq_t (dreq_t ), + .drsp_t (drsp_t ), + .pa_t (pa_t ), + .l0_pte_t (l0_pte_t ), + .id_t (req_id_t ), + .BootAddr (BootAddr ), + .SnitchPMACfg (SnitchPMACfg ), + .NumIntOutstandingLoads (NumIntOutstandingLoads), + .NumIntOutstandingMem (NumIntOutstandingMem ), + .VMSupport (1'b0 ), + .RVE (RVE ), + .FP_EN (FPEn ), + .Xdma (1'b0 ), + .RVF (RVF ), + .RVD (RVD ), + .RVV (RVV ), + .XDivSqrt (XDivSqrt ), + .XF16 (XF16 ), + .XF16ALT (XF16ALT ), + .XF8 (XF8 ), + .XF8ALT (XF8ALT ), + .FLEN (FLEN ) + ) i_snitch ( + .clk_i (clk_i ), + .rst_i (!rst_ni ), + .hart_id_i (hart_id_i[h] ), + .irq_i (irq_i[h] ), + .flush_i_valid_o (hive_req_o[h].flush_i_valid ), + .flush_i_ready_i (hive_rsp_i[h].flush_i_ready ), + .inst_addr_o (hive_req_o[h].inst_addr ), + .inst_cacheable_o (hive_req_o[h].inst_cacheable), + .inst_data_i (hive_rsp_i[h].inst_data ), + .inst_valid_o (hive_req_o[h].inst_valid ), + .inst_ready_i (hive_rsp_i[h].inst_ready ), + .acc_qreq_o (acc_snitch_req[h] ), + .acc_qrsp_i (acc_snitch_rsp[h] ), + .acc_qvalid_o (acc_snitch_qvalid[h] ), + .acc_qready_i (acc_snitch_qready[h] ), + .acc_prsp_i (acc_snitch_prsp[h] ), + .acc_pvalid_i (acc_snitch_pvalid[h] ), + .acc_pready_o (acc_snitch_pready[h] ), + .acc_mem_finished_i (spatz_mem_finished_h[h] ), + .acc_mem_str_finished_i(spatz_mem_str_finished_h[h] ), + .acc_st_rsp_done_i (spatz_st_rsp_done_h[h] ), + .data_req_o (snitch_dreq_d[h] ), + .data_rsp_i (snitch_drsp_d[h] ), + .ptw_valid_o (hive_req_o[h].ptw_valid ), + .ptw_ready_i (hive_rsp_i[h].ptw_ready ), + .ptw_va_o (hive_req_o[h].ptw_va ), + .ptw_ppn_o (hive_req_o[h].ptw_ppn ), + .ptw_pte_i (hive_rsp_i[h].ptw_pte ), + .ptw_is_4mega_i (hive_rsp_i[h].ptw_is_4mega ), + .fpu_rnd_mode_o (fpu_rnd_mode_h[h] ), + .fpu_fmt_mode_o (fpu_fmt_mode_h[h] ), + .fpu_status_i (fpu_status ), + .core_events_o (/* unused */ ) + ); + + // There is no shared muldiv in this configuration + assign hive_req_o[h].acc_qvalid = 1'b0; + assign hive_req_o[h].acc_pready = 1'b0; + assign hive_req_o[h].acc_req = '0; + end + + // Spatz issue/response bundle, owner-routed by the lock. + acc_issue_req_t spatz_issue_req; + acc_issue_rsp_t spatz_issue_rsp; + logic spatz_issue_valid, spatz_issue_ready; + acc_rsp_t spatz_rsp; + logic spatz_rsp_valid, spatz_rsp_ready; + + // Lock-module pass-through data interface (accept/deliver on the module's + // own registered output side; see cachepool_spatz_lock.sv). + dreq_t [1:0] lock_out_req; + drsp_t [1:0] lock_out_rsp; + + logic owner_id, locked, waiting; + logic req_fire, rsp_fire; + assign owner_id_o = owner_id; + + cachepool_spatz_lock #( + .AddrWidth (AddrWidth ), + .NumHosts (2 ), + .dreq_t (dreq_t ), + .drsp_t (drsp_t ), + .dreq_chan_t (dreq_chan_t ), + .drsp_chan_t (drsp_chan_t ), + .user_t (tcdm_user_t ), + .RegisterCoreReq (RegisterCoreReq ), + .RegisterCoreRsp (RegisterCoreRsp ), + .addr_t (addr_t ) + ) i_spatz_lock ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .in_req_i (snitch_dreq_d ), + .in_rsp_o (snitch_drsp_d ), + .out_req_o (lock_out_req ), + .out_rsp_i (lock_out_rsp ), + .req_fire_i (req_fire ), + .rsp_fire_i (rsp_fire ), + .owner_id_o (owner_id ), + .locked_o (locked ), + .waiting_o (waiting ), + .error_o (lock_error_o ), + .cluster_periph_start_address_i (tcdm_addr_base_i ) + ); + + // Acc mux: owner's issue/response stream reaches Spatz. New requests (from anyone) + // are blocked while waiting; the owner's in-flight responses keep draining + // regardless. Non-owner is faked-accepted while unlocked, stalled+flagged + // (by the lock, via the stalled data-interface path) once locked. + assign req_fire = spatz_issue_valid & spatz_issue_ready; + assign rsp_fire = spatz_rsp_valid & spatz_rsp_ready; + + always_comb begin + spatz_issue_req = acc_snitch_req[owner_id]; + spatz_issue_valid = waiting ? 1'b0 : acc_snitch_qvalid[owner_id]; + spatz_rsp_ready = acc_snitch_pready[owner_id]; + + acc_snitch_qready = '0; + acc_snitch_rsp = '0; + acc_snitch_prsp = '0; + acc_snitch_pvalid = '0; + + acc_snitch_qready[owner_id] = waiting ? 1'b0 : spatz_issue_ready; + acc_snitch_rsp[owner_id] = spatz_issue_rsp; + acc_snitch_prsp[owner_id] = spatz_rsp; + acc_snitch_pvalid[owner_id] = spatz_rsp_valid; + + if (!locked && !waiting) begin + acc_snitch_qready[1] = acc_snitch_qvalid[1]; + acc_snitch_rsp[1] = '{accept: 1'b1, default: '0}; + end + end + + // FPU rounding-mode/format CSRs: Spatz has one FPU, so it only ever sees the + // current owner's config; its status broadcasts back to both hosts. + fpnew_pkg::roundmode_e fpu_rnd_mode; + fpnew_pkg::fmt_mode_t fpu_fmt_mode; + assign fpu_rnd_mode = fpu_rnd_mode_h[owner_id]; + assign fpu_fmt_mode = fpu_fmt_mode_h[owner_id]; + + dreq_t fp_lsu_mem_req; + drsp_t fp_lsu_mem_rsp; + + tcdm_req_chan_t [NumMemPortsPerSpatz-1:0] spatz_mem_req; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_req_valid; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_req_ready; + tcdm_rsp_chan_t [NumMemPortsPerSpatz-1:0] spatz_mem_rsp, spatz_mem_fifo; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_rsp_valid; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_rsp_ready; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_rsp_empty, spatz_mem_rsp_full; + logic [NumMemPortsPerSpatz-1:0] spatz_mem_rsp_pop, spatz_mem_rsp_push; + localparam int unsigned SpatzRspFifoDepth = + (NumSpatzOutstandingLoads > 0) ? NumSpatzOutstandingLoads : 1; + + spatz #( + .NrMemPorts (NumMemPortsPerSpatz ), + .NumOutstandingLoads(NumSpatzOutstandingLoads), + .FPUImplementation (FPUImplementation ), + .RegisterRsp (RegisterOffloadRsp ), + .dreq_t (dreq_t ), + .drsp_t (drsp_t ), + .spatz_mem_req_t (tcdm_req_chan_t ), + .spatz_mem_rsp_t (tcdm_rsp_chan_t ), + .spatz_issue_req_t (acc_issue_req_t ), + .spatz_issue_rsp_t (acc_issue_rsp_t ), + .spatz_rsp_t (acc_rsp_t ) + ) i_spatz ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .testmode_i (testmode_i ), + .hart_id_i (hart_id_i[owner_id] ), + .issue_valid_i (spatz_issue_valid ), + .issue_ready_o (spatz_issue_ready ), + .issue_req_i (spatz_issue_req ), + .issue_rsp_o (spatz_issue_rsp ), + .rsp_valid_o (spatz_rsp_valid ), + .rsp_ready_i (spatz_rsp_ready ), + .rsp_o (spatz_rsp ), + .spatz_mem_req_o (spatz_mem_req ), + .spatz_mem_req_valid_o (spatz_mem_req_valid ), + .spatz_mem_req_ready_i (spatz_mem_req_ready ), + .spatz_mem_rsp_i (spatz_mem_rsp ), + .spatz_mem_rsp_valid_i (spatz_mem_rsp_valid ), + .spatz_mem_rsp_ready_o (spatz_mem_rsp_ready ), + .spatz_mem_finished_o (spatz_mem_finished ), + .spatz_mem_str_finished_o(spatz_mem_str_finished), + .spatz_st_rsp_done_o (spatz_st_rsp_done ), + .fp_lsu_mem_req_o (fp_lsu_mem_req ), + .fp_lsu_mem_rsp_i (fp_lsu_mem_rsp ), + .fpu_rnd_mode_i (fpu_rnd_mode ), + .fpu_fmt_mode_i (fpu_fmt_mode ), + .fpu_status_o (fpu_status ) + ); + + // Vector-FU TCDM ports: pinned to host 0's routing row in the tile-level + // cache crossbar (see note.md "Software / core-id assignment") -- there is + // only one physical row for this CC pair regardless of current owner; the + // acc mux above already re-attributes the response to the true owner once + // it re-enters the CC. + for (genvar p = 0; p < NumMemPortsPerSpatz; p++) begin : gen_spatz_mem_ports + assign tcdm_req_o[p] = '{ + q : spatz_mem_req[p], + q_valid: spatz_mem_req_valid[p] + }; + assign spatz_mem_req_ready[p] = tcdm_rsp_i[p].q_ready; + + fifo_v3 #( + .dtype (tcdm_rsp_chan_t ), + .DEPTH (SpatzRspFifoDepth ), + .FALL_THROUGH (1 ) + ) i_spatz_rsp_fifo ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .flush_i (1'b0 ), + .testmode_i(1'b0 ), + .data_i (tcdm_rsp_i[p].p ), + .push_i (spatz_mem_rsp_push[p] ), + .data_o (spatz_mem_fifo[p] ), + .pop_i (spatz_mem_rsp_pop[p] ), + .full_o (spatz_mem_rsp_full[p] ), + .empty_o (spatz_mem_rsp_empty[p]), + .usage_o (/* Unused */ ) + ); + assign tcdm_rsp_ready_o[p] = !spatz_mem_rsp_full[p]; + + always_comb begin + spatz_mem_rsp_valid[p] = !spatz_mem_rsp_empty[p]; + spatz_mem_rsp[p] = spatz_mem_fifo[p]; + // Only push once the handshake completes (valid & ready), else the fifo overflows silently. + spatz_mem_rsp_push[p] = tcdm_rsp_i[p].p_valid & tcdm_rsp_ready_o[p]; + spatz_mem_rsp_pop[p] = spatz_mem_rsp_valid[p] & spatz_mem_rsp_ready[p]; + end + end + + // --------------------------------------------------------------------- + // Per-host scalar crossbar: {SnitchHMem, FPUMem-when-owner==h} -> + // {CacheMemH, SpmStackH, PeriphH}. Two small crossbars instead of one + // all-to-all one -- see note.md "Per-host crossbars". + // --------------------------------------------------------------------- + typedef enum integer { + SnitchMst = 0, + FpuMst = 1 + } scalar_mem_mst_e; + + typedef enum integer { + CacheMem = 0, + SpmStack = 1, + Periph = 2 + } scalar_mem_slv_e; + + localparam int unsigned NrScalarXbarMst = 2; + localparam int unsigned NrScalarXbarSlv = 3; + + localparam int unsigned SelectMstWidth = cf_math_pkg::idx_width(NrScalarXbarSlv); + localparam int unsigned SelectSlvWidth = cf_math_pkg::idx_width(NrScalarXbarMst); + typedef logic [SelectMstWidth-1:0] select_mst_t; + typedef logic [SelectSlvWidth-1:0] select_slv_t; + + localparam int unsigned StackAddrWidth = $clog2(SpmStackDepth); + typedef logic [StackAddrWidth-1:0] tcdm_stack_addr_t; + typedef struct packed { + tcdm_user_t user; + logic valid; + logic write; + } stack_meta_t; + + drsp_t [1:0] fpu_rsp_per_host; + + assign fp_lsu_mem_rsp = fpu_rsp_per_host[owner_id]; + + // Per-host signals, hoisted out of gen_host as 2D arrays (indexed [host][...]) + // for debug visibility, rather than living inside separate generate scopes. + dreq_t [1:0] fpu_req_gated; + + dreq_chan_t [1:0][NrScalarXbarMst-1:0] core_req_chan; + drsp_chan_t [1:0][NrScalarXbarMst-1:0] core_rsp_chan; + logic [1:0][NrScalarXbarMst-1:0] core_req_valid, core_req_ready; + logic [1:0][NrScalarXbarMst-1:0] core_rsp_valid, core_rsp_ready; + + dreq_chan_t [1:0][NrScalarXbarSlv-1:0] mem_req_chan; + drsp_chan_t [1:0][NrScalarXbarSlv-1:0] mem_rsp_chan; + logic [1:0][NrScalarXbarSlv-1:0] mem_req_valid, mem_req_ready; + logic [1:0][NrScalarXbarSlv-1:0] mem_rsp_valid, mem_rsp_ready; + + dreq_t [1:0][NrScalarXbarSlv-1:0] mem_req; + drsp_t [1:0][NrScalarXbarSlv-1:0] mem_rsp; + + select_mst_t [1:0][NrScalarXbarMst-1:0] core_req_sel; + select_slv_t [1:0][NrScalarXbarSlv-1:0] core_selected, mem_rsp_sel; + + logic [1:0] is_totstack; + + logic [1:0] stack_valid, stack_we; + tcdm_stack_addr_t [1:0] stack_add; + strb_t [1:0] stack_be; + data_t [1:0] stack_rdata, stack_wdata; + stack_meta_t [1:0] stack_req_meta, stack_rsp_meta; + tcdm_req_t [1:0] stack_req; + tcdm_rsp_t [1:0] stack_rsp; + + for (genvar h = 0; h < 2; h++) begin : gen_host + // FPU's request only ever contends on the crossbar matching the current owner. + assign fpu_req_gated[h] = fp_lsu_mem_req; + assign fpu_req_gated[h].q_valid = fp_lsu_mem_req.q_valid && (owner_id == h[0]); + + assign core_req_chan [h][SnitchMst] = lock_out_req[h].q; + assign core_req_valid [h][SnitchMst] = lock_out_req[h].q_valid; + assign core_rsp_ready [h][SnitchMst] = lock_out_req[h].p_ready; + assign lock_out_rsp[h] = '{ + p : core_rsp_chan [h][SnitchMst], + p_valid: core_rsp_valid [h][SnitchMst], + q_ready: core_req_ready [h][SnitchMst] + }; + + assign core_req_chan [h][FpuMst] = fpu_req_gated[h].q; + assign core_req_valid [h][FpuMst] = fpu_req_gated[h].q_valid; + assign core_rsp_ready [h][FpuMst] = fpu_req_gated[h].p_ready; + assign fpu_rsp_per_host[h] = '{ + p : core_rsp_chan [h][FpuMst], + p_valid: core_rsp_valid [h][FpuMst], + q_ready: core_req_ready [h][FpuMst] + }; + + // Give SpmStack higher priority for winning an overlap; TotStack-range + // addresses share the CacheMem slot with MainMem (patched below). + always_comb begin + for (int i = 0; i < NrScalarXbarMst; i++) begin + core_req_sel[h][i] = Periph; + if ((tcdm_addr_base_i + cachepool_pkg::TCDMSize - cachepool_pkg::SpmStackSize <= core_req_chan[h][i].addr) + && (core_req_chan[h][i].addr < tcdm_addr_base_i + cachepool_pkg::TCDMSize)) begin + core_req_sel[h][i] = SpmStack; + end else if ((tcdm_addr_base_i <= core_req_chan[h][i].addr) + && (core_req_chan[h][i].addr < tcdm_addr_base_i + cachepool_pkg::TotStackSize)) begin + core_req_sel[h][i] = CacheMem; + end else if ((cachepool_pkg::DramAddr <= core_req_chan[h][i].addr) + && (core_req_chan[h][i].addr < cachepool_pkg::DramAddr + cachepool_pkg::DramSize)) begin + core_req_sel[h][i] = CacheMem; + end + end + end + + reqrsp_xbar #( + .NumInp (NrScalarXbarMst ), + .NumOut (NrScalarXbarSlv ), + .PipeReg (1'b0 ), + .ExtReqPrio (1'b0 ), + .ExtRspPrio (1'b0 ), + .tcdm_req_chan_t (dreq_chan_t ), + .tcdm_rsp_chan_t (drsp_chan_t ) + ) i_scalar_xbar ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .slv_req_i (core_req_chan [h] ), + .slv_req_valid_i (core_req_valid [h] ), + .slv_req_ready_o (core_req_ready [h] ), + .slv_rsp_o (core_rsp_chan [h] ), + .slv_rsp_valid_o (core_rsp_valid [h] ), + .slv_rsp_ready_i (core_rsp_ready [h] ), + .slv_sel_i (core_req_sel [h] ), + .slv_rr_i ('0 ), + .slv_selected_o (core_selected [h] ), + .mst_req_o (mem_req_chan [h] ), + .mst_req_valid_o (mem_req_valid [h] ), + .mst_req_ready_i (mem_req_ready [h] ), + .mst_rsp_i (mem_rsp_chan [h] ), + .mst_rr_i ('0 ), + .mst_rsp_valid_i (mem_rsp_valid [h] ), + .mst_rsp_ready_o (mem_rsp_ready [h] ), + .mst_sel_i (mem_rsp_sel [h] ) + ); + + // TotStack-range hits within the (merged) CacheMem slot get the hart-id + // bits patched to this host's own id -- always host h's id, regardless + // of which master won, since the FPU only ever wins here when it's + // already gating on owner_id == h. + assign is_totstack[h] = (tcdm_addr_base_i <= mem_req_chan[h][CacheMem].addr) + && (mem_req_chan[h][CacheMem].addr < tcdm_addr_base_i + cachepool_pkg::TotStackSize); + + always_comb begin + for (int i = 0; i < NrScalarXbarSlv; i++) begin + mem_req[h][i].q = mem_req_chan [h][i]; + mem_req[h][i].q_valid = mem_req_valid[h][i]; + mem_req[h][i].p_ready = mem_rsp_ready[h][i]; + mem_req[h][i].q.user.is_fpu = (core_selected[h][i] == FpuMst); + end + if (is_totstack[h]) begin + mem_req[h][CacheMem].q.addr[($clog2(cachepool_pkg::TotStackSize)-1)-:$clog2(cachepool_pkg::NumCores)] = + hart_id_i[h][$clog2(cachepool_pkg::NumCores)-1:0]; + end + for (int i = 0; i < NrScalarXbarSlv; i++) begin + mem_rsp_chan [h][i] = mem_rsp[h][i].p; + mem_rsp_valid[h][i] = mem_rsp[h][i].p_valid; + mem_req_ready[h][i] = mem_rsp[h][i].q_ready; + mem_rsp_sel [h][i] = (mem_rsp[h][i].p.user.is_fpu == FpuMst); + end + end + + assign data_req_o[h] = mem_req[h][Periph]; + assign mem_rsp[h][Periph] = data_rsp_i[h]; + + // Host's own dedicated Stack SPM + reqrsp_to_tcdm #( + .AddrWidth (AddrWidth ), + .DataWidth (DataWidth ), + .BufDepth (4 ), + .UserWidth ($bits(tcdm_user_t) ), + .reqrsp_req_t (dreq_t ), + .reqrsp_rsp_t (drsp_t ), + .tcdm_req_t (tcdm_req_t ), + .tcdm_rsp_t (tcdm_rsp_t ) + ) i_core_to_stack ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .reqrsp_req_i (mem_req[h][SpmStack] ), + .reqrsp_rsp_o (mem_rsp[h][SpmStack] ), + .tcdm_req_o (stack_req[h] ), + .tcdm_rsp_i (stack_rsp[h] ) + ); + + assign stack_valid[h] = stack_req[h].q_valid; + assign stack_we[h] = stack_req[h].q.write; + assign stack_add[h] = stack_req[h].q.addr[StackAddrWidth+1:2]; + assign stack_wdata[h] = stack_req[h].q.data; + assign stack_be[h] = stack_req[h].q.strb; + + assign stack_req_meta[h] = '{ + user: stack_req[h].q.user, + valid: stack_req[h].q_valid, + write: stack_req[h].q.write + }; + + assign stack_rsp[h].p.data = stack_rdata[h]; + assign stack_rsp[h].p.user = stack_rsp_meta[h].user; + assign stack_rsp[h].p.write = stack_rsp_meta[h].write; + assign stack_rsp[h].p_valid = stack_rsp_meta[h].valid; + assign stack_rsp[h].q_ready = 1'b1; + + tc_sram_impl #( + .NumWords (SpmStackDepth ), + .DataWidth (DataWidth ), + .ByteWidth (8 ), + .NumPorts (1 ), + .Latency (1 ), + .SimInit ("zeros" ) + ) i_spm_mem ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .impl_i ('0 ), + .impl_o (/* Unused */ ), + .req_i (stack_valid[h] ), + .we_i (stack_we[h] ), + .addr_i (stack_add[h] ), + .wdata_i (stack_wdata[h] ), + .be_i (stack_be[h] ), + .rdata_o (stack_rdata[h] ) + ); + + shift_reg #( + .dtype (stack_meta_t ), + .Depth (1 ) + ) i_req_meta_pipe ( + .clk_i (clk_i ), + .rst_ni(rst_ni ), + .d_i (stack_req_meta[h] ), + .d_o (stack_rsp_meta[h] ) + ); + + // Host's own dedicated scalar TCDM port to the cache. + reqrsp_to_tcdm #( + .AddrWidth (AddrWidth ), + .DataWidth (DataWidth ), + .BufDepth (4 ), + .reqrsp_req_t (dreq_t ), + .reqrsp_rsp_t (drsp_t ), + .UserWidth ($bits(tcdm_user_t) ), + .tcdm_req_t (tcdm_req_t ), + .tcdm_rsp_t (tcdm_rsp_t ) + ) i_reqrsp_to_tcdm ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .reqrsp_req_i (mem_req[h][CacheMem] ), + .reqrsp_rsp_o (mem_rsp[h][CacheMem] ), + .tcdm_req_o (tcdm_req_o[NumMemPortsPerSpatz+h] ), + .tcdm_rsp_i (tcdm_rsp_i[NumMemPortsPerSpatz+h] ) + ); + // Scalar port: left on its existing unconditional-accept behavior for now. + assign tcdm_rsp_ready_o[NumMemPortsPerSpatz+h] = 1'b1; + end + +endmodule diff --git a/hardware/src/cachepool_pkg.sv b/hardware/src/cachepool_pkg.sv index a7b38cd4..edc34d5b 100644 --- a/hardware/src/cachepool_pkg.sv +++ b/hardware/src/cachepool_pkg.sv @@ -42,6 +42,10 @@ package cachepool_pkg; localparam int unsigned NFpu = `ifdef SPATZ_NUM_FPU `SPATZ_NUM_FPU `else 0 `endif; localparam int unsigned NIpu = `ifdef SPATZ_NUM_IPU `SPATZ_NUM_IPU `else 1 `endif; + // Snitch scalar cores sharing one Spatz per CC (1 = cachepool_cc, 2 = cachepool_cc_dual). + // Whole-build choice, not per-tile; default 1 until config.mk wires a real value through. + localparam int unsigned NumScalarPerCC = `ifdef NUM_SCALAR_PER_CC `NUM_SCALAR_PER_CC `else 1 `endif; + localparam int unsigned NumIntOutstandingLoads = `ifdef SNITCH_MAX_TRANS `SNITCH_MAX_TRANS `else 0 `endif; localparam int unsigned NumIntOutstandingMem = `ifdef SNITCH_MAX_TRANS `SNITCH_MAX_TRANS `else 0 `endif; localparam int unsigned NumSpatzOutstandingLoads = `ifdef SPATZ_MAX_TRANS `SPATZ_MAX_TRANS `else 0 `endif; @@ -60,8 +64,9 @@ package cachepool_pkg; // How many cores within a tile? This is used to select the ports within a tile. localparam int unsigned LogNumCoresTile = $clog2(NumCoresTile); - // 4 ports from Spatz + 1 shared port from Snitch/FPU - localparam int unsigned NrTCDMPortsPerCore = 5; + // Spatz vector-FU ports (shared, one CC-slot's worth) + one dedicated scalar + // port per hart sharing that Spatz. + localparam int unsigned NrTCDMPortsPerCore = (NFpu > NIpu ? NFpu : NIpu) + NumScalarPerCC; // Intra-group remote ports per tile, in total. localparam int unsigned NumLGPortTile = NumLGPortCore * NrTCDMPortsPerCore; diff --git a/hardware/src/cachepool_spatz_lock.sv b/hardware/src/cachepool_spatz_lock.sv new file mode 100644 index 00000000..90c4f804 --- /dev/null +++ b/hardware/src/cachepool_spatz_lock.sv @@ -0,0 +1,329 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Author: Diyou Shen + +`include "common_cells/assertions.svh" +`include "common_cells/registers.svh" + +/// Spatz ownership lock/switch for a dual-Snitch cachepool_cc_dual. +/// Host 0 is the implicit owner while Idle; a write of 1 to the lock +/// address acquires ownership (blocks until granted), a write of 0 +/// releases it (owner only). Pure arbiter: does not route the acc/data +/// traffic itself (that lives in cachepool_cc_dual.sv, alongside its other +/// owner-based muxing) -- only decides ownership and exposes owner_id_o/ +/// locked_o/waiting_o for the CC to apply, and counts owner-path acc +/// handshakes (via req_fire_i/rsp_fire_i, fed back by the CC) to gate a +/// switch until fully drained. +/// Also hosts the pass-through memory-path register cuts (moved here from +/// the per-core spill registers in cachepool_cc.sv). +module cachepool_spatz_lock + import cachepool_peripheral_reg_pkg::*; +#( + parameter int unsigned AddrWidth = 0, + parameter int unsigned NumHosts = 2, + parameter type dreq_t = logic, + parameter type drsp_t = logic, + parameter type dreq_chan_t = logic, + parameter type drsp_chan_t = logic, + parameter type user_t = logic, + parameter bit RegisterCoreReq = 1'b0, + parameter bit RegisterCoreRsp = 1'b0, + /// Derived parameter *Do not override* + parameter type addr_t = logic [AddrWidth-1:0] +) ( + input logic clk_i, + input logic rst_ni, + + // memory interface, used to set the lock + input dreq_t [NumHosts-1:0] in_req_i, + output drsp_t [NumHosts-1:0] in_rsp_o, + output dreq_t [NumHosts-1:0] out_req_o, + input drsp_t [NumHosts-1:0] out_rsp_i, + + // Owner-path acc handshake fires, fed back by the CC (which does the + // actual acc routing) purely for drain counting. + input logic req_fire_i, + input logic rsp_fire_i, + + // owner/lock status and error + output logic owner_id_o, + output logic locked_o, + output logic waiting_o, + output logic error_o, + + // peripheral base address + input addr_t cluster_periph_start_address_i +); + + addr_t lock_addr; + assign lock_addr = cluster_periph_start_address_i + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_OFFSET; + + // Single lock FSM. Idle/Locked are the two "real" states (host 0 is the + // implicit owner while Idle); AcqWait/RelWait are drain-wait sub-states + // on the way to an actual ownership switch. + // Idle -(acquire, other host)-> AcqWait -(drain_done)-> Locked (new owner) + // Locked -(release, owner) -> RelWait -(drain_done)-> Idle (owner = host 0) + // Self-acquire (already owner) and read complete without a drain wait. + // A non-owner acquire/release, or any lock op during a wait, is stalled + // (never accepted) and sets the sticky error_o. + // + // Handshake timing: the q-channel is accepted (q_ready=1) exactly on the + // deciding cycle (the last cycle of a wait, or immediately for the + // no-wait cases); the response (p_valid=1) is then held starting the + // following cycle until the host takes it via p_ready. + typedef enum logic [1:0] { + Idle, + AcqWait, + RelWait, + Locked + } lock_state_e; + + lock_state_e lock_d, lock_q; + logic owner_d, owner_q; + logic error_d, error_q; + logic active_d, active_q; + user_t user_d, user_q; + + assign owner_id_o = owner_q; + assign locked_o = (lock_q == Locked); + assign waiting_o = waiting; + assign error_o = error_q; + + // Outstanding acc handshakes on the owner's path; must reach 0 before a wait can complete. + logic [7:0] outstanding_d, outstanding_q; + logic drain_done, waiting; + + assign waiting = (lock_q == AcqWait) || (lock_q == RelWait); + assign drain_done = (outstanding_q == '0); + + `ASSERT(NoOutstandingUnderflow, rsp_fire_i |-> (outstanding_q != '0)) + + always_comb begin + outstanding_d = outstanding_q; + if (req_fire_i && !rsp_fire_i) begin + // one insn issued + outstanding_d = outstanding_q + 8'd1; + end else if (!req_fire_i && rsp_fire_i) begin + // one insn finished + outstanding_d = outstanding_q - 8'd1; + end + end + + // write 1 for acquire, write 0 for release, read returns the lock status + logic [1:0] is_acquire, is_release, is_read, lock_hit; + + for (genvar i = 0; i < 2; i++) begin + assign is_acquire[i] = in_req_i[i].q.write && in_req_i[i].q.data[0]; + assign is_release[i] = in_req_i[i].q.write && !in_req_i[i].q.data[0]; + assign is_read[i] = !in_req_i[i].q.write; + assign lock_hit[i] = in_req_i[i].q_valid && (in_req_i[i].q.addr == lock_addr); + end + + // The owner's hit always wins arbitration, so a non-owner's (always-illegal-in-Locked) + // request can never starve the owner's legitimate one. + logic hit_any, winner; + assign hit_any = lock_hit[0] || lock_hit[1]; + assign winner = lock_hit[owner_q] ? owner_q : ~owner_q; + + // Pending, not-yet-delivered completion response (one at a time; a new lock op is + // only arbitrated once the previous one's response has been taken). + logic resp_pending_d, resp_pending_q; + logic resp_host_d, resp_host_q; + user_t resp_user_d, resp_user_q; + logic resp_read_d, resp_read_q; + logic [1:0] resp_status_d, resp_status_q; + + // Register cuts on the pass-through path (moved here from cachepool_cc.sv's per-core + // spill registers), placed at the module's output side. + logic [1:0] mem_req_valid, mem_req_ready, mem_rsp_valid; + drsp_chan_t [1:0] mem_rsp_chan; + + for (genvar i = 0; i < 2; i++) begin : gen_out_cut + assign mem_req_valid[i] = in_req_i[i].q_valid && !lock_hit[i]; + + spill_register #( + .T ( dreq_chan_t ), + .Bypass ( !RegisterCoreReq ) + ) i_spill_register_req ( + .clk_i, + .rst_ni, + .valid_i ( mem_req_valid[i] ), + .ready_o ( mem_req_ready[i] ), + .data_i ( in_req_i[i].q ), + .valid_o ( out_req_o[i].q_valid ), + .ready_i ( out_rsp_i[i].q_ready ), + .data_o ( out_req_o[i].q ) + ); + + spill_register #( + .T ( drsp_chan_t ), + .Bypass ( !RegisterCoreRsp ) + ) i_spill_register_rsp ( + .clk_i, + .rst_ni, + .valid_i ( out_rsp_i[i].p_valid ), + .ready_o ( out_req_o[i].p_ready ), + .data_i ( out_rsp_i[i].p ), + .valid_o ( mem_rsp_valid[i] ), + .ready_i ( in_req_i[i].p_ready ), + .data_o ( mem_rsp_chan[i] ) + ); + end + + always_comb begin + lock_d = lock_q; + owner_d = owner_q; + error_d = error_q; + active_d = active_q; + user_d = user_q; + resp_pending_d = resp_pending_q; + resp_host_d = resp_host_q; + resp_user_d = resp_user_q; + resp_read_d = resp_read_q; + resp_status_d = resp_status_q; + + // Default: pass through the (registered) memory path; blocked on a lock-address hit. + for (int i = 0; i < 2; i++) begin + in_rsp_o[i] = '0; + in_rsp_o[i].q_ready = lock_hit[i] ? 1'b0 : mem_req_ready[i]; + in_rsp_o[i].p_valid = mem_rsp_valid[i]; + in_rsp_o[i].p = mem_rsp_chan[i]; + end + + // Deliver a pending completion response, starting the cycle after it was accepted. + if (resp_pending_q) begin + in_rsp_o[resp_host_q] = '0; + in_rsp_o[resp_host_q].p_valid = 1'b1; + in_rsp_o[resp_host_q].p.user = resp_user_q; + if (resp_read_q) begin + in_rsp_o[resp_host_q].p.data[0] = resp_status_q[0]; + in_rsp_o[resp_host_q].p.data[1] = resp_status_q[1]; + end + if (in_req_i[resp_host_q].p_ready) resp_pending_d = 1'b0; + end + + case (lock_q) + // Host 0 is the implicit owner. Its own acquire/release are no-ops (immediate + // grant); host 1 may acquire (drains, then switches); host 1's release is + // meaningless (nothing to release) and stalled. + Idle: begin + if (hit_any && !resp_pending_q) begin + if (is_acquire[winner]) begin + if (winner == owner_q) begin + // host 0 self-acquire: immediate grant + lock_d = Locked; + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + resp_read_d = 1'b0; + end else begin + // host 1 wants the lock -> drain, then grant + active_d = winner; + user_d = in_req_i[winner].q.user; + lock_d = AcqWait; + end + end else if (is_release[winner]) begin + if (winner == owner_q) begin + // "self release": nothing to release, immediate no-op grant + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + resp_read_d = 1'b0; + end else begin + // host 1 release: stalled, never accepted + error_d = 1'b1; + end + end else if (is_read[winner]) begin + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + resp_read_d = 1'b1; + resp_status_d = {1'b0, owner_q}; + end + end + end + + // Only the owner's acquire (no-op) or release (drains, reverts to Idle) is ever + // legal; a non-owner's acquire/release is stalled and flagged. + Locked: begin + if (hit_any && !resp_pending_q) begin + if (is_acquire[winner]) begin + if (winner == owner_q) begin + // owner self-acquire: no-op, immediate grant + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + resp_read_d = 1'b0; + end else begin + // non-owner acquire: stalled, never accepted + error_d = 1'b1; + end + end else if (is_release[winner]) begin + if (winner == owner_q) begin + // owner release -> drain, then revert to Idle + active_d = winner; + user_d = in_req_i[winner].q.user; + lock_d = RelWait; + end else begin + // non-owner release: stalled, never accepted + error_d = 1'b1; + end + end else if (is_read[winner]) begin + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + resp_read_d = 1'b1; + resp_status_d = {1'b1, owner_q}; + end + end + end + + // No lock op is accepted from anyone while waiting. + AcqWait: begin + if (drain_done) begin + owner_d = active_q; + lock_d = Locked; + in_rsp_o[active_q].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = active_q; + resp_user_d = user_q; + resp_read_d = 1'b0; + end + end + + RelWait: begin + if (drain_done) begin + owner_d = 1'b0; + lock_d = Idle; + in_rsp_o[active_q].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = active_q; + resp_user_d = user_q; + resp_read_d = 1'b0; + end + end + + default: lock_d = Idle; + endcase + end + + `FF(lock_q, lock_d, Idle, clk_i, rst_ni) + `FF(owner_q, owner_d, 1'b0, clk_i, rst_ni) + `FF(error_q, error_d, 1'b0, clk_i, rst_ni) + `FF(active_q, active_d, 1'b0, clk_i, rst_ni) + `FF(user_q, user_d, '0, clk_i, rst_ni) + `FF(outstanding_q, outstanding_d, '0, clk_i, rst_ni) + `FF(resp_pending_q, resp_pending_d, 1'b0, clk_i, rst_ni) + `FF(resp_host_q, resp_host_d, 1'b0, clk_i, rst_ni) + `FF(resp_user_q, resp_user_d, '0, clk_i, rst_ni) + `FF(resp_read_q, resp_read_d, 1'b0, clk_i, rst_ni) + `FF(resp_status_q, resp_status_d, '0, clk_i, rst_ni) + +endmodule diff --git a/hardware/src/cachepool_tile.sv b/hardware/src/cachepool_tile.sv index eda36f64..260da607 100644 --- a/hardware/src/cachepool_tile.sv +++ b/hardware/src/cachepool_tile.sv @@ -35,8 +35,9 @@ module cachepool_tile /// Number of Core Complex (CC) slots in this tile. parameter int unsigned NumCC = 8, /// Number of Snitch scalar cores sharing one Spatz per CC (1 = today's - /// cachepool_cc, 2 = cachepool_cc_dual). Homogeneous per tile. - parameter int unsigned NumScalarPerCC = 1, + /// cachepool_cc, 2 = cachepool_cc_dual). Whole-build choice; defaults to + /// the package-level constant (single source of truth), not per-tile. + parameter int unsigned NumScalarPerCC = cachepool_pkg::NumScalarPerCC, /// Data/TCDM memory depth per cut (in words). parameter int unsigned TCDMDepth = 1024, /// Cluster peripheral address region size (in kB). @@ -218,7 +219,7 @@ module cachepool_tile localparam int unsigned NrSuperBanks = NrBanks / BanksPerSuperBank; function automatic int unsigned get_tcdm_ports(int unsigned core); - return spatz_pkg::N_FU + 1; + return spatz_pkg::N_FU + NumScalarPerCC; endfunction function automatic int unsigned get_tcdm_port_offs(int unsigned core_idx); @@ -355,13 +356,12 @@ module cachepool_tile tcdm_req_t [NrTCDMPortsCores-1:0] tcdm_req; tcdm_rsp_t [NrTCDMPortsCores-1:0] tcdm_rsp; - core_events_t [NumCC-1:0] core_events; - // snitch_icache_pkg::icache_events_t [NumCC-1:0] icache_events; - // 4. Memory Subsystem (Core side). - reqrsp_req_t [NumCC-1:0] core_req, filtered_core_req; - reqrsp_rsp_t [NumCC-1:0] core_rsp, filtered_core_rsp; + // 4. Memory Subsystem (Core side). Per-hart (not per-CC): each hart needs + // its own Periph/barrier port, one per hart sharing a CC included. + reqrsp_req_t [NrHarts-1:0] core_req, filtered_core_req; + reqrsp_rsp_t [NrHarts-1:0] core_rsp, filtered_core_rsp; // 8. L1 D$ @@ -377,8 +377,11 @@ module cachepool_tile tcdm_req_t [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_ctrl_req; tcdm_rsp_t [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_bank_rsp; - tcdm_req_t [NumL1CtrlTile-1:0] cache_amo_req; - tcdm_rsp_t [NumL1CtrlTile-1:0] cache_amo_rsp; + // One AMO-capable path per scalar port sharing this CC (indexed by the + // same j as cache_ctrl_req/cache_bank_rsp; only the last NumScalarPerCC + // planes are ever driven, see gen_cache_amo_connect). + tcdm_req_t [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_amo_req; + tcdm_rsp_t [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_amo_rsp; logic [NumL1CtrlTile-1:0][NrTCDMPortsPerCore-1:0] cache_req_valid; @@ -428,7 +431,7 @@ module cachepool_tile // Per-core response-side readiness, driven by each i_cachepool_cc instance. logic [NrTCDMPortsCores-1:0] tcdm_rsp_ready; logic [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_pready, cache_xbar_pready; - logic [NumL1CtrlTile-1:0] cache_amo_pready; + logic [NrTCDMPortsPerCore-1:0][NumL1CtrlTile-1:0] cache_amo_pready; always_comb begin : cache_flush_protection for (int j = 0; unsigned'(j) < NrTCDMPortsCores; j++) begin @@ -755,7 +758,10 @@ module cachepool_tile // Ports from Spatz can bypass this module for (genvar j = 0; j < NrTCDMPortsPerCore; j++) begin : gen_cache_amo_connect - if (j == NrTCDMPortsPerCore-1) begin : gen_amo + // One AMO-capable path per scalar port sharing this CC (the last + // NumScalarPerCC planes) -- each hart's own scalar port needs + // independent AMO capability, not just a single shared one. + if (j >= NrTCDMPortsPerCore - NumScalarPerCC) begin : gen_amo spatz_cache_amo #( .DataWidth ( DataWidth ), .CoreIDWidth ( CoreIDWidth ), @@ -770,9 +776,9 @@ module cachepool_tile .core_req_i (cache_ctrl_req [j][cb] ), .core_rsp_ready_i (cache_xbar_pready[j][cb] ), .core_rsp_o (cache_bank_rsp [j][cb] ), - .mem_req_o (cache_amo_req [cb] ), - .mem_rsp_ready_o (cache_amo_pready [cb] ), - .mem_rsp_i (cache_amo_rsp [cb] ) + .mem_req_o (cache_amo_req [j][cb] ), + .mem_rsp_ready_o (cache_amo_pready [j][cb] ), + .mem_rsp_i (cache_amo_rsp [j][cb] ) ); tcdm_req_t cache_req_reg; @@ -783,27 +789,27 @@ module cachepool_tile .Bypass ( 1'b0 ) ) i_spill_reg_cache_req ( .clk_i , - .rst_ni ( rst_ni ), - .valid_i ( cache_amo_req[cb].q_valid ), - .ready_o ( cache_amo_rsp[cb].q_ready ), - .data_i ( cache_amo_req[cb].q ), - .valid_o ( cache_req_reg.q_valid ), - .ready_i ( cache_rsp_reg.q_ready ), - .data_o ( cache_req_reg.q ) + .rst_ni ( rst_ni ), + .valid_i ( cache_amo_req[j][cb].q_valid ), + .ready_o ( cache_amo_rsp[j][cb].q_ready ), + .data_i ( cache_amo_req[j][cb].q ), + .valid_o ( cache_req_reg.q_valid ), + .ready_i ( cache_rsp_reg.q_ready ), + .data_o ( cache_req_reg.q ) ); spill_register #( .T ( tcdm_rsp_chan_t ), .Bypass ( 1'b1 ) ) i_spill_reg_cache_rsp ( - .clk_i ( clk_i ), - .rst_ni ( rst_ni ), - .valid_i ( cache_rsp_reg.p_valid ), - .ready_o ( cache_rsp_ready [cb][j] ), - .data_i ( cache_rsp_reg.p ), - .valid_o ( cache_amo_rsp [cb].p_valid), - .ready_i ( cache_amo_pready[cb] ), - .data_o ( cache_amo_rsp [cb].p ) + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .valid_i ( cache_rsp_reg.p_valid ), + .ready_o ( cache_rsp_ready [cb][j] ), + .data_i ( cache_rsp_reg.p ), + .valid_o ( cache_amo_rsp [j][cb].p_valid), + .ready_i ( cache_amo_pready[j][cb] ), + .data_o ( cache_amo_rsp [j][cb].p ) ); assign cache_req_valid[cb][j] = cache_req_reg.q_valid; @@ -1459,89 +1465,176 @@ module cachepool_tile for (genvar i = 0; i < NumCC; i++) begin : gen_cc localparam int unsigned TcdmPorts = get_tcdm_ports(i); localparam int unsigned TcdmPortsOffs = get_tcdm_port_offs(i); - // First hart index served by this CC slot. Only this hart is wired up - // for now (NumScalarPerCC>1's extra hart slots are added separately). + // First hart index served by this CC slot (the only one if + // NumScalarPerCC==1; cachepool_cc_dual below wires up HartIdx+1 too). localparam int unsigned HartIdx = i * NumScalarPerCC; - interrupts_t irq; + interrupts_t irq0; sync #(.STAGES (2)) - i_sync_debug (.clk_i, .rst_ni, .serial_i (debug_req_i), .serial_o (irq.debug)); + i_sync_debug0 (.clk_i, .rst_ni, .serial_i (debug_req_i), .serial_o (irq0.debug)); sync #(.STAGES (2)) - i_sync_meip (.clk_i, .rst_ni, .serial_i (meip_i), .serial_o (irq.meip)); + i_sync_meip0 (.clk_i, .rst_ni, .serial_i (meip_i), .serial_o (irq0.meip)); sync #(.STAGES (2)) - i_sync_mtip (.clk_i, .rst_ni, .serial_i (mtip_i), .serial_o (irq.mtip)); + i_sync_mtip0 (.clk_i, .rst_ni, .serial_i (mtip_i), .serial_o (irq0.mtip)); sync #(.STAGES (2)) - i_sync_msip (.clk_i, .rst_ni, .serial_i (msip_i), .serial_o (irq.msip)); - assign irq.mcip = cl_interrupt_i[HartIdx]; + i_sync_msip0 (.clk_i, .rst_ni, .serial_i (msip_i), .serial_o (irq0.msip)); + assign irq0.mcip = cl_interrupt_i[HartIdx]; tcdm_req_t [TcdmPorts-1:0] tcdm_req_wo_user; - logic [31:0] hart_id; - assign hart_id = hart_base_id_i + HartIdx; - - cachepool_cc #( - .BootAddr (BootAddr ), - .UartAddr (UartAddr ), - .RVE (1'b0 ), - .RVF (RVF ), - .RVD (RVD ), - .RVV (RVV ), - .AddrWidth (AxiAddrWidth ), - .DataWidth (NarrowDataWidth ), - .UserWidth (AxiUserWidth ), - .SnitchPMACfg (SnitchPMACfg ), - .dreq_t (reqrsp_req_t ), - .drsp_t (reqrsp_rsp_t ), - .dreq_chan_t (reqrsp_req_chan_t ), - .drsp_chan_t (reqrsp_rsp_chan_t ), - .tcdm_req_t (tcdm_req_t ), - .tcdm_user_t (tcdm_user_t ), - .tcdm_req_chan_t (tcdm_req_chan_t ), - .tcdm_rsp_t (tcdm_rsp_t ), - .tcdm_rsp_chan_t (tcdm_rsp_chan_t ), - .axi_req_t (axi_mst_tile_wide_req_t ), - .axi_ar_chan_t (axi_mst_tile_wide_ar_chan_t), - .axi_aw_chan_t (axi_mst_tile_wide_aw_chan_t), - .axi_rsp_t (axi_mst_tile_wide_resp_t ), - .hive_req_t (hive_req_t ), - .hive_rsp_t (hive_rsp_t ), - .acc_issue_req_t (acc_issue_req_t ), - .acc_issue_rsp_t (acc_issue_rsp_t ), - .acc_rsp_t (acc_rsp_t ), - .XDivSqrt (1'b0 ), - .XF16 (1'b1 ), - .XF16ALT (1'b0 ), - .XF8 (1'b1 ), - .XF8ALT (1'b0 ), - .IsoCrossing (1'b0 ), - .NumIntOutstandingLoads (NumIntOutstandingLoads ), - .NumIntOutstandingMem (NumIntOutstandingMem ), - .NumSpatzOutstandingLoads(NumSpatzOutstandingLoads ), - .FPUImplementation (FPUImplementation ), - .SpmStackDepth (SpmStackDepth ), - .RegisterOffloadRsp (RegisterOffloadRsp ), - .RegisterCoreReq (RegisterCoreReq ), - .RegisterCoreRsp (RegisterCoreRsp ), - .NumSpatzFPUs (NumSpatzFPUs ), - .NumSpatzIPUs (NumSpatzIPUs ), - .TCDMAddrWidth (SPMAddrWidth ) - ) i_cachepool_cc ( - .clk_i (clk_i ), - .rst_ni (rst_ni ), - .testmode_i (1'b0 ), - .hart_id_i (hart_id ), - .hive_req_o (hive_req[HartIdx] ), - .hive_rsp_i (hive_rsp[HartIdx] ), - .irq_i (irq ), - .data_req_o (core_req[i] ), - .data_rsp_i (core_rsp[i] ), - .tcdm_req_o (tcdm_req_wo_user ), - .tcdm_rsp_i (tcdm_rsp[TcdmPortsOffs +: TcdmPorts] ), - .tcdm_rsp_ready_o (tcdm_rsp_ready[TcdmPortsOffs +: TcdmPorts] ), - .core_events_o (core_events[i] ), - .tcdm_addr_base_i (tcdm_start_address ) - ); + logic [31:0] hart_id0; + assign hart_id0 = hart_base_id_i + HartIdx; + + if (NumScalarPerCC == 1) begin : gen_single + cachepool_cc #( + .BootAddr (BootAddr ), + .UartAddr (UartAddr ), + .RVE (1'b0 ), + .RVF (RVF ), + .RVD (RVD ), + .RVV (RVV ), + .AddrWidth (AxiAddrWidth ), + .DataWidth (NarrowDataWidth ), + .UserWidth (AxiUserWidth ), + .SnitchPMACfg (SnitchPMACfg ), + .dreq_t (reqrsp_req_t ), + .drsp_t (reqrsp_rsp_t ), + .dreq_chan_t (reqrsp_req_chan_t ), + .drsp_chan_t (reqrsp_rsp_chan_t ), + .tcdm_req_t (tcdm_req_t ), + .tcdm_user_t (tcdm_user_t ), + .tcdm_req_chan_t (tcdm_req_chan_t ), + .tcdm_rsp_t (tcdm_rsp_t ), + .tcdm_rsp_chan_t (tcdm_rsp_chan_t ), + .axi_req_t (axi_mst_tile_wide_req_t ), + .axi_ar_chan_t (axi_mst_tile_wide_ar_chan_t), + .axi_aw_chan_t (axi_mst_tile_wide_aw_chan_t), + .axi_rsp_t (axi_mst_tile_wide_resp_t ), + .hive_req_t (hive_req_t ), + .hive_rsp_t (hive_rsp_t ), + .acc_issue_req_t (acc_issue_req_t ), + .acc_issue_rsp_t (acc_issue_rsp_t ), + .acc_rsp_t (acc_rsp_t ), + .XDivSqrt (1'b0 ), + .XF16 (1'b1 ), + .XF16ALT (1'b0 ), + .XF8 (1'b1 ), + .XF8ALT (1'b0 ), + .IsoCrossing (1'b0 ), + .NumIntOutstandingLoads (NumIntOutstandingLoads ), + .NumIntOutstandingMem (NumIntOutstandingMem ), + .NumSpatzOutstandingLoads(NumSpatzOutstandingLoads ), + .FPUImplementation (FPUImplementation ), + .SpmStackDepth (SpmStackDepth ), + .RegisterOffloadRsp (RegisterOffloadRsp ), + .RegisterCoreReq (RegisterCoreReq ), + .RegisterCoreRsp (RegisterCoreRsp ), + .NumSpatzFPUs (NumSpatzFPUs ), + .NumSpatzIPUs (NumSpatzIPUs ), + .TCDMAddrWidth (SPMAddrWidth ) + ) i_cachepool_cc ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .testmode_i (1'b0 ), + .hart_id_i (hart_id0 ), + .hive_req_o (hive_req[HartIdx] ), + .hive_rsp_i (hive_rsp[HartIdx] ), + .irq_i (irq0 ), + .data_req_o (core_req[HartIdx] ), + .data_rsp_i (core_rsp[HartIdx] ), + .tcdm_req_o (tcdm_req_wo_user ), + .tcdm_rsp_i (tcdm_rsp[TcdmPortsOffs +: TcdmPorts] ), + .tcdm_rsp_ready_o (tcdm_rsp_ready[TcdmPortsOffs +: TcdmPorts] ), + .core_events_o (/* unused */ ), + .tcdm_addr_base_i (tcdm_start_address ) + ); + end else if (NumScalarPerCC == 2) begin : gen_dual + // Second hart's own interrupt sync + irq assembly (mirrors hart 0's above). + interrupts_t irq1; + + sync #(.STAGES (2)) + i_sync_debug1 (.clk_i, .rst_ni, .serial_i (debug_req_i), .serial_o (irq1.debug)); + sync #(.STAGES (2)) + i_sync_meip1 (.clk_i, .rst_ni, .serial_i (meip_i), .serial_o (irq1.meip)); + sync #(.STAGES (2)) + i_sync_mtip1 (.clk_i, .rst_ni, .serial_i (mtip_i), .serial_o (irq1.mtip)); + sync #(.STAGES (2)) + i_sync_msip1 (.clk_i, .rst_ni, .serial_i (msip_i), .serial_o (irq1.msip)); + assign irq1.mcip = cl_interrupt_i[HartIdx+1]; + + interrupts_t [1:0] irq_pair; + assign irq_pair[0] = irq0; + assign irq_pair[1] = irq1; + + logic [1:0][31:0] hart_id_pair; + assign hart_id_pair[0] = hart_id0; + assign hart_id_pair[1] = hart_base_id_i + HartIdx + 1; + + cachepool_cc_dual #( + .AddrWidth (AxiAddrWidth ), + .DataWidth (NarrowDataWidth ), + .UserWidth (AxiUserWidth ), + .SpmStackDepth (SpmStackDepth ), + .dreq_t (reqrsp_req_t ), + .drsp_t (reqrsp_rsp_t ), + .dreq_chan_t (reqrsp_req_chan_t ), + .drsp_chan_t (reqrsp_rsp_chan_t ), + .tcdm_req_t (tcdm_req_t ), + .tcdm_user_t (tcdm_user_t ), + .tcdm_req_chan_t (tcdm_req_chan_t ), + .tcdm_rsp_t (tcdm_rsp_t ), + .tcdm_rsp_chan_t (tcdm_rsp_chan_t ), + .TCDMAddrWidth (SPMAddrWidth ), + .hive_req_t (hive_req_t ), + .hive_rsp_t (hive_rsp_t ), + .acc_issue_req_t (acc_issue_req_t ), + .acc_issue_rsp_t (acc_issue_rsp_t ), + .acc_rsp_t (acc_rsp_t ), + .FPUImplementation (FPUImplementation ), + .BootAddr (BootAddr ), + .UartAddr (UartAddr ), + .RVE (1'b0 ), + .RVF (RVF ), + .RVD (RVD ), + .XDivSqrt (1'b0 ), + .XF8 (1'b1 ), + .XF16 (1'b1 ), + .XF16ALT (1'b0 ), + .XF8ALT (1'b0 ), + .NumIntOutstandingLoads (NumIntOutstandingLoads ), + .NumIntOutstandingMem (NumIntOutstandingMem ), + .NumSpatzOutstandingLoads(NumSpatzOutstandingLoads ), + .RVV (RVV ), + .NumSpatzFPUs (NumSpatzFPUs ), + .NumSpatzIPUs (NumSpatzIPUs ), + .IsoCrossing (1'b0 ), + .RegisterOffloadRsp (RegisterOffloadRsp ), + .RegisterCoreReq (RegisterCoreReq ), + .RegisterCoreRsp (RegisterCoreRsp ), + .SnitchPMACfg (SnitchPMACfg ) + ) i_cachepool_cc_dual ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .testmode_i (1'b0 ), + .hart_id_i (hart_id_pair ), + .irq_i (irq_pair ), + .hive_req_o (hive_req[HartIdx +: 2] ), + .hive_rsp_i (hive_rsp[HartIdx +: 2] ), + .data_req_o (core_req[HartIdx +: 2] ), + .data_rsp_i (core_rsp[HartIdx +: 2] ), + .tcdm_req_o (tcdm_req_wo_user ), + .tcdm_rsp_i (tcdm_rsp[TcdmPortsOffs +: TcdmPorts] ), + .tcdm_rsp_ready_o (tcdm_rsp_ready[TcdmPortsOffs +: TcdmPorts] ), + .tcdm_addr_base_i (tcdm_start_address ), + .owner_id_o (/* debug only, unused */ ), + .lock_error_o (/* debug only, unused */ ) + ); + end else begin : gen_unsupported + `ASSERT_INIT(NumScalarPerCCSupported, 1'b0, + "cachepool_tile: NumScalarPerCC configuration not supported (only 1 or 2)") + end + for (genvar j = 0; j < TcdmPorts; j++) begin : gen_tcdm_user always_comb begin tcdm_req[TcdmPortsOffs+j].q = tcdm_req_wo_user[j].q; @@ -1626,7 +1719,7 @@ module cachepool_tile // First-level barrier for CachePool system cachepool_tile_barrier #( .AddrWidth (AxiAddrWidth ), - .NrPorts (NumCC ), + .NrPorts (NrHarts ), .dreq_t (reqrsp_req_t ), .drsp_t (reqrsp_rsp_t ), .user_t (tcdm_user_t ) @@ -1646,7 +1739,7 @@ module cachepool_tile reqrsp_rsp_t core_to_periph_rsp; reqrsp_mux #( - .NrPorts (NumCC ), + .NrPorts (NrHarts ), .AddrWidth (AxiAddrWidth ), .DataWidth (NarrowDataWidth ), .UserWidth ($bits(tcdm_user_t)), diff --git a/hardware/tb/cachepool_monitor.sv b/hardware/tb/cachepool_monitor.sv index bd7cf053..a958991e 100644 --- a/hardware/tb/cachepool_monitor.sv +++ b/hardware/tb/cachepool_monitor.sv @@ -29,8 +29,21 @@ module cachepool_monitor `define TILE_PATH(gy, gx, t) \ i_cluster_wrapper.i_cluster.gen_group_y[gy].gen_group_x[gx].i_group.i_group.gen_tiles[t].gen_tile.i_tile + // CC_PATH/CC_SNITCH_PATH target the single-CC flavor (gen_single), the + // only flavor buildable today. CC_DUAL_PATH/CC_DUAL_SNITCH_PATH are the + // gen_dual equivalents (i_snitch nests one level deeper, under + // gen_snitch[h], since a dual CC has 2; i_spatz sits at the same + // relative depth in both flavors) for future dual-mode monitor code to + // opt into explicitly once that flavor is actually built and exercised. `define CC_PATH(gy, gx, t, c) \ - `TILE_PATH(gy, gx, t).gen_cc[c].i_cachepool_cc + `TILE_PATH(gy, gx, t).gen_cc[c].gen_single.i_cachepool_cc + `define CC_SNITCH_PATH(gy, gx, t, c, h) \ + `CC_PATH(gy, gx, t, c).i_snitch + + `define CC_DUAL_PATH(gy, gx, t, c) \ + `TILE_PATH(gy, gx, t).gen_cc[c].gen_dual.i_cachepool_cc_dual + `define CC_DUAL_SNITCH_PATH(gy, gx, t, c, h) \ + `CC_DUAL_PATH(gy, gx, t, c).gen_snitch[h].i_snitch `define CLUSTER_PATH i_cluster_wrapper.i_cluster @@ -618,21 +631,21 @@ module cachepool_monitor automatic logic inst_valid, inst_ready, stall; automatic logic is_no_instr, is_itlb, is_hazard, is_lsu, is_acc, is_fence; - inst_valid = `CC_PATH(gy, gx, t, c).i_snitch.inst_valid_o; - inst_ready = `CC_PATH(gy, gx, t, c).i_snitch.inst_ready_i; - stall = `CC_PATH(gy, gx, t, c).i_snitch.stall; + inst_valid = `CC_SNITCH_PATH(gy, gx, t, c, 0).inst_valid_o; + inst_ready = `CC_SNITCH_PATH(gy, gx, t, c, 0).inst_ready_i; + stall = `CC_SNITCH_PATH(gy, gx, t, c, 0).stall; is_no_instr = ~(inst_valid & inst_ready); - is_itlb = `CC_PATH(gy, gx, t, c).i_snitch.trans_active & - ~(`CC_PATH(gy, gx, t, c).i_snitch.itlb_valid & - `CC_PATH(gy, gx, t, c).i_snitch.itlb_ready); - is_hazard = ~(`CC_PATH(gy, gx, t, c).i_snitch.operands_ready & - `CC_PATH(gy, gx, t, c).i_snitch.dst_ready); - is_lsu = `CC_PATH(gy, gx, t, c).i_snitch.lsu_stall; - is_acc = `CC_PATH(gy, gx, t, c).i_snitch.acc_stall; - is_fence = `CC_PATH(gy, gx, t, c).i_snitch.fence_snitch_stall | - `CC_PATH(gy, gx, t, c).i_snitch.fence_spatz_stall | - `CC_PATH(gy, gx, t, c).i_snitch.fence_stall; + is_itlb = `CC_SNITCH_PATH(gy, gx, t, c, 0).trans_active & + ~(`CC_SNITCH_PATH(gy, gx, t, c, 0).itlb_valid & + `CC_SNITCH_PATH(gy, gx, t, c, 0).itlb_ready); + is_hazard = ~(`CC_SNITCH_PATH(gy, gx, t, c, 0).operands_ready & + `CC_SNITCH_PATH(gy, gx, t, c, 0).dst_ready); + is_lsu = `CC_SNITCH_PATH(gy, gx, t, c, 0).lsu_stall; + is_acc = `CC_SNITCH_PATH(gy, gx, t, c, 0).acc_stall; + is_fence = `CC_SNITCH_PATH(gy, gx, t, c, 0).fence_snitch_stall | + `CC_SNITCH_PATH(gy, gx, t, c, 0).fence_spatz_stall | + `CC_SNITCH_PATH(gy, gx, t, c, 0).fence_stall; inst_valid_cyc_d = session_edge ? '0 : inst_valid_cyc_q + (inst_valid ? 1 : 0); inst_accepted_d = session_edge ? '0 : @@ -660,13 +673,13 @@ module cachepool_monitor // outstanding. logic snitch_load_req_accept, snitch_load_resp_commit; assign snitch_load_req_accept = - `CC_PATH(gy, gx, t, c).i_snitch.data_req_o.q_valid & - `CC_PATH(gy, gx, t, c).i_snitch.data_rsp_i.q_ready & - ~`CC_PATH(gy, gx, t, c).i_snitch.data_req_o.q.write; + `CC_SNITCH_PATH(gy, gx, t, c, 0).data_req_o.q_valid & + `CC_SNITCH_PATH(gy, gx, t, c, 0).data_rsp_i.q_ready & + ~`CC_SNITCH_PATH(gy, gx, t, c, 0).data_req_o.q.write; assign snitch_load_resp_commit = - `CC_PATH(gy, gx, t, c).i_snitch.data_rsp_i.p_valid & - `CC_PATH(gy, gx, t, c).i_snitch.data_req_o.p_ready & - ~`CC_PATH(gy, gx, t, c).i_snitch.data_rsp_i.p.write; + `CC_SNITCH_PATH(gy, gx, t, c, 0).data_rsp_i.p_valid & + `CC_SNITCH_PATH(gy, gx, t, c, 0).data_req_o.p_ready & + ~`CC_SNITCH_PATH(gy, gx, t, c, 0).data_rsp_i.p.write; cnt_t load_req_ts_q[$]; cnt_t snitch_load_lat_sum_q, snitch_load_lat_cnt_q; @@ -866,6 +879,9 @@ module cachepool_monitor `undef TILE_PATH `undef CC_PATH + `undef CC_SNITCH_PATH + `undef CC_DUAL_PATH + `undef CC_DUAL_SNITCH_PATH `undef CLUSTER_PATH `endif diff --git a/software/snRuntime/include/cachepool_peripheral.h b/software/snRuntime/include/cachepool_peripheral.h index 8bad67a8..f118c267 100644 --- a/software/snRuntime/include/cachepool_peripheral.h +++ b/software/snRuntime/include/cachepool_peripheral.h @@ -24,22 +24,25 @@ extern "C" { // cores have #define CACHEPOOL_PERIPHERAL_HW_BARRIER_REG_OFFSET 0x0 +// Spatz ownership lock for dual-Snitch core complexes. Write 1 to acquire +#define CACHEPOOL_PERIPHERAL_SPATZ_LOCK_REG_OFFSET 0x4 + // Controls prefetching of the instruction cache. -#define CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_REG_OFFSET 0x4 +#define CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_REG_OFFSET 0x8 #define CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_ICACHE_PREFETCH_ENABLE_BIT 0 // Sets the status of the Spatz cluster. -#define CACHEPOOL_PERIPHERAL_SPATZ_STATUS_REG_OFFSET 0x8 +#define CACHEPOOL_PERIPHERAL_SPATZ_STATUS_REG_OFFSET 0xc #define CACHEPOOL_PERIPHERAL_SPATZ_STATUS_SPATZ_CLUSTER_PROBE_BIT 0 // Store cycle counts of kernels -#define CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_REG_OFFSET 0xc +#define CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_REG_OFFSET 0x10 // Controls the cluster boot process. -#define CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_REG_OFFSET 0x10 +#define CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_REG_OFFSET 0x14 // End of computation and exit status register -#define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_REG_OFFSET 0x14 +#define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_REG_OFFSET 0x18 #define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_MASK 0xf #define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_OFFSET 0 #define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_FIELD \ @@ -48,7 +51,7 @@ extern "C" { .index = CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_OFFSET}) // Controls the configurations of L1 DCache SPM size. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_REG_OFFSET 0x18 +#define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_REG_OFFSET 0x1c #define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_MASK 0x3ff #define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_OFFSET 0 #define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_FIELD \ @@ -57,7 +60,7 @@ extern "C" { .index = CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_OFFSET}) // Controls the L1 DCache flushing and invalidation. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_REG_OFFSET 0x1c +#define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_REG_OFFSET 0x20 #define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_INSN_MASK 0x3 #define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_INSN_OFFSET 0 #define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_INSN_FIELD \ @@ -71,10 +74,10 @@ extern "C" { #define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_MULTIREG_COUNT 2 // One-hot tile selection mask for private-partition flush. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_REG_OFFSET 0x20 +#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_REG_OFFSET 0x24 // One-hot tile selection mask for private-partition flush. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_REG_OFFSET 0x24 +#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_REG_OFFSET 0x28 // Tile participation mask for the cluster-level hardware barrier. #define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_TILE_FIELD_WIDTH 32 @@ -82,25 +85,25 @@ extern "C" { #define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_MULTIREG_COUNT 2 // Tile participation mask for the cluster-level hardware barrier. -#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_REG_OFFSET 0x28 +#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_REG_OFFSET 0x2c // Tile participation mask for the cluster-level hardware barrier. -#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_REG_OFFSET 0x2c +#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_REG_OFFSET 0x30 // Controls the L1 DCache flushing and invalidation. -#define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_REG_OFFSET 0x30 +#define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_REG_OFFSET 0x34 #define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_COMMIT_BIT 0 // Controls the L1 DCache flushing and invalidation. -#define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_REG_OFFSET 0x34 +#define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_REG_OFFSET 0x38 #define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_COMMIT_BIT 0 // Indicate the status of flushing -#define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_REG_OFFSET 0x38 +#define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_REG_OFFSET 0x3c #define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_STATUS_BIT 0 // Number of private banks configured per tile -#define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_REG_OFFSET 0x3c +#define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_REG_OFFSET 0x40 #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_MASK 0xf #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_OFFSET 0 #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_FIELD \ @@ -109,10 +112,10 @@ extern "C" { CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_OFFSET}) // Starting address of private L1D partition -#define CACHEPOOL_PERIPHERAL_L1D_ADDR_REG_OFFSET 0x40 +#define CACHEPOOL_PERIPHERAL_L1D_ADDR_REG_OFFSET 0x44 // Cache xbar offset setting -#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET 0x44 +#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET 0x48 #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_MASK 0x1f #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_OFFSET 0 #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_FIELD \ @@ -121,7 +124,7 @@ extern "C" { CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_OFFSET}) // Cache xbar offset setting -#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_REG_OFFSET 0x48 +#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_REG_OFFSET 0x4c #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_COMMIT_BIT 0 #ifdef __cplusplus From c6b7f5d1d06adbe67526e4f989fd46b2c1d7956e Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Wed, 19 Aug 2026 11:56:42 +0200 Subject: [PATCH 05/17] [SRC] Add the dual-scalar core hardware --- Bender.yml | 3 + hardware/src/acc_mux.sv | 163 +++++++++++++++++++++++ hardware/src/cachepool_cc_dual.sv | 100 +++++++------- hardware/src/cachepool_cluster.sv | 2 +- hardware/src/cachepool_group.sv | 8 +- hardware/src/cachepool_pkg.sv | 13 +- hardware/src/cachepool_spatz_lock.sv | 37 ++--- hardware/src/cachepool_tile.sv | 1 + hardware/tb/cachepool_cluster_wrapper.sv | 2 +- hardware/tb/cachepool_monitor.sv | 36 ++--- 10 files changed, 270 insertions(+), 95 deletions(-) create mode 100644 hardware/src/acc_mux.sv diff --git a/Bender.yml b/Bender.yml index e550590c..35f7ea4b 100644 --- a/Bender.yml +++ b/Bender.yml @@ -42,6 +42,9 @@ sources: - hardware/generated/floo_cachepool_noc_pkg.sv - hardware/src/cachepool_pkg.sv - hardware/src/cachepool_cc.sv + - hardware/src/cachepool_spatz_lock.sv + - hardware/src/acc_mux.sv + - hardware/src/cachepool_cc_dual.sv # Barrier - hardware/src/cachepool_tile_barrier.sv - hardware/src/cachepool_cluster_barrier.sv diff --git a/hardware/src/acc_mux.sv b/hardware/src/acc_mux.sv new file mode 100644 index 00000000..5f8b5f1b --- /dev/null +++ b/hardware/src/acc_mux.sv @@ -0,0 +1,163 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Author: Diyou Shen + +`include "common_cells/assertions.svh" +`include "common_cells/registers.svh" + +/// Arbitrates 2 Snitch acc interfaces onto 1 shared Spatz acc interface for +/// cachepool_cc_dual, gated by cachepool_spatz_lock's owner/locked/waiting +/// state. Locked: only the owner passes through, real access, no +/// arbitration. Idle: both hosts get real round-robin access, one request +/// outstanding at a time -- a FIFO tracks which host is owed the in-flight +/// response (pushed on grant, popped on response), so a new request is only +/// arbitrated once the FIFO is empty. +module acc_mux #( + parameter type acc_issue_req_t = logic, + parameter type acc_issue_rsp_t = logic, + parameter type acc_rsp_t = logic +) ( + input logic clk_i, + input logic rst_ni, + + input logic owner_id_i, + input logic locked_i, + input logic waiting_i, + + // Per-host Snitch acc interfaces + input acc_issue_req_t [1:0] acc_snitch_req_i, + output acc_issue_rsp_t [1:0] acc_snitch_rsp_o, + input logic [1:0] acc_snitch_qvalid_i, + output logic [1:0] acc_snitch_qready_o, + output acc_rsp_t [1:0] acc_snitch_prsp_o, + output logic [1:0] acc_snitch_pvalid_o, + input logic [1:0] acc_snitch_pready_i, + + // Shared Spatz acc interface + output acc_issue_req_t spatz_issue_req_o, + input acc_issue_rsp_t spatz_issue_rsp_i, + output logic spatz_issue_valid_o, + input logic spatz_issue_ready_i, + input acc_rsp_t spatz_rsp_i, + input logic spatz_rsp_valid_i, + output logic spatz_rsp_ready_o, + + // Drain feed for the lock's outstanding counter (owner path and Idle-mode + // arbitrated path both count, since both represent real Spatz traffic). + output logic req_fire_o, + output logic rsp_fire_o, + + // Host most recently issued to Spatz; gates spatz_mem_finished etc. in cachepool_cc_dual.sv. + output logic last_req_host_o +); + + logic req_fire_any; + assign req_fire_any = spatz_issue_valid_o & spatz_issue_ready_i; + + // Only a writeback-producing issue leaves something for the lock to drain + // before a switch; a non-writeback op completes at the same-cycle accept. + assign req_fire_o = req_fire_any & spatz_issue_rsp_i.writeback; + assign rsp_fire_o = spatz_rsp_valid_i & spatz_rsp_ready_o; + + // Idle-mode round-robin arbitration, offered only while nothing is already outstanding. + logic idle_gnt_i, idle_req_o; + logic idle_winner; + logic [1:0] idle_req_i; + acc_issue_req_t idle_data_o; + logic [1:0] idle_gnt_o; + + logic route_fifo_empty, route_fifo_push, route_fifo_pop, route_fifo_route_q; + logic route_fifo_full; + + assign idle_req_i = (!locked_i && !waiting_i && route_fifo_empty) ? acc_snitch_qvalid_i : 2'b00; + assign idle_gnt_i = spatz_issue_ready_i; + + rr_arb_tree #( + .NumIn (2 ), + .DataType (acc_issue_req_t), + .AxiVldRdy (1'b1 ), + .LockIn (1'b1 ) + ) i_idle_arb ( + .clk_i, + .rst_ni, + .flush_i (1'b0 ), + .rr_i ('0 ), + .req_i (idle_req_i ), + .gnt_o (idle_gnt_o ), + .data_i (acc_snitch_req_i), + .req_o (idle_req_o ), + .gnt_i (idle_gnt_i ), + .data_o (idle_data_o ), + .idx_o (idle_winner ) + ); + + // Only track requests with a real completion coming: non-writeback ops never produce a later acc_rsp_t. + assign route_fifo_push = idle_req_o && idle_gnt_i && spatz_issue_rsp_i.writeback; + assign route_fifo_pop = rsp_fire_o && !locked_i; + + // Depth 2 for safety margin; only 1 entry is ever pushed at a time. + fifo_v3 #( + .DEPTH (2 ), + .dtype (logic) + ) i_route_fifo ( + .clk_i, + .rst_ni, + .flush_i (1'b0 ), + .testmode_i (1'b0 ), + .full_o (route_fifo_full ), + .empty_o (route_fifo_empty ), + .usage_o ( ), + .data_i (idle_winner ), + .push_i (route_fifo_push ), + .data_o (route_fifo_route_q), + .pop_i (route_fifo_pop ) + ); + + logic last_req_host_d, last_req_host_q; + + always_comb begin + last_req_host_d = last_req_host_q; + if (req_fire_any) last_req_host_d = locked_i ? owner_id_i : idle_winner; + end + + `FF(last_req_host_q, last_req_host_d, 1'b0, clk_i, rst_ni) + assign last_req_host_o = last_req_host_q; + + always_comb begin + spatz_issue_req_o = acc_issue_req_t'('0); + spatz_issue_valid_o = 1'b0; + spatz_rsp_ready_o = 1'b0; + + acc_snitch_qready_o = '0; + acc_snitch_rsp_o = '0; + acc_snitch_prsp_o = '0; + acc_snitch_pvalid_o = '0; + + if (locked_i) begin + // Exclusive owner path: real access, no arbitration needed. + spatz_issue_req_o = acc_snitch_req_i[owner_id_i]; + spatz_issue_valid_o = acc_snitch_qvalid_i[owner_id_i]; + spatz_rsp_ready_o = acc_snitch_pready_i[owner_id_i]; + + acc_snitch_qready_o[owner_id_i] = spatz_issue_ready_i; + acc_snitch_rsp_o[owner_id_i] = spatz_issue_rsp_i; + acc_snitch_prsp_o[owner_id_i] = spatz_rsp_i; + acc_snitch_pvalid_o[owner_id_i] = spatz_rsp_valid_i; + end else begin + // Draining is unconditional on waiting_i; only new-request arbitration is not. + if (!route_fifo_empty) begin + spatz_rsp_ready_o = acc_snitch_pready_i[route_fifo_route_q]; + acc_snitch_prsp_o[route_fifo_route_q] = spatz_rsp_i; + acc_snitch_pvalid_o[route_fifo_route_q] = spatz_rsp_valid_i; + end else if (!waiting_i) begin + spatz_issue_req_o = idle_data_o; + spatz_issue_valid_o = idle_req_o; + acc_snitch_qready_o[idle_winner] = idle_req_o && spatz_issue_ready_i; + acc_snitch_rsp_o[idle_winner] = spatz_issue_rsp_i; + end + end + end + +endmodule diff --git a/hardware/src/cachepool_cc_dual.sv b/hardware/src/cachepool_cc_dual.sv index 5426efef..161b568c 100644 --- a/hardware/src/cachepool_cc_dual.sv +++ b/hardware/src/cachepool_cc_dual.sv @@ -103,6 +103,7 @@ module cachepool_cc_dual // Response-side ready, reflecting real downstream capacity (Spatz + scalar ports). output logic [TCDMPorts-1:0] tcdm_rsp_ready_o, input addr_t tcdm_addr_base_i, + input addr_t cluster_periph_start_address_i, // Spatz lock/switch status (debug) output logic owner_id_o, output logic lock_error_o @@ -124,6 +125,11 @@ module cachepool_cc_dual `REQRSP_TYPEDEF_ALL(reqrsp, addr_t, data_t, strb_t, tcdm_user_t) + logic owner_id, locked, waiting; + logic req_fire, rsp_fire; + logic last_req_host; + assign owner_id_o = owner_id; + // Raw per-host Snitch data port, and its acc issue/response bundle. dreq_t [1:0] snitch_dreq_d; drsp_t [1:0] snitch_drsp_d; @@ -138,10 +144,7 @@ module cachepool_cc_dual fpnew_pkg::fmt_mode_t [1:0] fpu_fmt_mode_h; fpnew_pkg::status_t fpu_status; - // Spatz memory-consistency signals (single Spatz instance). Forwarded to the - // owner only -- the non-owner never issued anything, so its own acc_mem_cnt_q - // never incremented; broadcasting these finish pulses to it too would - // decrement past 0 and roll over. The non-owner sees "nothing pending" instead. + // Spatz memory-consistency signals, forwarded only to last_req_host (not owner_id, which is stale in Idle mode). logic [1:0] spatz_mem_finished; logic [1:0] spatz_mem_str_finished; logic spatz_st_rsp_done; @@ -151,9 +154,9 @@ module cachepool_cc_dual logic [1:0] spatz_st_rsp_done_h; for (genvar h = 0; h < 2; h++) begin : gen_spatz_mem_consistency_gate - assign spatz_mem_finished_h[h] = (owner_id == h[0]) ? spatz_mem_finished : '0; - assign spatz_mem_str_finished_h[h] = (owner_id == h[0]) ? spatz_mem_str_finished : '0; - assign spatz_st_rsp_done_h[h] = (owner_id == h[0]) ? spatz_st_rsp_done : 1'b1; + assign spatz_mem_finished_h[h] = (last_req_host == h[0]) ? spatz_mem_finished : '0; + assign spatz_mem_str_finished_h[h] = (last_req_host == h[0]) ? spatz_mem_str_finished : '0; + assign spatz_st_rsp_done_h[h] = (last_req_host == h[0]) ? spatz_st_rsp_done : 1'b1; end for (genvar h = 0; h < 2; h++) begin : gen_snitch @@ -239,10 +242,6 @@ module cachepool_cc_dual dreq_t [1:0] lock_out_req; drsp_t [1:0] lock_out_rsp; - logic owner_id, locked, waiting; - logic req_fire, rsp_fire; - assign owner_id_o = owner_id; - cachepool_spatz_lock #( .AddrWidth (AddrWidth ), .NumHosts (2 ), @@ -267,36 +266,39 @@ module cachepool_cc_dual .locked_o (locked ), .waiting_o (waiting ), .error_o (lock_error_o ), - .cluster_periph_start_address_i (tcdm_addr_base_i ) + .cluster_periph_start_address_i (cluster_periph_start_address_i) ); - // Acc mux: owner's issue/response stream reaches Spatz. New requests (from anyone) - // are blocked while waiting; the owner's in-flight responses keep draining - // regardless. Non-owner is faked-accepted while unlocked, stalled+flagged - // (by the lock, via the stalled data-interface path) once locked. - assign req_fire = spatz_issue_valid & spatz_issue_ready; - assign rsp_fire = spatz_rsp_valid & spatz_rsp_ready; - - always_comb begin - spatz_issue_req = acc_snitch_req[owner_id]; - spatz_issue_valid = waiting ? 1'b0 : acc_snitch_qvalid[owner_id]; - spatz_rsp_ready = acc_snitch_pready[owner_id]; - - acc_snitch_qready = '0; - acc_snitch_rsp = '0; - acc_snitch_prsp = '0; - acc_snitch_pvalid = '0; - - acc_snitch_qready[owner_id] = waiting ? 1'b0 : spatz_issue_ready; - acc_snitch_rsp[owner_id] = spatz_issue_rsp; - acc_snitch_prsp[owner_id] = spatz_rsp; - acc_snitch_pvalid[owner_id] = spatz_rsp_valid; - - if (!locked && !waiting) begin - acc_snitch_qready[1] = acc_snitch_qvalid[1]; - acc_snitch_rsp[1] = '{accept: 1'b1, default: '0}; - end - end + // Acc mux: real round-robin access to Spatz for both hosts while + // unlocked, exclusive owner access while locked. See acc_mux.sv. + acc_mux #( + .acc_issue_req_t (acc_issue_req_t), + .acc_issue_rsp_t (acc_issue_rsp_t), + .acc_rsp_t (acc_rsp_t ) + ) i_acc_mux ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .owner_id_i (owner_id ), + .locked_i (locked ), + .waiting_i (waiting ), + .acc_snitch_req_i (acc_snitch_req ), + .acc_snitch_rsp_o (acc_snitch_rsp ), + .acc_snitch_qvalid_i (acc_snitch_qvalid ), + .acc_snitch_qready_o (acc_snitch_qready ), + .acc_snitch_prsp_o (acc_snitch_prsp ), + .acc_snitch_pvalid_o (acc_snitch_pvalid ), + .acc_snitch_pready_i (acc_snitch_pready ), + .spatz_issue_req_o (spatz_issue_req ), + .spatz_issue_rsp_i (spatz_issue_rsp ), + .spatz_issue_valid_o (spatz_issue_valid ), + .spatz_issue_ready_i (spatz_issue_ready ), + .spatz_rsp_i (spatz_rsp ), + .spatz_rsp_valid_i (spatz_rsp_valid ), + .spatz_rsp_ready_o (spatz_rsp_ready ), + .req_fire_o (req_fire ), + .rsp_fire_o (rsp_fire ), + .last_req_host_o (last_req_host ) + ); // FPU rounding-mode/format CSRs: Spatz has one FPU, so it only ever sees the // current owner's config; its status broadcasts back to both hosts. @@ -359,11 +361,8 @@ module cachepool_cc_dual .fpu_status_o (fpu_status ) ); - // Vector-FU TCDM ports: pinned to host 0's routing row in the tile-level - // cache crossbar (see note.md "Software / core-id assignment") -- there is - // only one physical row for this CC pair regardless of current owner; the - // acc mux above already re-attributes the response to the true owner once - // it re-enters the CC. + // Vector-FU TCDM ports pin to host 0's crossbar row (only one physical + // row per CC pair); acc_mux re-attributes responses to the true owner. for (genvar p = 0; p < NumMemPortsPerSpatz; p++) begin : gen_spatz_mem_ports assign tcdm_req_o[p] = '{ q : spatz_mem_req[p], @@ -467,8 +466,11 @@ module cachepool_cc_dual for (genvar h = 0; h < 2; h++) begin : gen_host // FPU's request only ever contends on the crossbar matching the current owner. - assign fpu_req_gated[h] = fp_lsu_mem_req; - assign fpu_req_gated[h].q_valid = fp_lsu_mem_req.q_valid && (owner_id == h[0]); + assign fpu_req_gated[h] = '{ + q : fp_lsu_mem_req.q, + q_valid: fp_lsu_mem_req.q_valid && (owner_id == h[0]), + p_ready: fp_lsu_mem_req.p_ready + }; assign core_req_chan [h][SnitchMst] = lock_out_req[h].q; assign core_req_valid [h][SnitchMst] = lock_out_req[h].q_valid; @@ -536,10 +538,8 @@ module cachepool_cc_dual .mst_sel_i (mem_rsp_sel [h] ) ); - // TotStack-range hits within the (merged) CacheMem slot get the hart-id - // bits patched to this host's own id -- always host h's id, regardless - // of which master won, since the FPU only ever wins here when it's - // already gating on owner_id == h. + // TotStack hits get patched with host h's own id regardless of which + // master won, since the FPU only ever wins here when already gated to owner_id==h. assign is_totstack[h] = (tcdm_addr_base_i <= mem_req_chan[h][CacheMem].addr) && (mem_req_chan[h][CacheMem].addr < tcdm_addr_base_i + cachepool_pkg::TotStackSize); diff --git a/hardware/src/cachepool_cluster.sv b/hardware/src/cachepool_cluster.sv index 8fee4b61..f8c7f24e 100644 --- a/hardware/src/cachepool_cluster.sv +++ b/hardware/src/cachepool_cluster.sv @@ -288,7 +288,7 @@ module cachepool_cluster .meip_i ( meip_i ), .mtip_i ( mtip_i ), .msip_i ( msip_i ), - .hart_base_id_i ( HartBaseId + 10'(g * NumCoreGroup) ), + .hart_base_id_i ( HartBaseId + 10'(g * NumCoreGroup * NumScalarPerCC) ), .tile_base_id_i ( TileIDWidth'(g * NumTilesPerGroup) ), .cluster_base_addr_i ( cluster_base_addr_i ), .private_start_addr_i ( private_start_addr ), diff --git a/hardware/src/cachepool_group.sv b/hardware/src/cachepool_group.sv index 1223ad1b..9aad4b7f 100644 --- a/hardware/src/cachepool_group.sv +++ b/hardware/src/cachepool_group.sv @@ -146,7 +146,7 @@ module cachepool_group /// Peripheral signals output icache_l1_events_t [NumCC-1:0] icache_events_o, input logic icache_prefetch_enable_i, - input logic [NumCC-1:0] cl_interrupt_i, + input logic [NumCC*NumScalarPerCC-1:0] cl_interrupt_i, input logic [$clog2(AxiAddrWidth)-1:0] dynamic_offset_i, input logic [$clog2(NumL1CtrlTile):0] l1d_private_i, input cache_insn_t l1d_insn_i, @@ -547,7 +547,7 @@ module cachepool_group for (genvar t = 0; t < NumTilesPerGroup; t ++) begin : gen_tiles logic [9:0] hart_base_id; - assign hart_base_id = hart_base_id_i + t * NumCoresTile; + assign hart_base_id = hart_base_id_i + t * NumCoresTile * NumScalarPerCC; logic [TileIDWidth-1:0] tile_id; assign tile_id = tile_base_id_i + TileIDWidth'(t); @@ -635,7 +635,7 @@ module cachepool_group // Peripherals .icache_events_o ( /* unused */ ), .icache_prefetch_enable_i ( icache_prefetch_enable_i ), - .cl_interrupt_i ( cl_interrupt_i [t*NumCoresTile+:NumCoresTile] ), + .cl_interrupt_i ( cl_interrupt_i [t*NumCoresTile*NumScalarPerCC +: NumCoresTile*NumScalarPerCC] ), .dynamic_offset_i ( dynamic_offset_i ), .l1d_insn_i ( l1d_insn_i ), .l1d_private_i ( l1d_private_i ), @@ -726,7 +726,7 @@ module cachepool_group // Peripherals .icache_events_o ( /* unused */ ), .icache_prefetch_enable_i ( icache_prefetch_enable_i ), - .cl_interrupt_i ( cl_interrupt_i [t*NumCoresTile+:NumCoresTile] ), + .cl_interrupt_i ( cl_interrupt_i [t*NumCoresTile*NumScalarPerCC +: NumCoresTile*NumScalarPerCC] ), .dynamic_offset_i ( dynamic_offset_i ), .l1d_insn_i ( l1d_insn_i ), .l1d_private_i ( l1d_private_i ), diff --git a/hardware/src/cachepool_pkg.sv b/hardware/src/cachepool_pkg.sv index edc34d5b..6af961ad 100644 --- a/hardware/src/cachepool_pkg.sv +++ b/hardware/src/cachepool_pkg.sv @@ -26,7 +26,8 @@ package cachepool_pkg; ////////////////// // GLOBAL HW // ////////////////// - localparam int unsigned NumCores = `ifdef NUM_CORES `NUM_CORES `else 0 `endif; + // Core Complex slot count (one Spatz per slot, shared by NumScalarPerCC harts). + localparam int unsigned NumCC = `ifdef NUM_CORES `NUM_CORES `else 0 `endif; localparam int unsigned NumTiles = `ifdef NUM_TILES `NUM_TILES `else 0 `endif; // TODO: not yet passed in through config, hardcode to 1 localparam int unsigned NumGroups = `ifdef NUM_GROUPS `NUM_GROUPS `else 1 `endif; @@ -46,6 +47,10 @@ package cachepool_pkg; // Whole-build choice, not per-tile; default 1 until config.mk wires a real value through. localparam int unsigned NumScalarPerCC = `ifdef NUM_SCALAR_PER_CC `NUM_SCALAR_PER_CC `else 1 `endif; + // Total hart count (NumCC CC-slots x NumScalarPerCC harts each) -- use for + // per-hart addressing; use NumCC for CC/Spatz-slot units (tile/group size). + localparam int unsigned NumCores = NumCC * NumScalarPerCC; + localparam int unsigned NumIntOutstandingLoads = `ifdef SNITCH_MAX_TRANS `SNITCH_MAX_TRANS `else 0 `endif; localparam int unsigned NumIntOutstandingMem = `ifdef SNITCH_MAX_TRANS `SNITCH_MAX_TRANS `else 0 `endif; localparam int unsigned NumSpatzOutstandingLoads = `ifdef SPATZ_MAX_TRANS `SPATZ_MAX_TRANS `else 0 `endif; @@ -56,7 +61,7 @@ package cachepool_pkg; // TILE CONFIG // /////////////////// // How many cores for each tile? - localparam int unsigned NumCoresTile = NumCores / NumTiles; + localparam int unsigned NumCoresTile = NumCC / NumTiles; // Intra-group remote ports per core (to other tiles in the same group). localparam int unsigned NumLGPortCore = `ifdef LG_PORT_PER_CORE `LG_PORT_PER_CORE `else 0 `endif; @@ -78,7 +83,7 @@ package cachepool_pkg; localparam int unsigned NumTilesPerGroup = NumTiles / NumGroups; // How many cores for each group? - localparam int unsigned NumCoreGroup = NumCores / NumGroups; + localparam int unsigned NumCoreGroup = NumCC / NumGroups; // How many remote group ports for each tile? localparam int unsigned NumRemoteGroupPortCore = `ifdef RG_PORT_PER_CORE `RG_PORT_PER_CORE `else 0 `endif; @@ -149,7 +154,7 @@ package cachepool_pkg; localparam int unsigned NumBank = `ifdef L1D_NUM_BANKS `L1D_NUM_BANKS `else 0 `endif; // NOTE: these are used by AXI/L2 as well, keep here but ordered as "cluster-level cache topology" - localparam int unsigned NumL1CacheCtrl = NumCores; + localparam int unsigned NumL1CacheCtrl = NumCC; localparam int unsigned NumL1CtrlTile = NumL1CacheCtrl / NumTiles; // Number of data banks assigned to each cache controller diff --git a/hardware/src/cachepool_spatz_lock.sv b/hardware/src/cachepool_spatz_lock.sv index 90c4f804..c582288d 100644 --- a/hardware/src/cachepool_spatz_lock.sv +++ b/hardware/src/cachepool_spatz_lock.sv @@ -11,11 +11,11 @@ /// Host 0 is the implicit owner while Idle; a write of 1 to the lock /// address acquires ownership (blocks until granted), a write of 0 /// releases it (owner only). Pure arbiter: does not route the acc/data -/// traffic itself (that lives in cachepool_cc_dual.sv, alongside its other -/// owner-based muxing) -- only decides ownership and exposes owner_id_o/ -/// locked_o/waiting_o for the CC to apply, and counts owner-path acc -/// handshakes (via req_fire_i/rsp_fire_i, fed back by the CC) to gate a -/// switch until fully drained. +/// traffic itself (that lives in acc_mux.sv/cachepool_cc_dual.sv) -- only +/// decides ownership and exposes owner_id_o/locked_o/waiting_o for the CC +/// to apply, and counts real Spatz acc handshakes (via +/// req_fire_i/rsp_fire_i, fed back by acc_mux) to gate a switch until +/// fully drained. /// Also hosts the pass-through memory-path register cuts (moved here from /// the per-core spill registers in cachepool_cc.sv). module cachepool_spatz_lock @@ -42,8 +42,7 @@ module cachepool_spatz_lock output dreq_t [NumHosts-1:0] out_req_o, input drsp_t [NumHosts-1:0] out_rsp_i, - // Owner-path acc handshake fires, fed back by the CC (which does the - // actual acc routing) purely for drain counting. + // Real Spatz acc handshake fires, fed back by acc_mux purely for drain counting. input logic req_fire_i, input logic rsp_fire_i, @@ -65,7 +64,8 @@ module cachepool_spatz_lock // on the way to an actual ownership switch. // Idle -(acquire, other host)-> AcqWait -(drain_done)-> Locked (new owner) // Locked -(release, owner) -> RelWait -(drain_done)-> Idle (owner = host 0) - // Self-acquire (already owner) and read complete without a drain wait. + // Self-acquire completes immediately only if already drained; otherwise it + // waits through AcqWait too. Reads always complete immediately. // A non-owner acquire/release, or any lock op during a wait, is stalled // (never accepted) and sets the sticky error_o. // @@ -86,15 +86,15 @@ module cachepool_spatz_lock logic active_d, active_q; user_t user_d, user_q; + // Outstanding acc handshakes on the owner's path; must reach 0 before a wait can complete. + logic [7:0] outstanding_d, outstanding_q; + logic drain_done, waiting; + assign owner_id_o = owner_q; assign locked_o = (lock_q == Locked); assign waiting_o = waiting; assign error_o = error_q; - // Outstanding acc handshakes on the owner's path; must reach 0 before a wait can complete. - logic [7:0] outstanding_d, outstanding_q; - logic drain_done, waiting; - assign waiting = (lock_q == AcqWait) || (lock_q == RelWait); assign drain_done = (outstanding_q == '0); @@ -205,14 +205,14 @@ module cachepool_spatz_lock end case (lock_q) - // Host 0 is the implicit owner. Its own acquire/release are no-ops (immediate - // grant); host 1 may acquire (drains, then switches); host 1's release is - // meaningless (nothing to release) and stalled. + // Host 0 is the implicit owner. Its own release is a no-op (immediate grant); + // its own acquire grants immediately only if drained, else waits like host 1's; + // host 1's release is meaningless (nothing to release) and stalled. Idle: begin if (hit_any && !resp_pending_q) begin if (is_acquire[winner]) begin - if (winner == owner_q) begin - // host 0 self-acquire: immediate grant + if (winner == owner_q && drain_done) begin + // host 0 self-acquire, nothing outstanding: immediate grant lock_d = Locked; in_rsp_o[winner].q_ready = 1'b1; resp_pending_d = 1'b1; @@ -220,7 +220,8 @@ module cachepool_spatz_lock resp_user_d = in_req_i[winner].q.user; resp_read_d = 1'b0; end else begin - // host 1 wants the lock -> drain, then grant + // host 1 wants the lock, or host 0 self-acquires while Idle-mode + // traffic (from either host) is still outstanding -> drain first active_d = winner; user_d = in_req_i[winner].q.user; lock_d = AcqWait; diff --git a/hardware/src/cachepool_tile.sv b/hardware/src/cachepool_tile.sv index 260da607..8a275510 100644 --- a/hardware/src/cachepool_tile.sv +++ b/hardware/src/cachepool_tile.sv @@ -1627,6 +1627,7 @@ module cachepool_tile .tcdm_rsp_i (tcdm_rsp[TcdmPortsOffs +: TcdmPorts] ), .tcdm_rsp_ready_o (tcdm_rsp_ready[TcdmPortsOffs +: TcdmPorts] ), .tcdm_addr_base_i (tcdm_start_address ), + .cluster_periph_start_address_i (cluster_periph_start_address), .owner_id_o (/* debug only, unused */ ), .lock_error_o (/* debug only, unused */ ) ); diff --git a/hardware/tb/cachepool_cluster_wrapper.sv b/hardware/tb/cachepool_cluster_wrapper.sv index d85d5b9f..33b3b911 100644 --- a/hardware/tb/cachepool_cluster_wrapper.sv +++ b/hardware/tb/cachepool_cluster_wrapper.sv @@ -73,7 +73,7 @@ module cachepool_cluster_wrapper .BootAddr (BootAddr ), .UartAddr (UartAddr ), .ClusterPeriphSize (64 ), - .NumCC (NumCores ), + .NumCC (NumCC ), .TCDMDepth (TCDMDepth ), .NrBanks (NumBank ), .ICacheLineWidth (ICacheLineWidth ), diff --git a/hardware/tb/cachepool_monitor.sv b/hardware/tb/cachepool_monitor.sv index a958991e..10af768d 100644 --- a/hardware/tb/cachepool_monitor.sv +++ b/hardware/tb/cachepool_monitor.sv @@ -29,21 +29,25 @@ module cachepool_monitor `define TILE_PATH(gy, gx, t) \ i_cluster_wrapper.i_cluster.gen_group_y[gy].gen_group_x[gx].i_group.i_group.gen_tiles[t].gen_tile.i_tile - // CC_PATH/CC_SNITCH_PATH target the single-CC flavor (gen_single), the - // only flavor buildable today. CC_DUAL_PATH/CC_DUAL_SNITCH_PATH are the - // gen_dual equivalents (i_snitch nests one level deeper, under - // gen_snitch[h], since a dual CC has 2; i_spatz sits at the same - // relative depth in both flavors) for future dual-mode monitor code to - // opt into explicitly once that flavor is actually built and exercised. - `define CC_PATH(gy, gx, t, c) \ - `TILE_PATH(gy, gx, t).gen_cc[c].gen_single.i_cachepool_cc - `define CC_SNITCH_PATH(gy, gx, t, c, h) \ - `CC_PATH(gy, gx, t, c).i_snitch - - `define CC_DUAL_PATH(gy, gx, t, c) \ - `TILE_PATH(gy, gx, t).gen_cc[c].gen_dual.i_cachepool_cc_dual - `define CC_DUAL_SNITCH_PATH(gy, gx, t, c, h) \ - `CC_DUAL_PATH(gy, gx, t, c).gen_snitch[h].i_snitch + // CACHEPOOL_DUAL_CC (plain definedness flag, set by the Makefile iff + // num_scalar_per_core==2) picks which sub-instance gen_cc wraps, since + // the toolchain's preprocessor doesn't support `if value-comparisons. + // CC_SNITCH_PATH takes a hart index h; only h=0 is probed below today + // (the monitor's c-loop still indexes CC slots 1:1 with harts, so + // per-hart dual-mode stat collection is unstarted, deferred). i_spatz + // sits at the same relative depth in both flavors, so CC_PATH's + // existing `.i_spatz.*` usages need no change either way. + `ifdef CACHEPOOL_DUAL_CC + `define CC_PATH(gy, gx, t, c) \ + `TILE_PATH(gy, gx, t).gen_cc[c].gen_dual.i_cachepool_cc_dual + `define CC_SNITCH_PATH(gy, gx, t, c, h) \ + `CC_PATH(gy, gx, t, c).gen_snitch[h].i_snitch + `else + `define CC_PATH(gy, gx, t, c) \ + `TILE_PATH(gy, gx, t).gen_cc[c].gen_single.i_cachepool_cc + `define CC_SNITCH_PATH(gy, gx, t, c, h) \ + `CC_PATH(gy, gx, t, c).i_snitch + `endif `define CLUSTER_PATH i_cluster_wrapper.i_cluster @@ -880,8 +884,6 @@ module cachepool_monitor `undef TILE_PATH `undef CC_PATH `undef CC_SNITCH_PATH - `undef CC_DUAL_PATH - `undef CC_DUAL_SNITCH_PATH `undef CLUSTER_PATH `endif From 79868748e1a9b5d082da31ffd5164308264ce3ee Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Wed, 19 Aug 2026 11:58:01 +0200 Subject: [PATCH 06/17] [CFG] Add dual scalar core configuration for testing --- config/cachepool_dual_4g.mk | 117 +++++++++++++++++++++++++++++++++ sim/scripts/vsim_core.tcl | 10 ++- sim/scripts/vsim_core_dual.tcl | 76 +++++++++++++++++++++ 3 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 config/cachepool_dual_4g.mk create mode 100644 sim/scripts/vsim_core_dual.tcl diff --git a/config/cachepool_dual_4g.mk b/config/cachepool_dual_4g.mk new file mode 100644 index 00000000..e9c5cd63 --- /dev/null +++ b/config/cachepool_dual_4g.mk @@ -0,0 +1,117 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Author: Diyou Shen, ETH Zurich + +######################### +## CachePool Cluster ## +######################### + +# Number of groups +num_groups ?= 4 + +# 2×2 mesh +num_groups_x ?= 2 + +# Number of tiles +num_tiles_per_group ?= 4 + +# Number of Core Complex slots (each holding num_scalar_per_core harts) +num_cores_per_tile ?= 4 + +# Core datawidth +data_width ?= 32 + +# Core addrwidth +addr_width ?= 32 + +num_lg_ports_per_core ?= 2 + +num_rg_ports_per_core ?= 1 + +num_noc_ports_per_tile ?= 4 + + +###################### +## CachePool Tile ## +###################### + +##### L1 Data Cache ##### + +# L1 data cacheline width (in Bit) +l1d_cacheline_width ?= 512 + +# L1 data cache banking factor (how many banks per core?) +l1d_bank_factor ?= 1 + +# L1 coalecsing window +l1d_coal_window ?= 2 + +# L1 data cache number of ways per +l1d_num_way ?= 4 + +# L1 data cache size per tile (KiB) +l1d_tile_size ?= 256 + +# L1 data cache tag width (TODO: should be calcualted) +l1d_tag_data_width ?= 92 + +# Use folded (skewed) data banks +l1d_use_folded ?= 1 + +# Fold-way group size for skewed folded banks (0 = auto: min(4, ways)) +l1d_fold_way_group ?= 0 + +# Use hash-based way selection (required by l1d_use_folded / l1d_use_fwd_buf) +l1d_use_hash_way ?= 1 + +# Enable the SRAM forwarding buffer (requires l1d_use_hash_way) +l1d_use_fwd_buf ?= 1 + +#################### +## CachePool CC ## +#################### +# Spatz fpu support? +spatz_fpu_en ?= 0 + +# Spatz number of FPU +spatz_num_fpu ?= 0 + +# Spatz number of IPU +spatz_num_ipu ?= 4 + +# Spatz max outstanding transactions +spatz_max_trans ?= 32 + +# Snitch/FPU max outstanding transactions +snitch_max_trans ?= 16 + +# 2 Snitch scalar harts sharing one Spatz per Core Complex +num_scalar_per_core ?= 2 + + +##################### +## L2 Main Memory ## +##################### +# DRAM type (selects DRAMSys config and sets default refill_data_width) +dram_type ?= HBM2 + +# L2 number of channels +l2_channel ?= 4 + +# L2 bank width (DRAM width, change with care) +l2_bank_width ?= 512 + +# L2 interleaving factor (in order of bank_width) +l2_interleave ?= 16 + + +################## +## Peripherals ## +################## +# Hardware stack size (in Byte) +stack_hw_size ?= 1024 + +# Stack size (total, including share and private, 32'h800) +stack_tot_size ?= 2048 diff --git a/sim/scripts/vsim_core.tcl b/sim/scripts/vsim_core.tcl index 30ee61ac..f6bf0316 100644 --- a/sim/scripts/vsim_core.tcl +++ b/sim/scripts/vsim_core.tcl @@ -18,8 +18,16 @@ if {$argc > 5 && "${6}" != ""} { quietly lappend parent_grp -group ${6} } -# The {*} syntax safely expands the list. +# Detect flavor at runtime (gen_single vs gen_dual) instead of relying on a +# compile-time flag, so this script works regardless of num_scalar_per_core. +if {[llength [find instances -nodu ${core_path}/gen_dual/i_cachepool_cc_dual]] > 0} { + do sim/scripts/vsim_core_dual.tcl ${1} ${2} ${3} ${4} ${5} ${6} + return +} + +# The {*} syntax safely expands the list. # If $parent_grp is empty, it safely ignores it instead of passing "". +quietly set core_path ${core_path}/gen_single add wave -noupdate {*}$parent_grp -group ${name} -group scalar_xbar ${core_path}/i_cachepool_cc/i_scalar_xbar/* add wave -noupdate {*}$parent_grp -group ${name} -group Params ${core_path}/i_cachepool_cc/BootAddr diff --git a/sim/scripts/vsim_core_dual.tcl b/sim/scripts/vsim_core_dual.tcl new file mode 100644 index 00000000..bbf1e34d --- /dev/null +++ b/sim/scripts/vsim_core_dual.tcl @@ -0,0 +1,76 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# Solderpad Hardware License, Version 0.51, see LICENSE for details. +# SPDX-License-Identifier: SHL-0.51 + +# Create group for a dual-CC Core Complex (2 Snitch harts sharing 1 Spatz). +# Not a mechanical mirror of vsim_core.tcl's single-CC dump: cachepool_cc_dual +# has a different internal structure (2x gen_snitch, 1 shared i_spatz, plus +# the lock/acc_mux), so this covers both harts' top-level Snitch signals, the +# shared Spatz, and the ownership lock/mux state, rather than replicating +# every internal Snitch/Spatz signal group per hart. +onerror {resume} +quietly WaveActivateNextPane {} 0 + +quietly set core_path ${4}/gen_dual +quietly set name g_${1}_t_${2}_c_${3} + +quietly set parent_grp [list] +if {$argc > 4 && "${5}" != ""} { + quietly lappend parent_grp -group ${5} +} +if {$argc > 5 && "${6}" != ""} { + quietly lappend parent_grp -group ${6} +} + +# Lock/switch state: which host owns Spatz, and the acc_mux fake-completion state. +add wave -noupdate {*}$parent_grp -group ${name} -group Lock ${core_path}/i_cachepool_cc_dual/i_spatz_lock/* +add wave -noupdate {*}$parent_grp -group ${name} -group AccMux ${core_path}/i_cachepool_cc_dual/i_acc_mux/* + +for {set h 0} {$h < 2} {incr h} { + quietly set snitch_path ${core_path}/i_cachepool_cc_dual/gen_snitch[${h}]/i_snitch + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -radix unsigned ${snitch_path}/hart_id_i + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -divider Instructions + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/inst_addr_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/inst_data_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/inst_valid_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/inst_ready_i + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -divider Load/Store + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/data_req_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/data_rsp_i + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -divider Accelerator + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_qreq_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_qrsp_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_qvalid_o + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_qready_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_prsp_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_pvalid_i + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} ${snitch_path}/acc_pready_o + + add wave -noupdate {*}$parent_grp -group ${name} -group Host${h} -group Internal ${snitch_path}/* +} + +quietly set spatz_path ${core_path}/i_cachepool_cc_dual/i_spatz + +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/issue_valid_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/issue_ready_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/issue_req_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/issue_rsp_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/rsp_valid_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/rsp_ready_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/rsp_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_req_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_req_valid_o +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_req_ready_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_rsp_i +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz ${spatz_path}/spatz_mem_rsp_valid_i + +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz -group VLSU ${spatz_path}/i_vlsu/* +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz -group VSLDU ${spatz_path}/i_vsldu/* +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz -group VFU ${spatz_path}/i_vfu/* +add wave -noupdate {*}$parent_grp -group ${name} -group Spatz -group Controller ${spatz_path}/i_controller/* + +add wave -noupdate {*}$parent_grp -group ${name} -group Internal ${core_path}/i_cachepool_cc_dual/* From 32c2949da1a38a709154aa121386f7d99fd40a41 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Wed, 19 Aug 2026 11:59:31 +0200 Subject: [PATCH 07/17] [SW] Add needed runtime functions to support dual-scalar core configuration --- software/snRuntime/CMakeLists.txt | 4 +++ software/snRuntime/include/snrt.h | 11 ++++++++ software/snRuntime/include/spatz_lock.h | 17 ++++++++++++ software/snRuntime/src/barrier.c | 24 +++++++++++++++++ software/snRuntime/src/spatz_lock.c | 21 +++++++++++++++ software/snRuntime/src/team.c | 8 ++++++ software/tests/idotp-32b/main.c | 36 +++++++++++++++++-------- util/scripts/gen_spatz_cfg.py | 6 ++++- 8 files changed, 115 insertions(+), 12 deletions(-) create mode 100644 software/snRuntime/include/spatz_lock.h create mode 100644 software/snRuntime/src/spatz_lock.c diff --git a/software/snRuntime/CMakeLists.txt b/software/snRuntime/CMakeLists.txt index 5c6b3cba..39debc5e 100644 --- a/software/snRuntime/CMakeLists.txt +++ b/software/snRuntime/CMakeLists.txt @@ -49,6 +49,9 @@ if(RUNTIME_TRACE) add_compile_definitions(__SNRT_USE_TRACE) endif() +set(NUM_SCALAR_PER_CORE "1" CACHE STRING "Snitch harts sharing one Spatz per Core Complex") +add_compile_definitions(SNRT_NUM_SCALAR_PER_CORE=${NUM_SCALAR_PER_CORE}) + include_directories( include vendor @@ -65,6 +68,7 @@ set(sources src/alloc.c src/interrupt.c src/l1cache.c + src/spatz_lock.c ) # platform specific sources diff --git a/software/snRuntime/include/snrt.h b/software/snRuntime/include/snrt.h index 886d552a..fb0cd952 100644 --- a/software/snRuntime/include/snrt.h +++ b/software/snRuntime/include/snrt.h @@ -82,6 +82,14 @@ extern uint32_t snrt_cluster_partial_barrier_mask(const uint32_t *cids, uint32_t /// by snrt_cluster_partial_barrier_mask()). O(1) — a single store. extern void snrt_cluster_partial_barrier(uint32_t local_mask); +/// Barrier across only host-0 (primary) harts. Callable unconditionally +/// from any hart; host-1 harts return immediately without participating. +extern void snrt_cluster_host0_barrier(); + +/// Barrier across only host-1 (secondary) harts. Callable unconditionally +/// from any hart; host-0 harts return immediately without participating. +extern void snrt_cluster_host1_barrier(); + static inline uint32_t __attribute__((pure)) snrt_hartid(); struct snrt_team_root *snrt_current_team(); extern struct snrt_peripherals *snrt_peripherals(); @@ -97,6 +105,9 @@ extern uint32_t snrt_cluster_core_per_tile(); extern uint32_t snrt_cluster_idx(); extern uint32_t snrt_cluster_num(); +/// whether this hart is host 0 of its Core Complex pair +extern int snrt_cluster_is_primary(); + /// get pointer to barrier register extern uint32_t _snrt_barrier_reg_ptr(); diff --git a/software/snRuntime/include/spatz_lock.h b/software/snRuntime/include/spatz_lock.h new file mode 100644 index 00000000..8f73f5ec --- /dev/null +++ b/software/snRuntime/include/spatz_lock.h @@ -0,0 +1,17 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. + +// SPDX-License-Identifier: Apache-2.0 + +#include "cachepool_peripheral.h" +#include "team.h" + +extern __thread struct snrt_team *_snrt_team_current; + +// Acquire/release Spatz ownership on a dual-Snitch Core Complex. Each store +// blocks in hardware until granted, so no software polling is needed. Only +// call release after a successful acquire on the same hart, and only from +// harts sharing a Spatz (cachepool_cc_dual) -- on a single-scalar-per-CC +// build these write a harmless, unused peripheral register. +void spatz_lock_acquire(void); +void spatz_lock_release(void); diff --git a/software/snRuntime/src/barrier.c b/software/snRuntime/src/barrier.c index c5485d42..0e39dc51 100644 --- a/software/snRuntime/src/barrier.c +++ b/software/snRuntime/src/barrier.c @@ -44,6 +44,30 @@ void snrt_cluster_partial_barrier(uint32_t local_mask) { *(volatile uint32_t *)_snrt_barrier_reg_ptr() = local_mask; } +/// Tile-local mask of every hart whose parity matches want_primary (host 0 +/// positions if 1, host 1 positions if 0), given sequential host0/host1 +/// pairing; with no pairing at all, every hart counts as host 0. +static uint32_t snrt_cc_role_mask(int want_primary) { + uint32_t cpt = snrt_cluster_core_per_tile(); + uint32_t mask = 0; +#if SNRT_NUM_SCALAR_PER_CORE == 2 + for (uint32_t i = (want_primary ? 0 : 1); i < cpt; i += 2) mask |= (1u << i); +#else + if (want_primary) for (uint32_t i = 0; i < cpt; i++) mask |= (1u << i); +#endif + return mask; +} + +void snrt_cluster_host0_barrier() { + if (!snrt_cluster_is_primary()) return; + snrt_cluster_partial_barrier(snrt_cc_role_mask(1)); +} + +void snrt_cluster_host1_barrier() { + if (snrt_cluster_is_primary()) return; + snrt_cluster_partial_barrier(snrt_cc_role_mask(0)); +} + /// Synchronize cores in a cluster with a software barrier void snrt_cluster_sw_barrier() { // Remember previous iteration diff --git a/software/snRuntime/src/spatz_lock.c b/software/snRuntime/src/spatz_lock.c new file mode 100644 index 00000000..f3d10a63 --- /dev/null +++ b/software/snRuntime/src/spatz_lock.c @@ -0,0 +1,21 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. + +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +void spatz_lock_acquire(void) { + volatile uint32_t *lock = + (uint32_t *)(_snrt_team_current->root->cluster_mem.end + + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_REG_OFFSET); + *lock = 1; +} + +void spatz_lock_release(void) { + volatile uint32_t *lock = + (uint32_t *)(_snrt_team_current->root->cluster_mem.end + + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_REG_OFFSET); + *lock = 0; +} diff --git a/software/snRuntime/src/team.c b/software/snRuntime/src/team.c index 4ab61e66..d5940491 100644 --- a/software/snRuntime/src/team.c +++ b/software/snRuntime/src/team.c @@ -54,6 +54,14 @@ uint32_t snrt_cluster_tile_idx() { uint32_t snrt_cluster_core_idx() { return _snrt_core_idx; } +int snrt_cluster_is_primary() { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + return (_snrt_core_idx % 2) == 0; +#else + return 1; +#endif +} + uint32_t snrt_cluster_core_num() { return _snrt_team_current->root->cluster_core_num; } diff --git a/software/tests/idotp-32b/main.c b/software/tests/idotp-32b/main.c index 576e92bd..160bc393 100644 --- a/software/tests/idotp-32b/main.c +++ b/software/tests/idotp-32b/main.c @@ -18,6 +18,7 @@ #include #include +#include #include #include DATAHEADER @@ -29,6 +30,12 @@ int main() { const uint32_t num_cores = snrt_cluster_core_num(); const uint32_t cid = snrt_cluster_core_idx(); + // Diagnostic: have host 0 of each pair hold the Spatz lock for the whole + // kernel, without gating any of the execution below on it. + if (snrt_cluster_is_primary()) { + spatz_lock_acquire(); + } + const int measure_iter = 2; // Byte-level interleaving for DRAM @@ -50,13 +57,13 @@ int main() { uint32_t offset = 31 - __builtin_clz(dim * sizeof(int)); // Set xbar policy - l1d_xbar_config(offset); + // l1d_xbar_config(offset); if (cid == 0) { printf ("round:%u, lmul:%u, dim:%u\n", rounds, lmul, dim); } - snrt_cluster_hw_barrier(); + snrt_cluster_host0_barrier(); // Reset timer uint32_t timer = (uint32_t)-1; @@ -66,12 +73,13 @@ int main() { int *a_int = dotp_A_dram + dim * cid; int *b_int = dotp_B_dram + dim * cid; + for (int iter = 0; iter < measure_iter; iter ++) { // Start dump if (cid == 0) start_kernel(); - snrt_cluster_hw_barrier(); + // snrt_cluster_hw_barrier(); // Start timer timer_tmp = benchmark_get_cycle(); @@ -92,8 +100,9 @@ int main() { result[cid] = acc; - // Wait for all cores to finish - snrt_cluster_hw_barrier(); + // Host 0 only: this kernel doesn't care about host 1's progress or + // data at all, so no barrier or read ever involves it. + snrt_cluster_host0_barrier(); // End timer and check if new best runtime if (cid == 0) { @@ -105,10 +114,16 @@ int main() { stop_kernel(); } - // Final reduction +#if SNRT_NUM_SCALAR_PER_CORE == 2 + const uint32_t pair_stride = 2; +#else + const uint32_t pair_stride = 1; +#endif + + // Final reduction over host 0's own slots only. if (cid == 0) { - // timer_tmp = benchmark_get_cycle() - timer_tmp; - for (uint32_t i = 1; i < num_cores; ++i) + acc = result[0]; + for (uint32_t i = pair_stride; i < num_cores; i += pair_stride) acc += result[i]; result[0] = acc; @@ -116,10 +131,9 @@ int main() { printf("results:%u\n", result[0]); #endif } - } - snrt_cluster_hw_barrier(); + snrt_cluster_host0_barrier(); // Check and display results if (cid == 0) { @@ -150,7 +164,7 @@ int main() { } // Wait for core 0 to display the results - snrt_cluster_hw_barrier(); + snrt_cluster_host0_barrier(); return 0; } \ No newline at end of file diff --git a/util/scripts/gen_spatz_cfg.py b/util/scripts/gen_spatz_cfg.py index 030695c3..e957868a 100644 --- a/util/scripts/gen_spatz_cfg.py +++ b/util/scripts/gen_spatz_cfg.py @@ -38,7 +38,11 @@ def main(): # Build dynamic values that depend on others num_cores = int(os.environ.get("num_cores", "4")) num_tiles = int(os.environ.get("num_tiles", "1")) - cores_array = ",".join(['{ $ref: "#/compute_core_template" }'] * num_cores) + num_scalar_per_core = int(os.environ.get("num_scalar_per_core", "1")) + # cluster.cores must list one entry per hart, not per CC slot, since + # generate_bootdata.py derives SNRT_BOOT_CORE_COUNT from its length. + num_harts = num_cores * num_scalar_per_core + cores_array = ",".join(['{ $ref: "#/compute_core_template" }'] * num_harts) # Convert spatz_fpu_en -> spatz_fpu_bool for HJSON spatz_fpu_bool = bool_str(os.environ.get("spatz_fpu_en", "0")) From cd3a5e6fed588927788dd78fedf8059bf74caa7d Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Wed, 19 Aug 2026 11:59:58 +0200 Subject: [PATCH 08/17] [SW] Add missing makefile for configuration --- config/config.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/config.mk b/config/config.mk index b0449bf8..ce97dde5 100644 --- a/config/config.mk +++ b/config/config.mk @@ -131,6 +131,9 @@ spatz_max_trans ?= 32 # Snitch/FPU max outstanding transactions snitch_max_trans ?= 16 +# Number of Snitch scalar harts sharing one Spatz per Core Complex (1 or 2) +num_scalar_per_core ?= 1 + ######################### ## AXI configuration ## From e09e462b72cec6bb7dc5802dc92864296a5dcae9 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Wed, 19 Aug 2026 12:03:52 +0200 Subject: [PATCH 09/17] [Makefile] Update venv environment to force python3.12. Update makefile to pass-in dual-scalar core configures --- Makefile | 11 ++++++++--- README.md | 3 +++ iis-env.sh | 9 ++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9edab93f..446db150 100644 --- a/Makefile +++ b/Makefile @@ -190,12 +190,12 @@ $(info FLOO_DIR: $(FLOO_DIR)) # Generates the sources for FlooNoC .PHONY: update-floonoc install-floogen clean-floonoc install-floogen: - pip install -e $(FLOO_DIR) --quiet + $(PYTHON) -m pip install -e $(FLOO_DIR) --quiet update-floonoc: $(FLOO_NOC) $(FLOO_NOC): $(FLOO_CFG) mkdir -p $(FLOO_GEN_OUTDIR) - PATH="$(HOME)/.local/bin:$(PATH)" floogen pkg -c $(FLOO_CFG) -o $(FLOO_GEN_OUTDIR) --no-format + floogen pkg -c $(FLOO_CFG) -o $(FLOO_GEN_OUTDIR) --no-format clean-floonoc: rm -f $(FLOO_NOC) @@ -298,6 +298,10 @@ VLOG_DEFS += -DSPATZ_NUM_FPU=$(spatz_num_fpu) VLOG_DEFS += -DSPATZ_NUM_IPU=$(spatz_num_ipu) VLOG_DEFS += -DSPATZ_MAX_TRANS=$(spatz_max_trans) VLOG_DEFS += -DSNITCH_MAX_TRANS=$(snitch_max_trans) +VLOG_DEFS += -DNUM_SCALAR_PER_CC=$(num_scalar_per_core) +ifeq ($(num_scalar_per_core),2) +VLOG_DEFS += -DCACHEPOOL_DUAL_CC +endif VLOG_DEFS += -DLG_PORT_PER_CORE=$(num_lg_ports_per_core) VLOG_DEFS += -DRG_PORT_PER_CORE=$(num_rg_ports_per_core) VLOG_DEFS += -DNOC_PORT_PER_TILE=$(num_noc_ports_per_tile) @@ -411,7 +415,8 @@ SW_CMAKE_FLAGS = \ -DLLVM_PATH=${LLVM_INSTALL_DIR} \ -DGCC_PATH=${GCC_INSTALL_DIR} \ -DPYTHON=${PYTHON} \ - -DBUILD_TESTS=ON + -DBUILD_TESTS=ON \ + -DNUM_SCALAR_PER_CORE=$(num_scalar_per_core) .PHONY: sw sw: generate bootrom gen-data diff --git a/README.md b/README.md index ed692ec8..ff48aa2d 100644 --- a/README.md +++ b/README.md @@ -204,9 +204,12 @@ Configuration names encode the number of groups and whether the FPU is enabled: | `cachepool_fpu_4g` | 4 | 2×2 | Yes | 4 | 4 | 64 | | `cachepool_fpu_16g` | 16 | 4×4 | Yes | 4 | 4 | 256 | | `cachepool_fpu_16g_tiny` | 16 | 4×4 | Yes | 2 | 2 | 64 | +| `cachepool_dual_4g` | 4 | 2×2 | No (IPU only) | 4 | 4 | 64 CCs / 128 harts | `cachepool_fpu_16g_tiny` shrinks tiles/group and cores/tile for a faster-to-build, faster-to-simulate smoke test of the full 16-group mesh topology. +`cachepool_dual_4g` sets `num_scalar_per_core=2`: each Core Complex holds 2 Snitch scalar harts sharing 1 Spatz unit via a hardware ownership lock, so "Cores" (Core Complex slots) and hart count diverge — 64 CCs, 128 harts total. Software runtime support for the shared-Spatz lock is still in progress (see `note.md`). + The Spatz cluster consumes **`config/cachepool.hjson`**, which is **generated** from: - `config/cachepool.hjson.tmpl` (skeleton with comments) - `config/config.mk` (source of truth) diff --git a/iis-env.sh b/iis-env.sh index 787ba0cb..341ee936 100644 --- a/iis-env.sh +++ b/iis-env.sh @@ -13,7 +13,14 @@ export INSTALL_DIR=/home/dishen/cachepool-32b/install # Python deps for hardware code generation (make generate) - venv instead of uv, # to match the existing local dev flow. -python3 -m venv cachepool +# python3.12 (not the default python3) is required: floogen needs Python >=3.10. +PYTHON=python3.12 +export PYTHON +$PYTHON -m venv cachepool source cachepool/bin/activate python3 -m pip install --quiet --upgrade pip python3 -m pip install --quiet -r requirements.txt + +# floogen (FlooNoC code generator) is pulled in via bender, not requirements.txt - +# install it editable into this venv so `floogen` on PATH resolves here. +make install-floogen PYTHON=python3 --no-print-directory From 5728dd24455d80ffc8fdf2b3a91de5b3d28e22bf Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Thu, 20 Aug 2026 12:36:36 +0200 Subject: [PATCH 10/17] [Makefile] Update makefile targets --- Makefile | 33 ++++++++++++++++++++++++--------- README.md | 4 +++- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 446db150..264d2f47 100644 --- a/Makefile +++ b/Makefile @@ -388,23 +388,32 @@ $(BANDWIDTH_DATA): $(TESTS_DIR)/bandwidth/script/data.json \ .PHONY: gen-data gen-data: $(ALL_GEN_DATA) +# Clean targets, used to delete generated files .PHONY: clean.data clean.data: rm -f $(ALL_GEN_DATA) .PHONY: clean.sw -clean.sw: +clean.sw: clean.data rm -rf ${SOFTWARE_DIR}/build -.PHONY: clean -clean: clean.sw clean.vsim clean.data +.PHONY: clean.generate +clean.generate: rm -rf $(HJSON_OUT) $(BOOTROM_DIR)/bootdata.cc \ $(BOOTROM_DIR)/bootdata_bootrom.cc \ $(BOOTROM_DIR)/bootrom.sv \ $(BOOTROM_DIR)/bootrom.dump \ $(BOOTROM_DIR)/bootrom.elf \ + $(CACHEPOOL_DIR)/wlf* \ + $(CACHEPOOL_DIR)/noc_profiling/* \ $(SNRT_BOOTINFO_H) +.PHONY: clean.hw +clean.hw: clean.vsim clean.generate + +.PHONY: clean +clean: clean.hw clean.sw + # Common CMake flags shared by sw and vsim targets. # vsim appends -DSNITCH_SIMULATOR to point tests at the compiled binary. SW_CMAKE_FLAGS = \ @@ -433,8 +442,11 @@ vsim: generate bootrom dpi ${SIMBIN_DIR}/cachepool_cluster.vsim ############ # Just a shortcut to build everything +.PHONY: hw +hw: generate bootrom vsim + .PHONY: all -all: generate bootrom vsim sw +all: hw sw ######## # Lint # @@ -468,9 +480,7 @@ avg-log: # NoC traffic visualization frontend (fork of https://github.com/ueqri/vis4mesh # with CachePool-specific fixes/features). Consumes the Vis4Mesh dataset # directories produced by util/scripts/noc_profiling_to_vis4mesh.py from -# hardware/tb/cachepool_noc_profiling.sv's per-cycle NoC logs. Fetched as a -# plain pinned clone (same pattern as toolchain.mk's riscv-gnu-toolchain/ -# llvm-project targets), not a git submodule. +# hardware/tb/cachepool_noc_profiling.sv's per-cycle NoC logs. NPM ?= npm VIS4MESH_DIR ?= ${CACHEPOOL_DIR}/util/vis4mesh VIS4MESH_REPO ?= https://github.com/DiyouS/vis4mesh.git @@ -536,13 +546,18 @@ help: @echo "SW Build:" @echo "" @echo "*sw*: build software (generate + bootrom + cmake); overwrites previous build" - @echo "*clean.sw*: remove the software build directory" + @echo "*clean.sw*: remove the software build directory (also runs clean.data)" @echo "" @echo "Simulation:" @echo "" @echo "*vsim*: build hardware for QuestaSim simulation (use 'sw' to build software separately)" + @echo "*hw*: shortcut for generate + bootrom + vsim" + @echo "*all*: shortcut for hw + sw" @echo "*clean.vsim*: remove the hardware simulation build [from sim/sim.mk]" - @echo "*clean*: remove SW build, vsim build, and all generated HW files" + @echo "*clean.data*: remove generated data files (gen-data outputs)" + @echo "*clean.generate*: remove generated HJSON/bootrom/wlf/noc_profiling files [from 'generate'/'bootrom']" + @echo "*clean.hw*: clean.vsim + clean.generate" + @echo "*clean*: clean.hw + clean.sw (remove SW build, vsim build, and all generated HW files)" @echo "" @echo "Lint:" @echo "" diff --git a/README.md b/README.md index ff48aa2d..ade27ed5 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ make vsim config=cachepool_fpu_4g Set `DEBUG=0` to disable `+acc` waveform visibility and speed up simulation (used by CI); default is `DEBUG=1`. +`make hw` is a shortcut for `generate bootrom vsim`; `make all` is a shortcut for `hw sw` (build everything). + #### Run the Simulation The wrapper script launches the simulation (GUI or CLI) and expects a software ELF path as argument: @@ -411,7 +413,7 @@ make lint config=cachepool_fpu_4g - If you change cacheline width, `AXI_USER_WIDTH` is derived (supported widths: 128→19, 256→18, 512→17). Unsupported widths error out at generation time. - `make generate` regenerates the FlooNoC package automatically; only run `make update-floonoc` standalone if you're iterating on a `config/floonoc_*.yml` topology file without a full generate. - `make sw` and `make vsim` are decoupled (hw/sw build independently); rebuild whichever side you changed. -- Use `make clean` when switching configs to prevent stale build artifacts. +- Use `make clean` when switching configs to prevent stale build artifacts; `clean.sw`/`clean.hw` (or their finer-grained parts `clean.data`/`clean.generate`/`clean.vsim`) clean only one side if you don't need a full rebuild. - Runtime functions `snrt_tile_id()` and `snrt_num_tiles()` are available to query tile topology from software. - Changing the partition mode or boundary address while the cache holds valid data requires a flush (`l1d_cluster_flush()` or the appropriate cluster-wide partition flush) before reconfiguring. - Set `DEBUG=0` to disable `+acc` and speed up simulation (used by CI); default is `DEBUG=1` for waveform visibility. From 7ae10e720560cb21288989850ae93de922759211 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Thu, 20 Aug 2026 12:39:21 +0200 Subject: [PATCH 11/17] [SW] Add new test for dual scalar core. Fix bug accordingly --- hardware/src/cachepool_cc_dual.sv | 7 ++ hardware/src/cachepool_spatz_lock.sv | 34 ++++++- software/CMakeLists.txt | 7 ++ software/snRuntime/CMakeLists.txt | 3 + software/tests/CMakeLists.txt | 2 + software/tests/spatz-lock-handoff/main.c | 109 +++++++++++++++++++++++ 6 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 software/tests/spatz-lock-handoff/main.c diff --git a/hardware/src/cachepool_cc_dual.sv b/hardware/src/cachepool_cc_dual.sv index 161b568c..436c5869 100644 --- a/hardware/src/cachepool_cc_dual.sv +++ b/hardware/src/cachepool_cc_dual.sv @@ -237,6 +237,10 @@ module cachepool_cc_dual acc_rsp_t spatz_rsp; logic spatz_rsp_valid, spatz_rsp_ready; + // LSU issue handshake fire, gates the lock's LSU drain counter. + logic spatz_lsu_issue_fire; + assign spatz_lsu_issue_fire = spatz_issue_valid && spatz_issue_ready && spatz_issue_rsp.loadstore; + // Lock-module pass-through data interface (accept/deliver on the module's // own registered output side; see cachepool_spatz_lock.sv). dreq_t [1:0] lock_out_req; @@ -262,6 +266,9 @@ module cachepool_cc_dual .out_rsp_i (lock_out_rsp ), .req_fire_i (req_fire ), .rsp_fire_i (rsp_fire ), + .spatz_lsu_issue_fire_i (spatz_lsu_issue_fire), + .spatz_mem_finished_i (spatz_mem_finished ), + .spatz_st_rsp_done_i (spatz_st_rsp_done ), .owner_id_o (owner_id ), .locked_o (locked ), .waiting_o (waiting ), diff --git a/hardware/src/cachepool_spatz_lock.sv b/hardware/src/cachepool_spatz_lock.sv index c582288d..4b541ddc 100644 --- a/hardware/src/cachepool_spatz_lock.sv +++ b/hardware/src/cachepool_spatz_lock.sv @@ -15,7 +15,9 @@ /// decides ownership and exposes owner_id_o/locked_o/waiting_o for the CC /// to apply, and counts real Spatz acc handshakes (via /// req_fire_i/rsp_fire_i, fed back by acc_mux) to gate a switch until -/// fully drained. +/// fully drained. Separately tracks outstanding vle/vse ops (invisible to +/// req_fire_i/rsp_fire_i, since they never produce an acc_rsp_t writeback) +/// via Spatz's own spatz_mem_finished_i/spatz_st_rsp_done_i. /// Also hosts the pass-through memory-path register cuts (moved here from /// the per-core spill registers in cachepool_cc.sv). module cachepool_spatz_lock @@ -46,6 +48,13 @@ module cachepool_spatz_lock input logic req_fire_i, input logic rsp_fire_i, + // Spatz LSU drain tracking: vle/vse never produce an acc_rsp_t completion + // (writeback=0), so they are invisible to req_fire_i/rsp_fire_i above and + // need their own outstanding count, gated by Spatz's own signals. + input logic spatz_lsu_issue_fire_i, + input logic [1:0] spatz_mem_finished_i, + input logic spatz_st_rsp_done_i, + // owner/lock status and error output logic owner_id_o, output logic locked_o, @@ -88,6 +97,9 @@ module cachepool_spatz_lock // Outstanding acc handshakes on the owner's path; must reach 0 before a wait can complete. logic [7:0] outstanding_d, outstanding_q; + // Outstanding Spatz vle/vse ops; mirrors snitch.sv's own acc_mem_cnt_q so a + // switch can't happen while a load/store is still draining through the cache. + logic [7:0] lsu_outstanding_d, lsu_outstanding_q; logic drain_done, waiting; assign owner_id_o = owner_q; @@ -96,9 +108,11 @@ module cachepool_spatz_lock assign error_o = error_q; assign waiting = (lock_q == AcqWait) || (lock_q == RelWait); - assign drain_done = (outstanding_q == '0); + assign drain_done = (outstanding_q == '0) && (lsu_outstanding_q == '0) && spatz_st_rsp_done_i; `ASSERT(NoOutstandingUnderflow, rsp_fire_i |-> (outstanding_q != '0)) + `ASSERT(NoLsuOutstandingUnderflow, + (spatz_mem_finished_i[0] || spatz_mem_finished_i[1]) |-> (lsu_outstanding_q != '0)) always_comb begin outstanding_d = outstanding_q; @@ -111,6 +125,19 @@ module cachepool_spatz_lock end end + always_comb begin + lsu_outstanding_d = lsu_outstanding_q; + if (spatz_lsu_issue_fire_i) begin + lsu_outstanding_d = lsu_outstanding_d + 8'd1; + end + if (spatz_mem_finished_i[0]) begin + lsu_outstanding_d = lsu_outstanding_d - 8'd1; + end + if (spatz_mem_finished_i[1]) begin + lsu_outstanding_d = lsu_outstanding_d - 8'd1; + end + end + // write 1 for acquire, write 0 for release, read returns the lock status logic [1:0] is_acquire, is_release, is_read, lock_hit; @@ -320,7 +347,8 @@ module cachepool_spatz_lock `FF(error_q, error_d, 1'b0, clk_i, rst_ni) `FF(active_q, active_d, 1'b0, clk_i, rst_ni) `FF(user_q, user_d, '0, clk_i, rst_ni) - `FF(outstanding_q, outstanding_d, '0, clk_i, rst_ni) + `FF(outstanding_q, outstanding_d, '0, clk_i, rst_ni) + `FF(lsu_outstanding_q, lsu_outstanding_d, '0, clk_i, rst_ni) `FF(resp_pending_q, resp_pending_d, 1'b0, clk_i, rst_ni) `FF(resp_host_q, resp_host_d, 1'b0, clk_i, rst_ni) `FF(resp_user_q, resp_user_d, '0, clk_i, rst_ni) diff --git a/software/CMakeLists.txt b/software/CMakeLists.txt index 88de4857..5aa368d3 100644 --- a/software/CMakeLists.txt +++ b/software/CMakeLists.txt @@ -15,6 +15,13 @@ project(snitch_cluster LANGUAGES C ASM) include(SnitchUtilities) enable_testing() + +# Set here (not just in snRuntime/CMakeLists.txt) so sibling directories +# added below, e.g. software/tests, also see this define -- add_compile_definitions +# is a directory property and does not propagate to sibling add_subdirectory trees. +set(NUM_SCALAR_PER_CORE "1" CACHE STRING "Snitch harts sharing one Spatz per Core Complex") +add_compile_definitions(SNRT_NUM_SCALAR_PER_CORE=${NUM_SCALAR_PER_CORE}) + add_subdirectory(${SNITCH_SOFTWARE_DIR}/snRuntime snRuntime) # add_subdirectory(${SPATZ_SOFTWARE_DIR}/riscvTests riscvTests) # add_subdirectory(${SPATZ_SOFTWARE_DIR}/spatzBenchmarks spatzBenchmarks) diff --git a/software/snRuntime/CMakeLists.txt b/software/snRuntime/CMakeLists.txt index 39debc5e..a489b963 100644 --- a/software/snRuntime/CMakeLists.txt +++ b/software/snRuntime/CMakeLists.txt @@ -49,6 +49,9 @@ if(RUNTIME_TRACE) add_compile_definitions(__SNRT_USE_TRACE) endif() +# Also set/applied in the parent software/CMakeLists.txt so sibling +# directories (e.g. software/tests) see the same define; kept here too for +# standalone snRuntime builds. set(NUM_SCALAR_PER_CORE "1" CACHE STRING "Snitch harts sharing one Spatz per Core Complex") add_compile_definitions(SNRT_NUM_SCALAR_PER_CORE=${NUM_SCALAR_PER_CORE}) diff --git a/software/tests/CMakeLists.txt b/software/tests/CMakeLists.txt index d82b0494..e12c0b6d 100644 --- a/software/tests/CMakeLists.txt +++ b/software/tests/CMakeLists.txt @@ -141,4 +141,6 @@ add_spatz_test_oneParam(idotp-32b idotp-32b/main.c 32768) add_spatz_test_zeroParam(load-store load-store/main.c) +add_spatz_test_zeroParam(spatz-lock-handoff spatz-lock-handoff/main.c) + add_spatz_test_zeroParam(bandwidth bandwidth/main.c) diff --git a/software/tests/spatz-lock-handoff/main.c b/software/tests/spatz-lock-handoff/main.c new file mode 100644 index 00000000..3df38863 --- /dev/null +++ b/software/tests/spatz-lock-handoff/main.c @@ -0,0 +1,109 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// SPDX-License-Identifier: Apache-2.0 + +// Lock handoff test: proves the Spatz lock actually serializes host0/host1, +// not just that Spatz produces correct data for independent requesters. +// v0 is physical Spatz register state shared across the lock handoff -- host0 +// loads a pattern into v0, then both hosts (racing for the lock) add a fixed +// role constant (host0: 1, host1: 2) directly to v0, and host0 stores the +// final result. Fixed constants (not cid) keep the expected sum identical +// across every pair, so a failure is easy to spot and debug. A lost update +// (lock failed to serialize) yields a wrong final sum, which a +// per-hart-disjoint-slice test could never detect. + +#include +#include +#include +#include +#include + +#define MAX_PAIRS 128U +#define TEST_LEN 64U // vector elements per pair, fits one vsetvli group +#define PATTERN 1U + +#if SNRT_NUM_SCALAR_PER_CORE == 2 +#define PAIR_SIZE 2U +#define EXPECT (PATTERN + 3U) // host0 adds 1, host1 adds 2 +#else +#define PAIR_SIZE 1U +#define EXPECT (PATTERN + 1U) // host0 only, adds 1 +#endif + +static uint32_t src_dram[TEST_LEN] __attribute__((section(".data"))) + = {[0 ... TEST_LEN - 1] = PATTERN}; +static uint32_t dst_dram[MAX_PAIRS][TEST_LEN] __attribute__((section(".data"))); +static uint32_t pair_errors[MAX_PAIRS] __attribute__((section(".data"))); + +int main() { + const uint32_t cid = snrt_cluster_core_idx(); + const uint32_t pair = cid / PAIR_SIZE; + const uint32_t add_val = snrt_cluster_is_primary() ? 1U : 2U; + uint32_t vlen; + + if (cid == 0) { + printf("*** spatz-lock-handoff test ***\n"); + printf("PATTERN=%u PAIR_SIZE=%u EXPECT=%u\n", PATTERN, PAIR_SIZE, EXPECT); + } + snrt_cluster_hw_barrier(); + + // Phase 1 (host0 only): load the seed pattern into v0 and hold the lock + // open just long enough to establish it -- no store, v0 carries the state. + if (snrt_cluster_is_primary()) { + spatz_lock_acquire(); + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vlen) : "r"(TEST_LEN)); + asm volatile("vle32.v v0, (%0)" : : "r"(src_dram)); + spatz_lock_release(); + } + snrt_cluster_hw_barrier(); + + // Phase 2 (both hosts): racing add directly on v0, serialized by the lock + // -- a lost update here means the lock failed to exclude concurrent access. + spatz_lock_acquire(); + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vlen) : "r"(TEST_LEN)); + asm volatile("vadd.vx v0, v0, %0" : : "r"(add_val)); + spatz_lock_release(); + snrt_cluster_hw_barrier(); + + // Phase 3 (host0 only): store the final accumulated v0 for checking. + if (snrt_cluster_is_primary()) { + spatz_lock_acquire(); + asm volatile("vsetvli %0, %1, e32, m8, ta, ma" : "=r"(vlen) : "r"(TEST_LEN)); + asm volatile("vse32.v v0, (%0)" : : "r"(dst_dram[pair])); + spatz_lock_release(); + + uint32_t errs = 0; + for (uint32_t i = 0; i < TEST_LEN; i++) { + if (dst_dram[pair][i] != EXPECT) { + errs++; + } + } + pair_errors[pair] = errs; + } + snrt_cluster_hw_barrier(); + + if (cid == 0) { + const uint32_t num_cores = snrt_cluster_core_num(); + const uint32_t num_pairs = num_cores / PAIR_SIZE; + uint32_t total_err = 0; + for (uint32_t p = 0; p < num_pairs; p++) { + total_err += pair_errors[p]; + } + + if (total_err == 0) { + printf("[PASS] spatz-lock-handoff: pairs=%u len=%u\n", num_pairs, TEST_LEN); + } else { + printf("[FAIL] spatz-lock-handoff: errors=%u pairs=%u len=%u\n", + total_err, num_pairs, TEST_LEN); + printf(" expect=%u\n", EXPECT); + for (uint32_t p = 0; p < num_pairs; p++) { + if (pair_errors[p] != 0) { + printf(" pair %u: dst_dram[0]=%u errs=%u\n", p, dst_dram[p][0], + pair_errors[p]); + } + } + } + } + snrt_cluster_hw_barrier(); + + return 0; +} From e2f40db151697b0825364aca939121a8dc8e883f Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Mon, 24 Aug 2026 11:42:00 +0200 Subject: [PATCH 12/17] [SW] Reconstruct software kernel folder to place them in better locations --- .gitlab-ci.yml | 2 +- Makefile | 22 +-- README.md | 4 +- software/CMakeLists.txt | 10 ++ software/README.md | 63 ++++++++ software/cache/CMakeLists.txt | 14 ++ software/{tests => cache}/byte-enable/main.c | 0 .../cache-coverage-min/main.c | 0 .../{tests => cache}/cache-coverage/main.c | 0 .../cache-line-rw-smoke/main.c | 0 .../cache-mix-pressure/main.c | 0 .../{tests => cache}/cache-mix-smoke/main.c | 0 .../{tests => cache}/cache-rlc-mimic/main.c | 0 .../{tests => cache}/cache-test-scalar/main.c | 0 .../{tests => cache}/cache-test-vector/main.c | 0 .../{tests => cache}/cache-vector-rw/main.c | 0 software/cmake/CachePoolTests.cmake | 57 +++++++ software/kernels/fp/CMakeLists.txt | 30 ++++ .../fp}/fdotp-32b/data/.gitignore | 0 .../fp}/fdotp-32b/data/layer.h | 0 .../fp}/fdotp-32b/kernel/fdotp.c | 0 .../fp}/fdotp-32b/kernel/fdotp.h | 0 .../{tests => kernels/fp}/fdotp-32b/main.c | 0 .../fp}/fdotp-32b/script/data_32768.json | 0 .../fp}/fdotp-32b/script/data_65536.json | 0 .../fp}/fdotp-32b/script/data_8192.json | 0 .../fp}/fdotp-32b/script/gen_data.py | 0 .../fp}/fft-32b/data/.gitignore | 0 .../fp}/fft-32b/data/data_1024_4.h | 0 .../fp}/fft-32b/kernel/fft.c | 0 .../fp}/fft-32b/kernel/fft.h | 0 software/{tests => kernels/fp}/fft-32b/main.c | 0 .../fp}/fft-32b/script/data_1024_16.json | 0 .../fp}/fft-32b/script/data_256_4.json | 0 .../fp}/fft-32b/script/fft_1024_4.json | 0 .../fp}/fft-32b/script/gen_data.py | 0 .../fp}/fmatmul-32b/data/.gitignore | 0 .../fp}/fmatmul-32b/data/layer.h | 0 .../fp}/fmatmul-32b/kernel/fmatmul.c | 0 .../fp}/fmatmul-32b/kernel/fmatmul.h | 0 .../{tests => kernels/fp}/fmatmul-32b/main.c | 0 .../fmatmul-32b/script/data_1024_32_32.json | 0 .../fmatmul-32b/script/data_1024_64_64.json | 0 .../fmatmul-32b/script/data_128_128_128.json | 0 .../fp}/fmatmul-32b/script/data_32_32_32.json | 0 .../fp}/fmatmul-32b/script/data_64_64_64.json | 0 .../fp}/fmatmul-32b/script/gen_data.py | 0 .../fp}/gemv-opt/data/.gitignore | 0 .../fp}/gemv-opt/data/layer.h | 0 .../fp}/gemv-opt/kernel/gemv.c | 0 .../fp}/gemv-opt/kernel/gemv.h | 0 .../{tests => kernels/fp}/gemv-opt/main.c | 0 .../fp}/gemv-opt/script/data_1024_128_32.json | 0 .../fp}/gemv-opt/script/data_128_128_32.json | 0 .../fp}/gemv-opt/script/data_256_128_32.json | 0 .../fp}/gemv-opt/script/data_512_128_32.json | 0 .../fp}/gemv-opt/script/gen_data.py | 0 .../fp}/gemv/data/.gitignore | 0 .../{tests => kernels/fp}/gemv/data/layer.h | 0 .../{tests => kernels/fp}/gemv/kernel/gemv.c | 0 .../{tests => kernels/fp}/gemv/kernel/gemv.h | 0 software/{tests => kernels/fp}/gemv/main.c | 0 .../fp}/gemv/script/data_1024_128_32.json | 0 .../fp}/gemv/script/data_128_128_32.json | 0 .../fp}/gemv/script/data_256_128_32.json | 0 .../fp}/gemv/script/data_512_128_32.json | 0 .../fp}/gemv/script/gen_data.py | 0 software/kernels/int/CMakeLists.txt | 8 + .../int}/idotp-32b/data/.gitignore | 0 .../int}/idotp-32b/data/layer.h | 0 .../int}/idotp-32b/kernel/idotp.c | 0 .../int}/idotp-32b/kernel/idotp.h | 0 .../{tests => kernels/int}/idotp-32b/main.c | 0 .../int}/idotp-32b/script/data_32768.json | 0 .../int}/idotp-32b/script/data_8192.json | 0 .../int}/idotp-32b/script/gen_data.py | 0 software/rlc/CMakeLists.txt | 16 ++ .../data/data_1_1350_10.h | 0 .../data/data_1_1350_100.h | 0 .../data/data_1_1350_1000.h | 0 .../data/data_1_1350_300.h | 0 .../data/data_1_2044_100.h | 0 .../data/rlc_flowchart | 0 .../data/rlc_flowchart.png | Bin .../kernel/data_move_vec.c | 0 .../kernel/data_move_vec.h | 0 .../kernel/llist.c | 0 .../kernel/llist.h | 0 .../kernel/mm.c | 0 .../kernel/mm.h | 0 .../kernel/printf_lock.c | 0 .../kernel/printf_lock.h | 0 .../kernel/rlc.c | 0 .../kernel/rlc.h | 0 .../main.c | 0 .../script/flowchart.py | 0 .../script/generate_pdcp_pkg.py | 0 .../script/pdcp_pkg.json | 0 software/sync/CMakeLists.txt | 9 ++ software/{tests => sync}/false-sharing/main.c | 0 .../mcs-lock/kernel/printf_lock.c | 0 .../mcs-lock}/kernel/printf_lock.h | 0 software/{tests => sync}/mcs-lock/main.c | 0 .../{tests => sync}/partial_barrier/main.c | 0 .../{tests => sync}/spatz-lock-handoff/main.c | 0 software/{tests => sync}/spin-lock/main.c | 0 software/tests/CMakeLists.txt | 140 +----------------- util/auto-benchmark/configs.sh | 2 +- 108 files changed, 225 insertions(+), 152 deletions(-) create mode 100644 software/README.md create mode 100644 software/cache/CMakeLists.txt rename software/{tests => cache}/byte-enable/main.c (100%) rename software/{tests => cache}/cache-coverage-min/main.c (100%) rename software/{tests => cache}/cache-coverage/main.c (100%) rename software/{tests => cache}/cache-line-rw-smoke/main.c (100%) rename software/{tests => cache}/cache-mix-pressure/main.c (100%) rename software/{tests => cache}/cache-mix-smoke/main.c (100%) rename software/{tests => cache}/cache-rlc-mimic/main.c (100%) rename software/{tests => cache}/cache-test-scalar/main.c (100%) rename software/{tests => cache}/cache-test-vector/main.c (100%) rename software/{tests => cache}/cache-vector-rw/main.c (100%) create mode 100644 software/cmake/CachePoolTests.cmake create mode 100644 software/kernels/fp/CMakeLists.txt rename software/{tests => kernels/fp}/fdotp-32b/data/.gitignore (100%) rename software/{tests => kernels/fp}/fdotp-32b/data/layer.h (100%) rename software/{tests => kernels/fp}/fdotp-32b/kernel/fdotp.c (100%) rename software/{tests => kernels/fp}/fdotp-32b/kernel/fdotp.h (100%) rename software/{tests => kernels/fp}/fdotp-32b/main.c (100%) rename software/{tests => kernels/fp}/fdotp-32b/script/data_32768.json (100%) rename software/{tests => kernels/fp}/fdotp-32b/script/data_65536.json (100%) rename software/{tests => kernels/fp}/fdotp-32b/script/data_8192.json (100%) rename software/{tests => kernels/fp}/fdotp-32b/script/gen_data.py (100%) rename software/{tests => kernels/fp}/fft-32b/data/.gitignore (100%) rename software/{tests => kernels/fp}/fft-32b/data/data_1024_4.h (100%) rename software/{tests => kernels/fp}/fft-32b/kernel/fft.c (100%) rename software/{tests => kernels/fp}/fft-32b/kernel/fft.h (100%) rename software/{tests => kernels/fp}/fft-32b/main.c (100%) rename software/{tests => kernels/fp}/fft-32b/script/data_1024_16.json (100%) rename software/{tests => kernels/fp}/fft-32b/script/data_256_4.json (100%) rename software/{tests => kernels/fp}/fft-32b/script/fft_1024_4.json (100%) rename software/{tests => kernels/fp}/fft-32b/script/gen_data.py (100%) rename software/{tests => kernels/fp}/fmatmul-32b/data/.gitignore (100%) rename software/{tests => kernels/fp}/fmatmul-32b/data/layer.h (100%) rename software/{tests => kernels/fp}/fmatmul-32b/kernel/fmatmul.c (100%) rename software/{tests => kernels/fp}/fmatmul-32b/kernel/fmatmul.h (100%) rename software/{tests => kernels/fp}/fmatmul-32b/main.c (100%) rename software/{tests => kernels/fp}/fmatmul-32b/script/data_1024_32_32.json (100%) rename software/{tests => kernels/fp}/fmatmul-32b/script/data_1024_64_64.json (100%) rename software/{tests => kernels/fp}/fmatmul-32b/script/data_128_128_128.json (100%) rename software/{tests => kernels/fp}/fmatmul-32b/script/data_32_32_32.json (100%) rename software/{tests => kernels/fp}/fmatmul-32b/script/data_64_64_64.json (100%) rename software/{tests => kernels/fp}/fmatmul-32b/script/gen_data.py (100%) rename software/{tests => kernels/fp}/gemv-opt/data/.gitignore (100%) rename software/{tests => kernels/fp}/gemv-opt/data/layer.h (100%) rename software/{tests => kernels/fp}/gemv-opt/kernel/gemv.c (100%) rename software/{tests => kernels/fp}/gemv-opt/kernel/gemv.h (100%) rename software/{tests => kernels/fp}/gemv-opt/main.c (100%) rename software/{tests => kernels/fp}/gemv-opt/script/data_1024_128_32.json (100%) rename software/{tests => kernels/fp}/gemv-opt/script/data_128_128_32.json (100%) rename software/{tests => kernels/fp}/gemv-opt/script/data_256_128_32.json (100%) rename software/{tests => kernels/fp}/gemv-opt/script/data_512_128_32.json (100%) rename software/{tests => kernels/fp}/gemv-opt/script/gen_data.py (100%) rename software/{tests => kernels/fp}/gemv/data/.gitignore (100%) rename software/{tests => kernels/fp}/gemv/data/layer.h (100%) rename software/{tests => kernels/fp}/gemv/kernel/gemv.c (100%) rename software/{tests => kernels/fp}/gemv/kernel/gemv.h (100%) rename software/{tests => kernels/fp}/gemv/main.c (100%) rename software/{tests => kernels/fp}/gemv/script/data_1024_128_32.json (100%) rename software/{tests => kernels/fp}/gemv/script/data_128_128_32.json (100%) rename software/{tests => kernels/fp}/gemv/script/data_256_128_32.json (100%) rename software/{tests => kernels/fp}/gemv/script/data_512_128_32.json (100%) rename software/{tests => kernels/fp}/gemv/script/gen_data.py (100%) create mode 100644 software/kernels/int/CMakeLists.txt rename software/{tests => kernels/int}/idotp-32b/data/.gitignore (100%) rename software/{tests => kernels/int}/idotp-32b/data/layer.h (100%) rename software/{tests => kernels/int}/idotp-32b/kernel/idotp.c (100%) rename software/{tests => kernels/int}/idotp-32b/kernel/idotp.h (100%) rename software/{tests => kernels/int}/idotp-32b/main.c (100%) rename software/{tests => kernels/int}/idotp-32b/script/data_32768.json (100%) rename software/{tests => kernels/int}/idotp-32b/script/data_8192.json (100%) rename software/{tests => kernels/int}/idotp-32b/script/gen_data.py (100%) create mode 100644 software/rlc/CMakeLists.txt rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/data/data_1_1350_10.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/data/data_1_1350_100.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/data/data_1_1350_1000.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/data/data_1_1350_300.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/data/data_1_2044_100.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart.png (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/kernel/llist.c (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/kernel/llist.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/kernel/mm.c (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/kernel/mm.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c (100%) rename software/{tests/mcs-lock => rlc/multi_producer_single_consumer_double_linked_list}/kernel/printf_lock.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/kernel/rlc.c (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/kernel/rlc.h (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/main.c (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/script/flowchart.py (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/script/generate_pdcp_pkg.py (100%) rename software/{tests => rlc}/multi_producer_single_consumer_double_linked_list/script/pdcp_pkg.json (100%) create mode 100644 software/sync/CMakeLists.txt rename software/{tests => sync}/false-sharing/main.c (100%) rename software/{tests => sync}/mcs-lock/kernel/printf_lock.c (100%) rename software/{tests/multi_producer_single_consumer_double_linked_list => sync/mcs-lock}/kernel/printf_lock.h (100%) rename software/{tests => sync}/mcs-lock/main.c (100%) rename software/{tests => sync}/partial_barrier/main.c (100%) rename software/{tests => sync}/spatz-lock-handoff/main.c (100%) rename software/{tests => sync}/spin-lock/main.c (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc708ce0..04a9c647 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ variables: python3: 'python3' # Config to build and test CI_CONFIG: 'cachepool_fpu_4g' - SW_PREFIX: 'test-cachepool-' + SW_PREFIX: 'test-' default: tags: [shared] diff --git a/Makefile b/Makefile index 264d2f47..d8a252ee 100644 --- a/Makefile +++ b/Makefile @@ -355,24 +355,28 @@ include sim/sim.mk TESTS_DIR := $(SOFTWARE_DIR)/tests +# Test-group roots (relative to SOFTWARE_DIR) scanned for data-generated tests. +TEST_ROOTS := cache sync kernels/fp kernels/int rlc tests + # Auto-discover every test that has a script/gen_data.py. # Convention: script/data_.json → data/data_.h # Adding a new test or a new data variant needs no Makefile changes: # - new test: drop gen_data.py + data_.json into its script/ dir # - new variant: add data_.json to an existing test's script/ dir -DATA_TESTS := $(patsubst $(TESTS_DIR)/%/script/gen_data.py,%, \ - $(wildcard $(TESTS_DIR)/*/script/gen_data.py)) +DATA_TESTS := $(patsubst $(SOFTWARE_DIR)/%/script/gen_data.py,%, \ + $(foreach root,$(TEST_ROOTS), \ + $(wildcard $(SOFTWARE_DIR)/$(root)/*/script/gen_data.py))) define gen_data_rules -$(1)_DATA := $$(patsubst $(TESTS_DIR)/$(1)/script/data_%.json, \ - $(TESTS_DIR)/$(1)/data/data_%.h, \ - $$(wildcard $(TESTS_DIR)/$(1)/script/data_*.json)) +$(1)_DATA := $$(patsubst $(SOFTWARE_DIR)/$(1)/script/data_%.json, \ + $(SOFTWARE_DIR)/$(1)/data/data_%.h, \ + $$(wildcard $(SOFTWARE_DIR)/$(1)/script/data_*.json)) ALL_GEN_DATA += $$($(1)_DATA) -$(TESTS_DIR)/$(1)/data/data_%.h: \ - $(TESTS_DIR)/$(1)/script/data_%.json \ - $(TESTS_DIR)/$(1)/script/gen_data.py - $$(PYTHON) $(TESTS_DIR)/$(1)/script/gen_data.py -c $$< +$(SOFTWARE_DIR)/$(1)/data/data_%.h: \ + $(SOFTWARE_DIR)/$(1)/script/data_%.json \ + $(SOFTWARE_DIR)/$(1)/script/gen_data.py + $$(PYTHON) $(SOFTWARE_DIR)/$(1)/script/gen_data.py -c $$< endef $(foreach test,$(DATA_TESTS),$(eval $(call gen_data_rules,$(test)))) diff --git a/README.md b/README.md index ade27ed5..4366ea8d 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ A lightweight benchmarking automation flow is provided under `util/auto-benchmar CONFIGS="cachepool_fpu_4g cachepool_fpu_16g" KERNELS="fdotp-32b_M32768 ffft-64b_M16384 fmatmul-64b_M2048" - PREFIX="test-cachepool-" + PREFIX="test-" ROOT_PATH=../.. 2. Run all builds and simulations: @@ -346,7 +346,7 @@ For a spatial, time-scrubbable view of NoC traffic (as opposed to `cachepool_mon 1. **Enable the profiler and run a kernel** (default `noc_profiling ?= 1`, so this is on unless you passed `noc_profiling=0`): ```sh make vsim config= noc_profiling=1 - ./sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/test-cachepool- + ./sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/test- ``` L1 (inter-group cache-access mesh) logs (`router_g*.log`, `tile_g*.log`) only populate with `num_rg_ports_per_core > 0` (a multi-group config); L2 (DRAM-refill mesh) logs (`l2_router_g*.log`) and per-core PE logs (`pe_g*.log`) are always populated. diff --git a/software/CMakeLists.txt b/software/CMakeLists.txt index 5aa368d3..64b8ed7e 100644 --- a/software/CMakeLists.txt +++ b/software/CMakeLists.txt @@ -29,5 +29,15 @@ add_subdirectory(${SNITCH_SOFTWARE_DIR}/snRuntime snRuntime) option(ENABLE_CACHEPOOL_TESTS "Enable CachePool tests for spatz" OFF) if (ENABLE_CACHEPOOL_TESTS) message(STATUS "Adding CachePool tests") + # Shared macros (add_spatz_test_*) and common libraries (benchmark, + # spin_lock, mcs_lock); must be included before the group subdirectories + # below, since they use these macros/libraries in their own CMakeLists. + include(${CACHEPOOL_DIR}/software/cmake/CachePoolTests.cmake) + + add_subdirectory(${CACHEPOOL_DIR}/software/cache cache) + add_subdirectory(${CACHEPOOL_DIR}/software/sync sync) + add_subdirectory(${CACHEPOOL_DIR}/software/kernels/fp kernels_fp) + add_subdirectory(${CACHEPOOL_DIR}/software/kernels/int kernels_int) + add_subdirectory(${CACHEPOOL_DIR}/software/rlc rlc) add_subdirectory(${CACHEPOOL_DIR}/software/tests CachePoolTests) endif() diff --git a/software/README.md b/software/README.md new file mode 100644 index 00000000..589e53d8 --- /dev/null +++ b/software/README.md @@ -0,0 +1,63 @@ +# CachePool Software + +``` +software/ +├── snRuntime/ # bare-metal C runtime (barriers, l1 cache control, printf, +│ # allocator, Spatz lock, ...), linked into every test +├── cache/ # L1D cache correctness/stress tests +├── sync/ # locks, barriers, and other concurrency primitives +├── kernels/ +│ ├── fp/ # floating-point vector kernels (fdotp, fmatmul, gemv, fft, ...) +│ └── int/ # integer vector kernels (idotp, ...) +├── rlc/ # RLC (multi-producer/single-consumer) application test(s) +├── tests/ # smoke tests / one-off repros that don't fit another group, +│ # plus the shared benchmark/spin_lock/mcs_lock helper libraries +│ # and headers (tests/benchmark, tests/include) used by all groups +└── cmake/ # shared CMake macros and toolchain files +``` + +Each test lives in its own directory (e.g. `kernels/fp/fdotp-32b/`): +`main.c`, plus, for data-driven kernels, `kernel/` (kernel source), `data/` +(generated headers, gitignored), and `script/` (`gen_data.py` + +`data_.json` golden-value definitions). + +## Building + +From the repository root: + +``` +make sw config= # build all registered tests +``` + +Binaries land in `software/build/CachePoolTests/`, named +`test-`. Run one against the RTL simulator with: + +``` +./sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/test- +``` + +## Adding a test + +1. Create the test's directory under the group it belongs to (`cache/`, + `sync/`, `kernels/fp/`, `kernels/int/`, `rlc/`, or `tests/` for + ungrouped smoke tests/repros). +2. Register it in that group's `CMakeLists.txt` with one of the + `add_spatz_test_{zeroParam,oneParam,twoParam,threeParam}` macros + (`zeroParam` for a fixed test, `oneParam`/`twoParam`/`threeParam` for + tests swept over 1-3 named parameters via `DATAHEADER`) — mirror an + existing entry in that file. +3. For a data-driven kernel, add `script/gen_data.py` and one + `script/data_.json` per parameter combination; golden headers + under `data/` are generated automatically by `make gen-data` (also run + as part of `make sw`). + +## Adding a new group + +Add a directory under `software/`, give it a `CMakeLists.txt` with its own +`add_spatz_test_*` calls, and add an `add_subdirectory(...)` line for it in +`software/CMakeLists.txt`, inside the `ENABLE_CACHEPOOL_TESTS` block. + +## Removing a test + +Delete its directory and the corresponding `add_spatz_test_*` line(s) from +the group's `CMakeLists.txt`. diff --git a/software/cache/CMakeLists.txt b/software/cache/CMakeLists.txt new file mode 100644 index 00000000..4c3c6282 --- /dev/null +++ b/software/cache/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_spatz_test_zeroParam(byte-enable byte-enable/main.c) +add_spatz_test_zeroParam(cache-line-rw-smoke cache-line-rw-smoke/main.c) +add_spatz_test_zeroParam(cache-test-scalar cache-test-scalar/main.c) +add_spatz_test_zeroParam(cache-test-vector cache-test-vector/main.c) +add_spatz_test_zeroParam(cache-mix-smoke cache-mix-smoke/main.c) +add_spatz_test_zeroParam(cache-mix-pressure cache-mix-pressure/main.c) +add_spatz_test_zeroParam(cache-rlc-mimic cache-rlc-mimic/main.c) +add_spatz_test_zeroParam(cache-vector-rw cache-vector-rw/main.c) +add_spatz_test_zeroParam(cache-coverage cache-coverage/main.c) +add_spatz_test_zeroParam(cache-coverage-min cache-coverage-min/main.c) diff --git a/software/tests/byte-enable/main.c b/software/cache/byte-enable/main.c similarity index 100% rename from software/tests/byte-enable/main.c rename to software/cache/byte-enable/main.c diff --git a/software/tests/cache-coverage-min/main.c b/software/cache/cache-coverage-min/main.c similarity index 100% rename from software/tests/cache-coverage-min/main.c rename to software/cache/cache-coverage-min/main.c diff --git a/software/tests/cache-coverage/main.c b/software/cache/cache-coverage/main.c similarity index 100% rename from software/tests/cache-coverage/main.c rename to software/cache/cache-coverage/main.c diff --git a/software/tests/cache-line-rw-smoke/main.c b/software/cache/cache-line-rw-smoke/main.c similarity index 100% rename from software/tests/cache-line-rw-smoke/main.c rename to software/cache/cache-line-rw-smoke/main.c diff --git a/software/tests/cache-mix-pressure/main.c b/software/cache/cache-mix-pressure/main.c similarity index 100% rename from software/tests/cache-mix-pressure/main.c rename to software/cache/cache-mix-pressure/main.c diff --git a/software/tests/cache-mix-smoke/main.c b/software/cache/cache-mix-smoke/main.c similarity index 100% rename from software/tests/cache-mix-smoke/main.c rename to software/cache/cache-mix-smoke/main.c diff --git a/software/tests/cache-rlc-mimic/main.c b/software/cache/cache-rlc-mimic/main.c similarity index 100% rename from software/tests/cache-rlc-mimic/main.c rename to software/cache/cache-rlc-mimic/main.c diff --git a/software/tests/cache-test-scalar/main.c b/software/cache/cache-test-scalar/main.c similarity index 100% rename from software/tests/cache-test-scalar/main.c rename to software/cache/cache-test-scalar/main.c diff --git a/software/tests/cache-test-vector/main.c b/software/cache/cache-test-vector/main.c similarity index 100% rename from software/tests/cache-test-vector/main.c rename to software/cache/cache-test-vector/main.c diff --git a/software/tests/cache-vector-rw/main.c b/software/cache/cache-vector-rw/main.c similarity index 100% rename from software/tests/cache-vector-rw/main.c rename to software/cache/cache-vector-rw/main.c diff --git a/software/cmake/CachePoolTests.cmake b/software/cmake/CachePoolTests.cmake new file mode 100644 index 00000000..fef08168 --- /dev/null +++ b/software/cmake/CachePoolTests.cmake @@ -0,0 +1,57 @@ +# Copyright 2020 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Shared test-registration macros and common libraries for all CachePool +# test groups (cache/, sync/, kernels/fp/, kernels/int/, rlc/, tests/). +# Included once from the top-level software/CMakeLists.txt, before any +# group's add_subdirectory(), so its macros/libraries are visible to them. + +set(CACHEPOOL_TESTS_COMMON_DIR ${CACHEPOOL_DIR}/software/tests) + +# All test groups' binaries land in one place, software/build/CachePoolTests/, +# same as before the cache/sync/kernels/rlc split -- CI (.gitlab-ci.yml) and +# util/auto-benchmark reference that fixed path regardless of which group +# CMakeLists.txt registered the test. +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/CachePoolTests) + +include_directories(${CACHEPOOL_TESTS_COMMON_DIR}/include) +include_directories(${SNRUNTIME_INCLUDE_DIRS}) + +add_compile_options(-O3 -g -ffunction-sections) +add_compile_options(-DELEN=64) + +add_library(benchmark ${CACHEPOOL_TESTS_COMMON_DIR}/benchmark/benchmark.c) +add_library(spin_lock ${CACHEPOOL_TESTS_COMMON_DIR}/benchmark/spin_lock.c) +add_library(mcs_lock ${CACHEPOOL_TESTS_COMMON_DIR}/benchmark/mcs_lock.c) + +enable_testing() +set(SNITCH_TEST_PREFIX "") + +# Macro to generate golden values +macro(add_spatz_test_zeroParam name file) + set(target_name ${name}) + add_snitch_test(${target_name} ${file}) + target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) +endmacro() + +macro(add_spatz_test_oneParam name file param1) + set(target_name ${name}_M${param1}) + add_snitch_test(${target_name} ${file}) + target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}.h") +endmacro() + +macro(add_spatz_test_twoParam name file param1 param2) + set(target_name ${name}_M${param1}_N${param2}) + add_snitch_test(${target_name} ${file}) + target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}.h") +endmacro() + +macro(add_spatz_test_threeParam name file param1 param2 param3) + set(target_name ${name}_M${param1}_N${param2}_K${param3}) + add_snitch_test(${target_name} ${file}) + target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) + target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h") +endmacro() diff --git a/software/kernels/fp/CMakeLists.txt b/software/kernels/fp/CMakeLists.txt new file mode 100644 index 00000000..f1053a29 --- /dev/null +++ b/software/kernels/fp/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_library(fdotp-32b fdotp-32b/kernel/fdotp.c) +add_library(fmatmul-32b fmatmul-32b/kernel/fmatmul.c) + +add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 8192) +add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 32768) +add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 65536) + +add_spatz_test_threeParam(gemv gemv/main.c 128 128 32) +add_spatz_test_threeParam(gemv gemv/main.c 256 128 32) +add_spatz_test_threeParam(gemv gemv/main.c 512 128 32) +add_spatz_test_threeParam(gemv gemv/main.c 1024 128 32) + +add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 128 128 32) +add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 256 128 32) +add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 512 128 32) +add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 1024 128 32) + +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 32 32 32) +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 64 64 64) +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 128 128 128) +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 1024 32 32) +add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 1024 64 64) + +add_spatz_test_twoParam(fft-32b fft-32b/main.c 256 4) +add_spatz_test_twoParam(fft-32b fft-32b/main.c 1024 4) +add_spatz_test_twoParam(fft-32b fft-32b/main.c 1024 16) diff --git a/software/tests/fdotp-32b/data/.gitignore b/software/kernels/fp/fdotp-32b/data/.gitignore similarity index 100% rename from software/tests/fdotp-32b/data/.gitignore rename to software/kernels/fp/fdotp-32b/data/.gitignore diff --git a/software/tests/fdotp-32b/data/layer.h b/software/kernels/fp/fdotp-32b/data/layer.h similarity index 100% rename from software/tests/fdotp-32b/data/layer.h rename to software/kernels/fp/fdotp-32b/data/layer.h diff --git a/software/tests/fdotp-32b/kernel/fdotp.c b/software/kernels/fp/fdotp-32b/kernel/fdotp.c similarity index 100% rename from software/tests/fdotp-32b/kernel/fdotp.c rename to software/kernels/fp/fdotp-32b/kernel/fdotp.c diff --git a/software/tests/fdotp-32b/kernel/fdotp.h b/software/kernels/fp/fdotp-32b/kernel/fdotp.h similarity index 100% rename from software/tests/fdotp-32b/kernel/fdotp.h rename to software/kernels/fp/fdotp-32b/kernel/fdotp.h diff --git a/software/tests/fdotp-32b/main.c b/software/kernels/fp/fdotp-32b/main.c similarity index 100% rename from software/tests/fdotp-32b/main.c rename to software/kernels/fp/fdotp-32b/main.c diff --git a/software/tests/fdotp-32b/script/data_32768.json b/software/kernels/fp/fdotp-32b/script/data_32768.json similarity index 100% rename from software/tests/fdotp-32b/script/data_32768.json rename to software/kernels/fp/fdotp-32b/script/data_32768.json diff --git a/software/tests/fdotp-32b/script/data_65536.json b/software/kernels/fp/fdotp-32b/script/data_65536.json similarity index 100% rename from software/tests/fdotp-32b/script/data_65536.json rename to software/kernels/fp/fdotp-32b/script/data_65536.json diff --git a/software/tests/fdotp-32b/script/data_8192.json b/software/kernels/fp/fdotp-32b/script/data_8192.json similarity index 100% rename from software/tests/fdotp-32b/script/data_8192.json rename to software/kernels/fp/fdotp-32b/script/data_8192.json diff --git a/software/tests/fdotp-32b/script/gen_data.py b/software/kernels/fp/fdotp-32b/script/gen_data.py similarity index 100% rename from software/tests/fdotp-32b/script/gen_data.py rename to software/kernels/fp/fdotp-32b/script/gen_data.py diff --git a/software/tests/fft-32b/data/.gitignore b/software/kernels/fp/fft-32b/data/.gitignore similarity index 100% rename from software/tests/fft-32b/data/.gitignore rename to software/kernels/fp/fft-32b/data/.gitignore diff --git a/software/tests/fft-32b/data/data_1024_4.h b/software/kernels/fp/fft-32b/data/data_1024_4.h similarity index 100% rename from software/tests/fft-32b/data/data_1024_4.h rename to software/kernels/fp/fft-32b/data/data_1024_4.h diff --git a/software/tests/fft-32b/kernel/fft.c b/software/kernels/fp/fft-32b/kernel/fft.c similarity index 100% rename from software/tests/fft-32b/kernel/fft.c rename to software/kernels/fp/fft-32b/kernel/fft.c diff --git a/software/tests/fft-32b/kernel/fft.h b/software/kernels/fp/fft-32b/kernel/fft.h similarity index 100% rename from software/tests/fft-32b/kernel/fft.h rename to software/kernels/fp/fft-32b/kernel/fft.h diff --git a/software/tests/fft-32b/main.c b/software/kernels/fp/fft-32b/main.c similarity index 100% rename from software/tests/fft-32b/main.c rename to software/kernels/fp/fft-32b/main.c diff --git a/software/tests/fft-32b/script/data_1024_16.json b/software/kernels/fp/fft-32b/script/data_1024_16.json similarity index 100% rename from software/tests/fft-32b/script/data_1024_16.json rename to software/kernels/fp/fft-32b/script/data_1024_16.json diff --git a/software/tests/fft-32b/script/data_256_4.json b/software/kernels/fp/fft-32b/script/data_256_4.json similarity index 100% rename from software/tests/fft-32b/script/data_256_4.json rename to software/kernels/fp/fft-32b/script/data_256_4.json diff --git a/software/tests/fft-32b/script/fft_1024_4.json b/software/kernels/fp/fft-32b/script/fft_1024_4.json similarity index 100% rename from software/tests/fft-32b/script/fft_1024_4.json rename to software/kernels/fp/fft-32b/script/fft_1024_4.json diff --git a/software/tests/fft-32b/script/gen_data.py b/software/kernels/fp/fft-32b/script/gen_data.py similarity index 100% rename from software/tests/fft-32b/script/gen_data.py rename to software/kernels/fp/fft-32b/script/gen_data.py diff --git a/software/tests/fmatmul-32b/data/.gitignore b/software/kernels/fp/fmatmul-32b/data/.gitignore similarity index 100% rename from software/tests/fmatmul-32b/data/.gitignore rename to software/kernels/fp/fmatmul-32b/data/.gitignore diff --git a/software/tests/fmatmul-32b/data/layer.h b/software/kernels/fp/fmatmul-32b/data/layer.h similarity index 100% rename from software/tests/fmatmul-32b/data/layer.h rename to software/kernels/fp/fmatmul-32b/data/layer.h diff --git a/software/tests/fmatmul-32b/kernel/fmatmul.c b/software/kernels/fp/fmatmul-32b/kernel/fmatmul.c similarity index 100% rename from software/tests/fmatmul-32b/kernel/fmatmul.c rename to software/kernels/fp/fmatmul-32b/kernel/fmatmul.c diff --git a/software/tests/fmatmul-32b/kernel/fmatmul.h b/software/kernels/fp/fmatmul-32b/kernel/fmatmul.h similarity index 100% rename from software/tests/fmatmul-32b/kernel/fmatmul.h rename to software/kernels/fp/fmatmul-32b/kernel/fmatmul.h diff --git a/software/tests/fmatmul-32b/main.c b/software/kernels/fp/fmatmul-32b/main.c similarity index 100% rename from software/tests/fmatmul-32b/main.c rename to software/kernels/fp/fmatmul-32b/main.c diff --git a/software/tests/fmatmul-32b/script/data_1024_32_32.json b/software/kernels/fp/fmatmul-32b/script/data_1024_32_32.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_1024_32_32.json rename to software/kernels/fp/fmatmul-32b/script/data_1024_32_32.json diff --git a/software/tests/fmatmul-32b/script/data_1024_64_64.json b/software/kernels/fp/fmatmul-32b/script/data_1024_64_64.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_1024_64_64.json rename to software/kernels/fp/fmatmul-32b/script/data_1024_64_64.json diff --git a/software/tests/fmatmul-32b/script/data_128_128_128.json b/software/kernels/fp/fmatmul-32b/script/data_128_128_128.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_128_128_128.json rename to software/kernels/fp/fmatmul-32b/script/data_128_128_128.json diff --git a/software/tests/fmatmul-32b/script/data_32_32_32.json b/software/kernels/fp/fmatmul-32b/script/data_32_32_32.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_32_32_32.json rename to software/kernels/fp/fmatmul-32b/script/data_32_32_32.json diff --git a/software/tests/fmatmul-32b/script/data_64_64_64.json b/software/kernels/fp/fmatmul-32b/script/data_64_64_64.json similarity index 100% rename from software/tests/fmatmul-32b/script/data_64_64_64.json rename to software/kernels/fp/fmatmul-32b/script/data_64_64_64.json diff --git a/software/tests/fmatmul-32b/script/gen_data.py b/software/kernels/fp/fmatmul-32b/script/gen_data.py similarity index 100% rename from software/tests/fmatmul-32b/script/gen_data.py rename to software/kernels/fp/fmatmul-32b/script/gen_data.py diff --git a/software/tests/gemv-opt/data/.gitignore b/software/kernels/fp/gemv-opt/data/.gitignore similarity index 100% rename from software/tests/gemv-opt/data/.gitignore rename to software/kernels/fp/gemv-opt/data/.gitignore diff --git a/software/tests/gemv-opt/data/layer.h b/software/kernels/fp/gemv-opt/data/layer.h similarity index 100% rename from software/tests/gemv-opt/data/layer.h rename to software/kernels/fp/gemv-opt/data/layer.h diff --git a/software/tests/gemv-opt/kernel/gemv.c b/software/kernels/fp/gemv-opt/kernel/gemv.c similarity index 100% rename from software/tests/gemv-opt/kernel/gemv.c rename to software/kernels/fp/gemv-opt/kernel/gemv.c diff --git a/software/tests/gemv-opt/kernel/gemv.h b/software/kernels/fp/gemv-opt/kernel/gemv.h similarity index 100% rename from software/tests/gemv-opt/kernel/gemv.h rename to software/kernels/fp/gemv-opt/kernel/gemv.h diff --git a/software/tests/gemv-opt/main.c b/software/kernels/fp/gemv-opt/main.c similarity index 100% rename from software/tests/gemv-opt/main.c rename to software/kernels/fp/gemv-opt/main.c diff --git a/software/tests/gemv-opt/script/data_1024_128_32.json b/software/kernels/fp/gemv-opt/script/data_1024_128_32.json similarity index 100% rename from software/tests/gemv-opt/script/data_1024_128_32.json rename to software/kernels/fp/gemv-opt/script/data_1024_128_32.json diff --git a/software/tests/gemv-opt/script/data_128_128_32.json b/software/kernels/fp/gemv-opt/script/data_128_128_32.json similarity index 100% rename from software/tests/gemv-opt/script/data_128_128_32.json rename to software/kernels/fp/gemv-opt/script/data_128_128_32.json diff --git a/software/tests/gemv-opt/script/data_256_128_32.json b/software/kernels/fp/gemv-opt/script/data_256_128_32.json similarity index 100% rename from software/tests/gemv-opt/script/data_256_128_32.json rename to software/kernels/fp/gemv-opt/script/data_256_128_32.json diff --git a/software/tests/gemv-opt/script/data_512_128_32.json b/software/kernels/fp/gemv-opt/script/data_512_128_32.json similarity index 100% rename from software/tests/gemv-opt/script/data_512_128_32.json rename to software/kernels/fp/gemv-opt/script/data_512_128_32.json diff --git a/software/tests/gemv-opt/script/gen_data.py b/software/kernels/fp/gemv-opt/script/gen_data.py similarity index 100% rename from software/tests/gemv-opt/script/gen_data.py rename to software/kernels/fp/gemv-opt/script/gen_data.py diff --git a/software/tests/gemv/data/.gitignore b/software/kernels/fp/gemv/data/.gitignore similarity index 100% rename from software/tests/gemv/data/.gitignore rename to software/kernels/fp/gemv/data/.gitignore diff --git a/software/tests/gemv/data/layer.h b/software/kernels/fp/gemv/data/layer.h similarity index 100% rename from software/tests/gemv/data/layer.h rename to software/kernels/fp/gemv/data/layer.h diff --git a/software/tests/gemv/kernel/gemv.c b/software/kernels/fp/gemv/kernel/gemv.c similarity index 100% rename from software/tests/gemv/kernel/gemv.c rename to software/kernels/fp/gemv/kernel/gemv.c diff --git a/software/tests/gemv/kernel/gemv.h b/software/kernels/fp/gemv/kernel/gemv.h similarity index 100% rename from software/tests/gemv/kernel/gemv.h rename to software/kernels/fp/gemv/kernel/gemv.h diff --git a/software/tests/gemv/main.c b/software/kernels/fp/gemv/main.c similarity index 100% rename from software/tests/gemv/main.c rename to software/kernels/fp/gemv/main.c diff --git a/software/tests/gemv/script/data_1024_128_32.json b/software/kernels/fp/gemv/script/data_1024_128_32.json similarity index 100% rename from software/tests/gemv/script/data_1024_128_32.json rename to software/kernels/fp/gemv/script/data_1024_128_32.json diff --git a/software/tests/gemv/script/data_128_128_32.json b/software/kernels/fp/gemv/script/data_128_128_32.json similarity index 100% rename from software/tests/gemv/script/data_128_128_32.json rename to software/kernels/fp/gemv/script/data_128_128_32.json diff --git a/software/tests/gemv/script/data_256_128_32.json b/software/kernels/fp/gemv/script/data_256_128_32.json similarity index 100% rename from software/tests/gemv/script/data_256_128_32.json rename to software/kernels/fp/gemv/script/data_256_128_32.json diff --git a/software/tests/gemv/script/data_512_128_32.json b/software/kernels/fp/gemv/script/data_512_128_32.json similarity index 100% rename from software/tests/gemv/script/data_512_128_32.json rename to software/kernels/fp/gemv/script/data_512_128_32.json diff --git a/software/tests/gemv/script/gen_data.py b/software/kernels/fp/gemv/script/gen_data.py similarity index 100% rename from software/tests/gemv/script/gen_data.py rename to software/kernels/fp/gemv/script/gen_data.py diff --git a/software/kernels/int/CMakeLists.txt b/software/kernels/int/CMakeLists.txt new file mode 100644 index 00000000..2c01e58f --- /dev/null +++ b/software/kernels/int/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_library(idotp-32b idotp-32b/kernel/idotp.c) + +add_spatz_test_oneParam(idotp-32b idotp-32b/main.c 8192) +add_spatz_test_oneParam(idotp-32b idotp-32b/main.c 32768) diff --git a/software/tests/idotp-32b/data/.gitignore b/software/kernels/int/idotp-32b/data/.gitignore similarity index 100% rename from software/tests/idotp-32b/data/.gitignore rename to software/kernels/int/idotp-32b/data/.gitignore diff --git a/software/tests/idotp-32b/data/layer.h b/software/kernels/int/idotp-32b/data/layer.h similarity index 100% rename from software/tests/idotp-32b/data/layer.h rename to software/kernels/int/idotp-32b/data/layer.h diff --git a/software/tests/idotp-32b/kernel/idotp.c b/software/kernels/int/idotp-32b/kernel/idotp.c similarity index 100% rename from software/tests/idotp-32b/kernel/idotp.c rename to software/kernels/int/idotp-32b/kernel/idotp.c diff --git a/software/tests/idotp-32b/kernel/idotp.h b/software/kernels/int/idotp-32b/kernel/idotp.h similarity index 100% rename from software/tests/idotp-32b/kernel/idotp.h rename to software/kernels/int/idotp-32b/kernel/idotp.h diff --git a/software/tests/idotp-32b/main.c b/software/kernels/int/idotp-32b/main.c similarity index 100% rename from software/tests/idotp-32b/main.c rename to software/kernels/int/idotp-32b/main.c diff --git a/software/tests/idotp-32b/script/data_32768.json b/software/kernels/int/idotp-32b/script/data_32768.json similarity index 100% rename from software/tests/idotp-32b/script/data_32768.json rename to software/kernels/int/idotp-32b/script/data_32768.json diff --git a/software/tests/idotp-32b/script/data_8192.json b/software/kernels/int/idotp-32b/script/data_8192.json similarity index 100% rename from software/tests/idotp-32b/script/data_8192.json rename to software/kernels/int/idotp-32b/script/data_8192.json diff --git a/software/tests/idotp-32b/script/gen_data.py b/software/kernels/int/idotp-32b/script/gen_data.py similarity index 100% rename from software/tests/idotp-32b/script/gen_data.py rename to software/kernels/int/idotp-32b/script/gen_data.py diff --git a/software/rlc/CMakeLists.txt b/software/rlc/CMakeLists.txt new file mode 100644 index 00000000..a9495e07 --- /dev/null +++ b/software/rlc/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_library(printf_lock multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c) +add_library(llist multi_producer_single_consumer_double_linked_list/kernel/llist.c) +add_library(mm multi_producer_single_consumer_double_linked_list/kernel/mm.c) +add_library(rlc multi_producer_single_consumer_double_linked_list/kernel/rlc.c) +add_library(data_move_vec multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c) + +# add_snitch_test(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c) +# add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 1000) +# add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 2044 100) +add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 300) +add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 100) +add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 10) diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_10.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_10.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_10.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_10.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_100.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_100.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_100.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_100.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_1000.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_1000.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_1000.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_1000.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_300.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_300.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_1350_300.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_1350_300.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_2044_100.h b/software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_2044_100.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/data_1_2044_100.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/data_1_2044_100.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart b/software/rlc/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart.png b/software/rlc/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart.png similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart.png rename to software/rlc/multi_producer_single_consumer_double_linked_list/data/rlc_flowchart.png diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/llist.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/llist.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/llist.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/llist.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/llist.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/llist.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/llist.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/llist.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/mm.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/mm.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/mm.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/mm.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/mm.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/mm.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/mm.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/mm.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c diff --git a/software/tests/mcs-lock/kernel/printf_lock.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.h similarity index 100% rename from software/tests/mcs-lock/kernel/printf_lock.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/rlc.c b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/rlc.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/rlc.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/rlc.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/rlc.h b/software/rlc/multi_producer_single_consumer_double_linked_list/kernel/rlc.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/rlc.h rename to software/rlc/multi_producer_single_consumer_double_linked_list/kernel/rlc.h diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/main.c b/software/rlc/multi_producer_single_consumer_double_linked_list/main.c similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/main.c rename to software/rlc/multi_producer_single_consumer_double_linked_list/main.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/script/flowchart.py b/software/rlc/multi_producer_single_consumer_double_linked_list/script/flowchart.py similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/script/flowchart.py rename to software/rlc/multi_producer_single_consumer_double_linked_list/script/flowchart.py diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/script/generate_pdcp_pkg.py b/software/rlc/multi_producer_single_consumer_double_linked_list/script/generate_pdcp_pkg.py similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/script/generate_pdcp_pkg.py rename to software/rlc/multi_producer_single_consumer_double_linked_list/script/generate_pdcp_pkg.py diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/script/pdcp_pkg.json b/software/rlc/multi_producer_single_consumer_double_linked_list/script/pdcp_pkg.json similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/script/pdcp_pkg.json rename to software/rlc/multi_producer_single_consumer_double_linked_list/script/pdcp_pkg.json diff --git a/software/sync/CMakeLists.txt b/software/sync/CMakeLists.txt new file mode 100644 index 00000000..018f9d13 --- /dev/null +++ b/software/sync/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2025 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +add_spatz_test_zeroParam(spin-lock spin-lock/main.c) +add_spatz_test_zeroParam(mcs-lock mcs-lock/main.c) +add_spatz_test_zeroParam(partial_barrier partial_barrier/main.c)# SOCMIPO +add_spatz_test_zeroParam(spatz-lock-handoff spatz-lock-handoff/main.c) +add_spatz_test_zeroParam(false-sharing false-sharing/main.c) diff --git a/software/tests/false-sharing/main.c b/software/sync/false-sharing/main.c similarity index 100% rename from software/tests/false-sharing/main.c rename to software/sync/false-sharing/main.c diff --git a/software/tests/mcs-lock/kernel/printf_lock.c b/software/sync/mcs-lock/kernel/printf_lock.c similarity index 100% rename from software/tests/mcs-lock/kernel/printf_lock.c rename to software/sync/mcs-lock/kernel/printf_lock.c diff --git a/software/tests/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.h b/software/sync/mcs-lock/kernel/printf_lock.h similarity index 100% rename from software/tests/multi_producer_single_consumer_double_linked_list/kernel/printf_lock.h rename to software/sync/mcs-lock/kernel/printf_lock.h diff --git a/software/tests/mcs-lock/main.c b/software/sync/mcs-lock/main.c similarity index 100% rename from software/tests/mcs-lock/main.c rename to software/sync/mcs-lock/main.c diff --git a/software/tests/partial_barrier/main.c b/software/sync/partial_barrier/main.c similarity index 100% rename from software/tests/partial_barrier/main.c rename to software/sync/partial_barrier/main.c diff --git a/software/tests/spatz-lock-handoff/main.c b/software/sync/spatz-lock-handoff/main.c similarity index 100% rename from software/tests/spatz-lock-handoff/main.c rename to software/sync/spatz-lock-handoff/main.c diff --git a/software/tests/spin-lock/main.c b/software/sync/spin-lock/main.c similarity index 100% rename from software/tests/spin-lock/main.c rename to software/sync/spin-lock/main.c diff --git a/software/tests/CMakeLists.txt b/software/tests/CMakeLists.txt index e12c0b6d..0c133516 100644 --- a/software/tests/CMakeLists.txt +++ b/software/tests/CMakeLists.txt @@ -2,145 +2,7 @@ # Licensed under the Apache License, Version 2.0, see LICENSE for details. # SPDX-License-Identifier: Apache-2.0 -cmake_minimum_required(VERSION 3.13) - -# Allow spatzBenchmarks to be built as a standalone library. -if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) - list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) - set(CMAKE_TOOLCHAIN_FILE toolchain-gcc CACHE STRING "Toolchain to use") - - project(Benchmarks LANGUAGES C ASM) - include(SnitchUtilities) - - # Build the runtime. - add_subdirectory(${SPATZ_DIR}/sw/snRuntime snRuntime) -endif() - -include_directories(include) -include_directories(${SNRUNTIME_INCLUDE_DIRS}) - -add_compile_options(-O3 -g -ffunction-sections) - - - -# Macro to generate golden values -macro(add_spatz_test_zeroParam name file) - set(target_name ${name}) - add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - # target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name}) -endmacro() - -macro(add_spatz_test_oneParam name file param1) - set(target_name ${name}_M${param1}) - add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}.h") -endmacro() - -macro(add_spatz_test_twoParam name file param1 param2) - set(target_name ${name}_M${param1}_N${param2}) - add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}.h") -endmacro() - -macro(add_spatz_test_threeParam name file param1 param2 param3) - set(target_name ${name}_M${param1}_N${param2}_K${param3}) - add_snitch_test(${target_name} ${file}) - target_link_libraries(test-${SNITCH_TEST_PREFIX}${target_name} benchmark spin_lock mcs_lock ${SNITCH_RUNTIME}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${target_name} PUBLIC DATAHEADER="data/data_${param1}_${param2}_${param3}.h") -endmacro() - - - - -# Benchmark library -add_library(benchmark benchmark/benchmark.c) -add_library(spin_lock benchmark/spin_lock.c) -add_library(mcs_lock benchmark/mcs_lock.c) - -add_compile_options(-O3 -g -ffunction-sections) -add_compile_options(-DELEN=64) - -include_directories(include) -include_directories(${SNRUNTIME_INCLUDE_DIRS}) - -# Kernels -## Vector Kernels -add_library(fdotp-32b fdotp-32b/kernel/fdotp.c) -add_library(idotp-32b idotp-32b/kernel/idotp.c) -add_library(fmatmul-32b fmatmul-32b/kernel/fmatmul.c) - - -## RLC -add_library(printf_lock multi_producer_single_consumer_double_linked_list/kernel/printf_lock.c) -add_library(llist multi_producer_single_consumer_double_linked_list/kernel/llist.c) -add_library(mm multi_producer_single_consumer_double_linked_list/kernel/mm.c) -add_library(rlc multi_producer_single_consumer_double_linked_list/kernel/rlc.c) -add_library(data_move_vec multi_producer_single_consumer_double_linked_list/kernel/data_move_vec.c) - -# Tests -enable_testing() -set(SNITCH_TEST_PREFIX cachepool-) - -## RLC -add_spatz_test_zeroParam(spin-lock spin-lock/main.c) -add_spatz_test_zeroParam(partial_barrier partial_barrier/main.c)# SOCMIPO -add_spatz_test_zeroParam(partial_barrier_benchmark partial_barrier_benchmark/main.c)# SOCMIPO -add_spatz_test_zeroParam(mcs-lock mcs-lock/main.c) -add_spatz_test_zeroParam(byte-enable byte-enable/main.c) -add_spatz_test_zeroParam(cache-line-rw-smoke cache-line-rw-smoke/main.c) add_spatz_test_zeroParam(minimal-tile0-repro minimal-tile0-repro/main.c) -add_spatz_test_zeroParam(cache-test-scalar cache-test-scalar/main.c) -add_spatz_test_zeroParam(cache-test-vector cache-test-vector/main.c) -add_spatz_test_zeroParam(cache-mix-smoke cache-mix-smoke/main.c) -add_spatz_test_zeroParam(cache-mix-pressure cache-mix-pressure/main.c) -add_spatz_test_zeroParam(cache-rlc-mimic cache-rlc-mimic/main.c) -add_spatz_test_zeroParam(cache-vector-rw cache-vector-rw/main.c) -add_spatz_test_zeroParam(cache-coverage cache-coverage/main.c) -add_spatz_test_zeroParam(cache-coverage-min cache-coverage-min/main.c) - -# add_snitch_test(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c) -# add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 1000) -# add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 2044 100) -add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 300) -add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 100) -add_spatz_test_threeParam(multi_producer_single_consumer_double_linked_list multi_producer_single_consumer_double_linked_list/main.c 1 1350 10) - -## Vector -### Floating-Point -add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 8192) -add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 32768) -add_spatz_test_oneParam(fdotp-32b fdotp-32b/main.c 65536) - -add_spatz_test_threeParam(gemv gemv/main.c 128 128 32) -add_spatz_test_threeParam(gemv gemv/main.c 256 128 32) -add_spatz_test_threeParam(gemv gemv/main.c 512 128 32) -add_spatz_test_threeParam(gemv gemv/main.c 1024 128 32) - -add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 128 128 32) -add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 256 128 32) -add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 512 128 32) -add_spatz_test_threeParam(gemv-opt gemv-opt/main.c 1024 128 32) - -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 32 32 32) -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 64 64 64) -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 128 128 128) -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 1024 32 32) -add_spatz_test_threeParam(fmatmul-32b fmatmul-32b/main.c 1024 64 64) - -add_spatz_test_twoParam(fft-32b fft-32b/main.c 256 4) -add_spatz_test_twoParam(fft-32b fft-32b/main.c 1024 4) -add_spatz_test_twoParam(fft-32b fft-32b/main.c 1024 16) - -### Integer-Point -add_spatz_test_oneParam(idotp-32b idotp-32b/main.c 8192) -add_spatz_test_oneParam(idotp-32b idotp-32b/main.c 32768) - - add_spatz_test_zeroParam(load-store load-store/main.c) - -add_spatz_test_zeroParam(spatz-lock-handoff spatz-lock-handoff/main.c) - +add_spatz_test_zeroParam(partial_barrier_benchmark partial_barrier_benchmark/main.c)# SOCMIPO add_spatz_test_zeroParam(bandwidth bandwidth/main.c) diff --git a/util/auto-benchmark/configs.sh b/util/auto-benchmark/configs.sh index b7e424f6..f5fe28fe 100755 --- a/util/auto-benchmark/configs.sh +++ b/util/auto-benchmark/configs.sh @@ -5,5 +5,5 @@ # Configs and kernel suffixes (without prefix) CONFIGS="cachepool_fpu_4g" KERNELS="fdotp-32b_M65536 gemv_M1024_N128_K32 fmatmul-32b_M1024_N32_K32 fft-32b_M1024_N16 " -PREFIX="test-cachepool-" # common prefix for all kernels +PREFIX="test-" # common prefix for all kernels ROOT_PATH=../.. # adjust if needed (path to repo root) From d755220735f65aef97a3fd7d5029de7d0e6a5988 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Tue, 25 Aug 2026 14:40:09 +0200 Subject: [PATCH 13/17] [SRC] Update FSM implementation to have a cleaner result informing and failing algorithm --- README.md | 10 +- .../cachepool_peripheral_reg.hjson | 25 +- .../cachepool_peripheral_reg_pkg.sv | 108 +++--- .../cachepool_peripheral_reg_top.sv | 156 ++++---- hardware/src/acc_mux.sv | 29 +- hardware/src/cachepool_cc_dual.sv | 8 +- hardware/src/cachepool_spatz_lock.sv | 332 ++++++++++-------- hardware/src/cachepool_tile.sv | 3 +- software/snRuntime/README.md | 43 +++ .../snRuntime/include/cachepool_peripheral.h | 43 +-- software/snRuntime/include/spatz_lock.h | 34 +- software/snRuntime/src/spatz_lock.c | 37 +- 12 files changed, 515 insertions(+), 313 deletions(-) diff --git a/README.md b/README.md index 4366ea8d..c0bc4a7e 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ Configuration names encode the number of groups and whether the FPU is enabled: `cachepool_fpu_16g_tiny` shrinks tiles/group and cores/tile for a faster-to-build, faster-to-simulate smoke test of the full 16-group mesh topology. -`cachepool_dual_4g` sets `num_scalar_per_core=2`: each Core Complex holds 2 Snitch scalar harts sharing 1 Spatz unit via a hardware ownership lock, so "Cores" (Core Complex slots) and hart count diverge — 64 CCs, 128 harts total. Software runtime support for the shared-Spatz lock is still in progress (see `note.md`). +`cachepool_dual_4g` sets `num_scalar_per_core=2`: each Core Complex holds 2 Snitch scalar harts sharing 1 Spatz unit via a hardware ownership lock (`cachepool_spatz_lock.sv`), so "Cores" (Core Complex slots) and hart count diverge — 64 CCs, 128 harts total. The lock never blocks a core's pipeline: every acquire/release attempt (`software/snRuntime/include/spatz_lock.h`) completes immediately with an outcome (granted / denied / granted-but-still-draining), so a hart contending for a lock it doesn't get can always retry or do something else instead of hanging. See `note.md` for the full state-machine design. The Spatz cluster consumes **`config/cachepool.hjson`**, which is **generated** from: - `config/cachepool.hjson.tmpl` (skeleton with comments) @@ -375,6 +375,14 @@ For a spatial, time-scrubbable view of NoC traffic (as opposed to `cachepool_mon The default system uses a 32-bit Snitch core with a Spatz RVV accelerator. Double-precision is disabled by default for scalability; enable the FPU flavor (`cachepool_fpu.mk`) for single/half precision support. +### Dual-scalar flavor (`cachepool_cc_dual.sv`) + +Set `num_scalar_per_core=2` (e.g. `cachepool_dual_4g`, see [Configurations](#configurations)) to build a Core Complex with **2 Snitch scalar harts sharing 1 Spatz unit**. Motivating workload: tasks where not every hart needs the vector unit at the same time, and control hands Spatz off between the pair at coarse task boundaries. The two harts get sequential hart/core IDs (e.g. cid 0/1 within a pair); `snrt_cluster_is_primary()` (`snrt.h`) tells a hart whether it's the pair's default owner (even `cid`) or its partner (odd `cid`). + +Ownership is arbitrated by `cachepool_spatz_lock.sv`, a small hardware FSM (`Free`/`Locked`/`AcqWait`/`RelWait`) intercepting two dedicated peripheral addresses. It never blocks a core's pipeline: every acquire/release attempt is a plain **load** that always completes immediately, returning an outcome (`FAIL`/`SUCCESS`/`SUCCESS-WAIT`) instead of stalling the hart — so a hart that doesn't get the lock can retry, back off, or do other work instead of hanging. See `software/snRuntime/README.md` for the software API and `note.md` for the full state-machine design. + +Two per-role partial-barrier helpers, `snrt_cluster_host0_barrier()`/`snrt_cluster_host1_barrier()` (`snrt.h`), let a kernel synchronize only the pair's default owners (or only their partners) without hand-writing a participant mask; both degrade to an ordinary full barrier on a single-scalar-per-CC build, so kernels using them don't need a config-specific `#if`. + ## Stack Each core complex has a local **stack SPM**. Its depth is configured via parameters in `config/config.mk` (forwarded to RTL). If the stack exceeds the local SPM, it spills into the cache space (indexed with core ID bits). diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson index bc3949bd..f9deba9c 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson @@ -32,16 +32,31 @@ }] }, { - name: "SPATZ_LOCK", - desc: '''Spatz ownership lock for dual-Snitch core complexes. Write 1 to acquire - (blocks until granted), write 0 to release.''' + name: "SPATZ_LOCK_ACQUIRE", + desc: '''Spatz ownership acquire for dual-Snitch core complexes. Load + attempts to acquire; the loaded value encodes the outcome (fail/ + success/success-wait) plus current owner/locked status.''' swaccess: "ro", hwaccess: "hrw", hwext: "true", fields: [{ bits: "31:0", - name: "SPATZ_LOCK", - desc: "Spatz ownership lock register." + name: "SPATZ_LOCK_ACQUIRE", + desc: "Spatz ownership acquire register." + }] + }, + { + name: "SPATZ_LOCK_RELEASE", + desc: '''Spatz ownership release for dual-Snitch core complexes. Load + attempts to release; the loaded value encodes the outcome (fail/ + success/success-wait) plus current owner/locked status.''' + swaccess: "ro", + hwaccess: "hrw", + hwext: "true", + fields: [{ + bits: "31:0", + name: "SPATZ_LOCK_RELEASE", + desc: "Spatz ownership release register." }] }, { diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv index 2fce913c..6b038474 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv @@ -22,7 +22,11 @@ package cachepool_peripheral_reg_pkg; typedef struct packed { logic [31:0] q; - } cachepool_peripheral_reg2hw_spatz_lock_reg_t; + } cachepool_peripheral_reg2hw_spatz_lock_acquire_reg_t; + + typedef struct packed { + logic [31:0] q; + } cachepool_peripheral_reg2hw_spatz_lock_release_reg_t; typedef struct packed { logic q; @@ -90,7 +94,11 @@ package cachepool_peripheral_reg_pkg; typedef struct packed { logic [31:0] d; - } cachepool_peripheral_hw2reg_spatz_lock_reg_t; + } cachepool_peripheral_hw2reg_spatz_lock_acquire_reg_t; + + typedef struct packed { + logic [31:0] d; + } cachepool_peripheral_hw2reg_spatz_lock_release_reg_t; typedef struct packed { logic d; @@ -113,8 +121,9 @@ package cachepool_peripheral_reg_pkg; // Register -> HW type typedef struct packed { - cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [317:286] - cachepool_peripheral_reg2hw_spatz_lock_reg_t spatz_lock; // [285:254] + cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [349:318] + cachepool_peripheral_reg2hw_spatz_lock_acquire_reg_t spatz_lock_acquire; // [317:286] + cachepool_peripheral_reg2hw_spatz_lock_release_reg_t spatz_lock_release; // [285:254] cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t icache_prefetch_enable; // [253:253] cachepool_peripheral_reg2hw_spatz_status_reg_t spatz_status; // [252:252] cachepool_peripheral_reg2hw_spatz_cycle_reg_t spatz_cycle; // [251:220] @@ -134,8 +143,9 @@ package cachepool_peripheral_reg_pkg; // HW -> register type typedef struct packed { - cachepool_peripheral_hw2reg_hw_barrier_reg_t hw_barrier; // [70:39] - cachepool_peripheral_hw2reg_spatz_lock_reg_t spatz_lock; // [38:7] + cachepool_peripheral_hw2reg_hw_barrier_reg_t hw_barrier; // [102:71] + cachepool_peripheral_hw2reg_spatz_lock_acquire_reg_t spatz_lock_acquire; // [70:39] + cachepool_peripheral_hw2reg_spatz_lock_release_reg_t spatz_lock_release; // [38:7] cachepool_peripheral_hw2reg_l1d_spm_commit_reg_t l1d_spm_commit; // [6:5] cachepool_peripheral_hw2reg_l1d_insn_commit_reg_t l1d_insn_commit; // [4:3] cachepool_peripheral_hw2reg_l1d_flush_status_reg_t l1d_flush_status; // [2:2] @@ -144,36 +154,39 @@ package cachepool_peripheral_reg_pkg; // Register offsets parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_OFFSET = 7'h 0; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_OFFSET = 7'h 4; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET = 7'h 8; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET = 7'h c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET = 7'h 10; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET = 7'h 14; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET = 7'h 18; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET = 7'h 1c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET = 7'h 20; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET = 7'h 24; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET = 7'h 28; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET = 7'h 2c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET = 7'h 30; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET = 7'h 34; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET = 7'h 38; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET = 7'h 3c; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET = 7'h 40; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET = 7'h 44; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET = 7'h 48; - parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET = 7'h 4c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_OFFSET = 7'h 4; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_OFFSET = 7'h 8; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET = 7'h c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET = 7'h 10; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET = 7'h 14; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET = 7'h 18; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET = 7'h 1c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET = 7'h 20; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET = 7'h 24; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET = 7'h 28; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET = 7'h 2c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET = 7'h 30; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET = 7'h 34; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET = 7'h 38; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET = 7'h 3c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET = 7'h 40; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET = 7'h 44; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET = 7'h 48; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET = 7'h 4c; + parameter logic [BlockAw-1:0] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET = 7'h 50; // Reset values for hwext registers and their fields parameter logic [31:0] CACHEPOOL_PERIPHERAL_HW_BARRIER_RESVAL = 32'h 0; - parameter logic [31:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RESVAL = 32'h 0; + parameter logic [31:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_RESVAL = 32'h 0; + parameter logic [31:0] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_RESVAL = 32'h 0; parameter logic [0:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_RESVAL = 1'h 0; parameter logic [0:0] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_STATUS_RESVAL = 1'h 0; // Register index typedef enum int { CACHEPOOL_PERIPHERAL_HW_BARRIER, - CACHEPOOL_PERIPHERAL_SPATZ_LOCK, + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE, + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE, CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE, CACHEPOOL_PERIPHERAL_SPATZ_STATUS, CACHEPOOL_PERIPHERAL_SPATZ_CYCLE, @@ -195,27 +208,28 @@ package cachepool_peripheral_reg_pkg; } cachepool_peripheral_id_e; // Register width information to check illegal writes - parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [20] = '{ + parameter logic [3:0] CACHEPOOL_PERIPHERAL_PERMIT [21] = '{ 4'b 1111, // index[ 0] CACHEPOOL_PERIPHERAL_HW_BARRIER - 4'b 1111, // index[ 1] CACHEPOOL_PERIPHERAL_SPATZ_LOCK - 4'b 0001, // index[ 2] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE - 4'b 0001, // index[ 3] CACHEPOOL_PERIPHERAL_SPATZ_STATUS - 4'b 1111, // index[ 4] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE - 4'b 1111, // index[ 5] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL - 4'b 0001, // index[ 6] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT - 4'b 0011, // index[ 7] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM - 4'b 0001, // index[ 8] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN - 4'b 1111, // index[ 9] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0 - 4'b 1111, // index[10] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1 - 4'b 1111, // index[11] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0 - 4'b 1111, // index[12] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1 - 4'b 0001, // index[13] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT - 4'b 0001, // index[14] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT - 4'b 0001, // index[15] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS - 4'b 0001, // index[16] CACHEPOOL_PERIPHERAL_L1D_PRIVATE - 4'b 1111, // index[17] CACHEPOOL_PERIPHERAL_L1D_ADDR - 4'b 0001, // index[18] CACHEPOOL_PERIPHERAL_XBAR_OFFSET - 4'b 0001 // index[19] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT + 4'b 1111, // index[ 1] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE + 4'b 1111, // index[ 2] CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE + 4'b 0001, // index[ 3] CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE + 4'b 0001, // index[ 4] CACHEPOOL_PERIPHERAL_SPATZ_STATUS + 4'b 1111, // index[ 5] CACHEPOOL_PERIPHERAL_SPATZ_CYCLE + 4'b 1111, // index[ 6] CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL + 4'b 0001, // index[ 7] CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT + 4'b 0011, // index[ 8] CACHEPOOL_PERIPHERAL_CFG_L1D_SPM + 4'b 0001, // index[ 9] CACHEPOOL_PERIPHERAL_CFG_L1D_INSN + 4'b 1111, // index[10] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0 + 4'b 1111, // index[11] CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1 + 4'b 1111, // index[12] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0 + 4'b 1111, // index[13] CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1 + 4'b 0001, // index[14] CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT + 4'b 0001, // index[15] CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT + 4'b 0001, // index[16] CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS + 4'b 0001, // index[17] CACHEPOOL_PERIPHERAL_L1D_PRIVATE + 4'b 1111, // index[18] CACHEPOOL_PERIPHERAL_L1D_ADDR + 4'b 0001, // index[19] CACHEPOOL_PERIPHERAL_XBAR_OFFSET + 4'b 0001 // index[20] CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT }; endpackage diff --git a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv index efa820df..6932befe 100644 --- a/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv +++ b/hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv @@ -70,8 +70,10 @@ module cachepool_peripheral_reg_top #( // or _{wd|we|qs} if field == 1 or 0 logic [31:0] hw_barrier_qs; logic hw_barrier_re; - logic [31:0] spatz_lock_qs; - logic spatz_lock_re; + logic [31:0] spatz_lock_acquire_qs; + logic spatz_lock_acquire_re; + logic [31:0] spatz_lock_release_qs; + logic spatz_lock_release_re; logic icache_prefetch_enable_wd; logic icache_prefetch_enable_we; logic spatz_status_wd; @@ -141,19 +143,35 @@ module cachepool_peripheral_reg_top #( ); - // R[spatz_lock]: V(True) + // R[spatz_lock_acquire]: V(True) prim_subreg_ext #( .DW (32) - ) u_spatz_lock ( - .re (spatz_lock_re), + ) u_spatz_lock_acquire ( + .re (spatz_lock_acquire_re), .we (1'b0), .wd ('0), - .d (hw2reg.spatz_lock.d), + .d (hw2reg.spatz_lock_acquire.d), .qre (), .qe (), - .q (reg2hw.spatz_lock.q ), - .qs (spatz_lock_qs) + .q (reg2hw.spatz_lock_acquire.q ), + .qs (spatz_lock_acquire_qs) + ); + + + // R[spatz_lock_release]: V(True) + + prim_subreg_ext #( + .DW (32) + ) u_spatz_lock_release ( + .re (spatz_lock_release_re), + .we (1'b0), + .wd ('0), + .d (hw2reg.spatz_lock_release.d), + .qre (), + .qe (), + .q (reg2hw.spatz_lock_release.q ), + .qs (spatz_lock_release_qs) ); @@ -636,29 +654,30 @@ module cachepool_peripheral_reg_top #( - logic [19:0] addr_hit; + logic [20:0] addr_hit; always_comb begin addr_hit = '0; addr_hit[ 0] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_OFFSET); - addr_hit[ 1] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_LOCK_OFFSET); - addr_hit[ 2] = (reg_addr == CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET); - addr_hit[ 3] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET); - addr_hit[ 4] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET); - addr_hit[ 5] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET); - addr_hit[ 6] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET); - addr_hit[ 7] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET); - addr_hit[ 8] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET); - addr_hit[ 9] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET); - addr_hit[10] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET); - addr_hit[11] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET); - addr_hit[12] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET); - addr_hit[13] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET); - addr_hit[14] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET); - addr_hit[15] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET); - addr_hit[16] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET); - addr_hit[17] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET); - addr_hit[18] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET); - addr_hit[19] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET); + addr_hit[ 1] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_OFFSET); + addr_hit[ 2] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_OFFSET); + addr_hit[ 3] = (reg_addr == CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_OFFSET); + addr_hit[ 4] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_STATUS_OFFSET); + addr_hit[ 5] = (reg_addr == CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_OFFSET); + addr_hit[ 6] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_OFFSET); + addr_hit[ 7] = (reg_addr == CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_OFFSET); + addr_hit[ 8] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_OFFSET); + addr_hit[ 9] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_OFFSET); + addr_hit[10] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_OFFSET); + addr_hit[11] = (reg_addr == CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_OFFSET); + addr_hit[12] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_OFFSET); + addr_hit[13] = (reg_addr == CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_OFFSET); + addr_hit[14] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_OFFSET); + addr_hit[15] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_OFFSET); + addr_hit[16] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_OFFSET); + addr_hit[17] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_PRIVATE_OFFSET); + addr_hit[18] = (reg_addr == CACHEPOOL_PERIPHERAL_L1D_ADDR_OFFSET); + addr_hit[19] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET); + addr_hit[20] = (reg_addr == CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_OFFSET); end assign addrmiss = (reg_re || reg_we) ? ~|addr_hit : 1'b0 ; @@ -685,64 +704,67 @@ module cachepool_peripheral_reg_top #( (addr_hit[16] & (|(CACHEPOOL_PERIPHERAL_PERMIT[16] & ~reg_be))) | (addr_hit[17] & (|(CACHEPOOL_PERIPHERAL_PERMIT[17] & ~reg_be))) | (addr_hit[18] & (|(CACHEPOOL_PERIPHERAL_PERMIT[18] & ~reg_be))) | - (addr_hit[19] & (|(CACHEPOOL_PERIPHERAL_PERMIT[19] & ~reg_be))))); + (addr_hit[19] & (|(CACHEPOOL_PERIPHERAL_PERMIT[19] & ~reg_be))) | + (addr_hit[20] & (|(CACHEPOOL_PERIPHERAL_PERMIT[20] & ~reg_be))))); end assign hw_barrier_re = addr_hit[0] & reg_re & !reg_error; - assign spatz_lock_re = addr_hit[1] & reg_re & !reg_error; + assign spatz_lock_acquire_re = addr_hit[1] & reg_re & !reg_error; + + assign spatz_lock_release_re = addr_hit[2] & reg_re & !reg_error; - assign icache_prefetch_enable_we = addr_hit[2] & reg_we & !reg_error; + assign icache_prefetch_enable_we = addr_hit[3] & reg_we & !reg_error; assign icache_prefetch_enable_wd = reg_wdata[0]; - assign spatz_status_we = addr_hit[3] & reg_we & !reg_error; + assign spatz_status_we = addr_hit[4] & reg_we & !reg_error; assign spatz_status_wd = reg_wdata[0]; - assign spatz_cycle_we = addr_hit[4] & reg_we & !reg_error; + assign spatz_cycle_we = addr_hit[5] & reg_we & !reg_error; assign spatz_cycle_wd = reg_wdata[31:0]; - assign cluster_boot_control_we = addr_hit[5] & reg_we & !reg_error; + assign cluster_boot_control_we = addr_hit[6] & reg_we & !reg_error; assign cluster_boot_control_wd = reg_wdata[31:0]; - assign cluster_eoc_exit_we = addr_hit[6] & reg_we & !reg_error; + assign cluster_eoc_exit_we = addr_hit[7] & reg_we & !reg_error; assign cluster_eoc_exit_wd = reg_wdata[3:0]; - assign cfg_l1d_spm_we = addr_hit[7] & reg_we & !reg_error; + assign cfg_l1d_spm_we = addr_hit[8] & reg_we & !reg_error; assign cfg_l1d_spm_wd = reg_wdata[9:0]; - assign cfg_l1d_insn_we = addr_hit[8] & reg_we & !reg_error; + assign cfg_l1d_insn_we = addr_hit[9] & reg_we & !reg_error; assign cfg_l1d_insn_wd = reg_wdata[1:0]; - assign cfg_l1d_tile_sel_0_we = addr_hit[9] & reg_we & !reg_error; + assign cfg_l1d_tile_sel_0_we = addr_hit[10] & reg_we & !reg_error; assign cfg_l1d_tile_sel_0_wd = reg_wdata[31:0]; - assign cfg_l1d_tile_sel_1_we = addr_hit[10] & reg_we & !reg_error; + assign cfg_l1d_tile_sel_1_we = addr_hit[11] & reg_we & !reg_error; assign cfg_l1d_tile_sel_1_wd = reg_wdata[31:0]; - assign hw_barrier_participation_mask_0_we = addr_hit[11] & reg_we & !reg_error; + assign hw_barrier_participation_mask_0_we = addr_hit[12] & reg_we & !reg_error; assign hw_barrier_participation_mask_0_wd = reg_wdata[31:0]; - assign hw_barrier_participation_mask_1_we = addr_hit[12] & reg_we & !reg_error; + assign hw_barrier_participation_mask_1_we = addr_hit[13] & reg_we & !reg_error; assign hw_barrier_participation_mask_1_wd = reg_wdata[31:0]; - assign l1d_spm_commit_we = addr_hit[13] & reg_we & !reg_error; + assign l1d_spm_commit_we = addr_hit[14] & reg_we & !reg_error; assign l1d_spm_commit_wd = reg_wdata[0]; - assign l1d_insn_commit_we = addr_hit[14] & reg_we & !reg_error; + assign l1d_insn_commit_we = addr_hit[15] & reg_we & !reg_error; assign l1d_insn_commit_wd = reg_wdata[0]; - assign l1d_flush_status_re = addr_hit[15] & reg_re & !reg_error; + assign l1d_flush_status_re = addr_hit[16] & reg_re & !reg_error; - assign l1d_private_we = addr_hit[16] & reg_we & !reg_error; + assign l1d_private_we = addr_hit[17] & reg_we & !reg_error; assign l1d_private_wd = reg_wdata[3:0]; - assign l1d_addr_we = addr_hit[17] & reg_we & !reg_error; + assign l1d_addr_we = addr_hit[18] & reg_we & !reg_error; assign l1d_addr_wd = reg_wdata[31:0]; - assign xbar_offset_we = addr_hit[18] & reg_we & !reg_error; + assign xbar_offset_we = addr_hit[19] & reg_we & !reg_error; assign xbar_offset_wd = reg_wdata[4:0]; - assign xbar_offset_commit_we = addr_hit[19] & reg_we & !reg_error; + assign xbar_offset_commit_we = addr_hit[20] & reg_we & !reg_error; assign xbar_offset_commit_wd = reg_wdata[0]; // Read data return @@ -754,11 +776,11 @@ module cachepool_peripheral_reg_top #( end addr_hit[1]: begin - reg_rdata_next[31:0] = spatz_lock_qs; + reg_rdata_next[31:0] = spatz_lock_acquire_qs; end addr_hit[2]: begin - reg_rdata_next[0] = '0; + reg_rdata_next[31:0] = spatz_lock_release_qs; end addr_hit[3]: begin @@ -766,66 +788,70 @@ module cachepool_peripheral_reg_top #( end addr_hit[4]: begin - reg_rdata_next[31:0] = spatz_cycle_qs; + reg_rdata_next[0] = '0; end addr_hit[5]: begin - reg_rdata_next[31:0] = cluster_boot_control_qs; + reg_rdata_next[31:0] = spatz_cycle_qs; end addr_hit[6]: begin - reg_rdata_next[3:0] = cluster_eoc_exit_qs; + reg_rdata_next[31:0] = cluster_boot_control_qs; end addr_hit[7]: begin - reg_rdata_next[9:0] = cfg_l1d_spm_qs; + reg_rdata_next[3:0] = cluster_eoc_exit_qs; end addr_hit[8]: begin - reg_rdata_next[1:0] = cfg_l1d_insn_qs; + reg_rdata_next[9:0] = cfg_l1d_spm_qs; end addr_hit[9]: begin - reg_rdata_next[31:0] = cfg_l1d_tile_sel_0_qs; + reg_rdata_next[1:0] = cfg_l1d_insn_qs; end addr_hit[10]: begin - reg_rdata_next[31:0] = cfg_l1d_tile_sel_1_qs; + reg_rdata_next[31:0] = cfg_l1d_tile_sel_0_qs; end addr_hit[11]: begin - reg_rdata_next[31:0] = hw_barrier_participation_mask_0_qs; + reg_rdata_next[31:0] = cfg_l1d_tile_sel_1_qs; end addr_hit[12]: begin - reg_rdata_next[31:0] = hw_barrier_participation_mask_1_qs; + reg_rdata_next[31:0] = hw_barrier_participation_mask_0_qs; end addr_hit[13]: begin - reg_rdata_next[0] = l1d_spm_commit_qs; + reg_rdata_next[31:0] = hw_barrier_participation_mask_1_qs; end addr_hit[14]: begin - reg_rdata_next[0] = l1d_insn_commit_qs; + reg_rdata_next[0] = l1d_spm_commit_qs; end addr_hit[15]: begin - reg_rdata_next[0] = l1d_flush_status_qs; + reg_rdata_next[0] = l1d_insn_commit_qs; end addr_hit[16]: begin - reg_rdata_next[3:0] = l1d_private_qs; + reg_rdata_next[0] = l1d_flush_status_qs; end addr_hit[17]: begin - reg_rdata_next[31:0] = l1d_addr_qs; + reg_rdata_next[3:0] = l1d_private_qs; end addr_hit[18]: begin - reg_rdata_next[4:0] = xbar_offset_qs; + reg_rdata_next[31:0] = l1d_addr_qs; end addr_hit[19]: begin + reg_rdata_next[4:0] = xbar_offset_qs; + end + + addr_hit[20]: begin reg_rdata_next[0] = xbar_offset_commit_qs; end diff --git a/hardware/src/acc_mux.sv b/hardware/src/acc_mux.sv index 5f8b5f1b..9249936c 100644 --- a/hardware/src/acc_mux.sv +++ b/hardware/src/acc_mux.sv @@ -14,6 +14,11 @@ /// outstanding at a time -- a FIFO tracks which host is owed the in-flight /// response (pushed on grant, popped on response), so a new request is only /// arbitrated once the FIFO is empty. +/// Exclusive-owner-path response draining stays active through owner_active_i +/// (Locked and RelWait, since owner_id_i is valid through both), decoupled +/// from new-issue forwarding (locked_i only, steady-state Locked) -- a +/// response already in flight when release is requested would otherwise +/// have no drain path once locked_i drops for RelWait. module acc_mux #( parameter type acc_issue_req_t = logic, parameter type acc_issue_rsp_t = logic, @@ -25,6 +30,7 @@ module acc_mux #( input logic owner_id_i, input logic locked_i, input logic waiting_i, + input logic owner_active_i, // Per-host Snitch acc interfaces input acc_issue_req_t [1:0] acc_snitch_req_i, @@ -95,7 +101,7 @@ module acc_mux #( // Only track requests with a real completion coming: non-writeback ops never produce a later acc_rsp_t. assign route_fifo_push = idle_req_o && idle_gnt_i && spatz_issue_rsp_i.writeback; - assign route_fifo_pop = rsp_fire_o && !locked_i; + assign route_fifo_pop = rsp_fire_o && !owner_active_i; // Depth 2 for safety margin; only 1 entry is ever pushed at a time. fifo_v3 #( @@ -135,16 +141,21 @@ module acc_mux #( acc_snitch_prsp_o = '0; acc_snitch_pvalid_o = '0; - if (locked_i) begin - // Exclusive owner path: real access, no arbitration needed. - spatz_issue_req_o = acc_snitch_req_i[owner_id_i]; - spatz_issue_valid_o = acc_snitch_qvalid_i[owner_id_i]; - spatz_rsp_ready_o = acc_snitch_pready_i[owner_id_i]; - - acc_snitch_qready_o[owner_id_i] = spatz_issue_ready_i; - acc_snitch_rsp_o[owner_id_i] = spatz_issue_rsp_i; + if (owner_active_i) begin + // Exclusive owner path: response draining stays active through Locked + // and RelWait; new-issue forwarding only happens in steady-state + // Locked, never while a switch is pending. + spatz_rsp_ready_o = acc_snitch_pready_i[owner_id_i]; acc_snitch_prsp_o[owner_id_i] = spatz_rsp_i; acc_snitch_pvalid_o[owner_id_i] = spatz_rsp_valid_i; + + if (locked_i) begin + spatz_issue_req_o = acc_snitch_req_i[owner_id_i]; + spatz_issue_valid_o = acc_snitch_qvalid_i[owner_id_i]; + + acc_snitch_qready_o[owner_id_i] = spatz_issue_ready_i; + acc_snitch_rsp_o[owner_id_i] = spatz_issue_rsp_i; + end end else begin // Draining is unconditional on waiting_i; only new-request arbitration is not. if (!route_fifo_empty) begin diff --git a/hardware/src/cachepool_cc_dual.sv b/hardware/src/cachepool_cc_dual.sv index 436c5869..ade2d2e6 100644 --- a/hardware/src/cachepool_cc_dual.sv +++ b/hardware/src/cachepool_cc_dual.sv @@ -105,8 +105,7 @@ module cachepool_cc_dual input addr_t tcdm_addr_base_i, input addr_t cluster_periph_start_address_i, // Spatz lock/switch status (debug) - output logic owner_id_o, - output logic lock_error_o + output logic owner_id_o ); // FMA architecture is "merged" -> mulexp and macexp instructions are supported @@ -125,7 +124,7 @@ module cachepool_cc_dual `REQRSP_TYPEDEF_ALL(reqrsp, addr_t, data_t, strb_t, tcdm_user_t) - logic owner_id, locked, waiting; + logic owner_id, locked, waiting, owner_active; logic req_fire, rsp_fire; logic last_req_host; assign owner_id_o = owner_id; @@ -272,7 +271,7 @@ module cachepool_cc_dual .owner_id_o (owner_id ), .locked_o (locked ), .waiting_o (waiting ), - .error_o (lock_error_o ), + .owner_active_o (owner_active ), .cluster_periph_start_address_i (cluster_periph_start_address_i) ); @@ -288,6 +287,7 @@ module cachepool_cc_dual .owner_id_i (owner_id ), .locked_i (locked ), .waiting_i (waiting ), + .owner_active_i (owner_active ), .acc_snitch_req_i (acc_snitch_req ), .acc_snitch_rsp_o (acc_snitch_rsp ), .acc_snitch_qvalid_i (acc_snitch_qvalid ), diff --git a/hardware/src/cachepool_spatz_lock.sv b/hardware/src/cachepool_spatz_lock.sv index 4b541ddc..623eef9d 100644 --- a/hardware/src/cachepool_spatz_lock.sv +++ b/hardware/src/cachepool_spatz_lock.sv @@ -8,16 +8,20 @@ `include "common_cells/registers.svh" /// Spatz ownership lock/switch for a dual-Snitch cachepool_cc_dual. -/// Host 0 is the implicit owner while Idle; a write of 1 to the lock -/// address acquires ownership (blocks until granted), a write of 0 -/// releases it (owner only). Pure arbiter: does not route the acc/data -/// traffic itself (that lives in acc_mux.sv/cachepool_cc_dual.sv) -- only -/// decides ownership and exposes owner_id_o/locked_o/waiting_o for the CC -/// to apply, and counts real Spatz acc handshakes (via -/// req_fire_i/rsp_fire_i, fed back by acc_mux) to gate a switch until -/// fully drained. Separately tracks outstanding vle/vse ops (invisible to -/// req_fire_i/rsp_fire_i, since they never produce an acc_rsp_t writeback) -/// via Spatz's own spatz_mem_finished_i/spatz_st_rsp_done_i. +/// Host 0 is the implicit owner while Free. Two dedicated addresses are +/// intercepted: a load to ACQUIRE attempts to acquire, a load to RELEASE +/// attempts to release -- both always complete immediately (q_ready is +/// never withheld), and the loaded word encodes the outcome (fail/ +/// success/success-wait) plus current owner/locked status, so software is +/// never blocked in hardware and can always retry or bail out on its own. +/// Pure arbiter: does not route the acc/data traffic itself (that lives in +/// acc_mux.sv/cachepool_cc_dual.sv) -- only decides ownership and exposes +/// owner_id_o/locked_o/waiting_o/owner_active_o for the CC to apply, and +/// counts real Spatz acc handshakes (via req_fire_i/rsp_fire_i, fed back +/// by acc_mux) to gate a switch until fully drained. Separately tracks +/// outstanding vle/vse ops (invisible to req_fire_i/rsp_fire_i, since they +/// never produce an acc_rsp_t writeback) via Spatz's own +/// spatz_mem_finished_i/spatz_st_rsp_done_i. /// Also hosts the pass-through memory-path register cuts (moved here from /// the per-core spill registers in cachepool_cc.sv). module cachepool_spatz_lock @@ -38,7 +42,7 @@ module cachepool_spatz_lock input logic clk_i, input logic rst_ni, - // memory interface, used to set the lock + // memory interface, used to attempt acquire/release input dreq_t [NumHosts-1:0] in_req_i, output drsp_t [NumHosts-1:0] in_rsp_o, output dreq_t [NumHosts-1:0] out_req_o, @@ -55,59 +59,76 @@ module cachepool_spatz_lock input logic [1:0] spatz_mem_finished_i, input logic spatz_st_rsp_done_i, - // owner/lock status and error + // owner/lock status output logic owner_id_o, output logic locked_o, output logic waiting_o, - output logic error_o, + // True while the exclusive owner path may still have in-flight traffic to + // drain (Locked and RelWait); acc_mux uses this to keep response draining + // active through RelWait even though new-issue forwarding must stop. + output logic owner_active_o, // peripheral base address input addr_t cluster_periph_start_address_i ); - addr_t lock_addr; - assign lock_addr = cluster_periph_start_address_i + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_OFFSET; - - // Single lock FSM. Idle/Locked are the two "real" states (host 0 is the - // implicit owner while Idle); AcqWait/RelWait are drain-wait sub-states - // on the way to an actual ownership switch. - // Idle -(acquire, other host)-> AcqWait -(drain_done)-> Locked (new owner) - // Locked -(release, owner) -> RelWait -(drain_done)-> Idle (owner = host 0) - // Self-acquire completes immediately only if already drained; otherwise it - // waits through AcqWait too. Reads always complete immediately. - // A non-owner acquire/release, or any lock op during a wait, is stalled - // (never accepted) and sets the sticky error_o. + addr_t acquire_addr, release_addr; + assign acquire_addr = cluster_periph_start_address_i + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_OFFSET; + assign release_addr = cluster_periph_start_address_i + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_OFFSET; + + // Single lock FSM. Free/Locked are the two "real" states (host 0 is the + // implicit owner while Free); AcqWait/RelWait are drain-wait sub-states + // on the way to an actual ownership switch, resolved autonomously once + // drain_done -- no response is owed for that internal transition, since + // the triggering request already got a SUCCESS-WAIT answer up front. + // Free -(acquire, other host)-> AcqWait -(drain_done)-> Locked (new owner) + // Locked -(release, owner) -> RelWait -(drain_done)-> Free (owner = host 0) + // Every hit always completes immediately (q_ready is never withheld); + // the response encodes an outcome (FAIL/SUCCESS/SUCCESS-WAIT) instead of + // stalling the requester. A hit during AcqWait/RelWait, from either host, + // always gets FAIL(PENDING) -- only the one already-in-flight transition + // is ever active at a time. // // Handshake timing: the q-channel is accepted (q_ready=1) exactly on the - // deciding cycle (the last cycle of a wait, or immediately for the - // no-wait cases); the response (p_valid=1) is then held starting the + // deciding cycle; the response (p_valid=1) is then held starting the // following cycle until the host takes it via p_ready. typedef enum logic [1:0] { - Idle, + Free, AcqWait, RelWait, Locked } lock_state_e; + // Response payload bit layout (see note.md): [1:0]=outcome, [4:2]=reason + // (valid only on FAIL), [5]=owner, [6]=locked -- all post-decision. + typedef enum logic [1:0] { + OutcomeFail = 2'd0, + OutcomeSuccess = 2'd1, + OutcomeSuccessWait = 2'd2 + } lock_outcome_e; + + typedef enum logic [2:0] { + ReasonNotOwner = 3'd0, + ReasonBusy = 3'd1, + ReasonPending = 3'd2 + } lock_fail_reason_e; + lock_state_e lock_d, lock_q; logic owner_d, owner_q; - logic error_d, error_q; logic active_d, active_q; - user_t user_d, user_q; // Outstanding acc handshakes on the owner's path; must reach 0 before a wait can complete. logic [7:0] outstanding_d, outstanding_q; // Outstanding Spatz vle/vse ops; mirrors snitch.sv's own acc_mem_cnt_q so a // switch can't happen while a load/store is still draining through the cache. logic [7:0] lsu_outstanding_d, lsu_outstanding_q; - logic drain_done, waiting; + logic drain_done; - assign owner_id_o = owner_q; - assign locked_o = (lock_q == Locked); - assign waiting_o = waiting; - assign error_o = error_q; + assign owner_id_o = owner_q; + assign locked_o = (lock_q == Locked); + assign waiting_o = (lock_q == AcqWait) || (lock_q == RelWait); + assign owner_active_o = (lock_q == Locked) || (lock_q == RelWait); - assign waiting = (lock_q == AcqWait) || (lock_q == RelWait); assign drain_done = (outstanding_q == '0) && (lsu_outstanding_q == '0) && spatz_st_rsp_done_i; `ASSERT(NoOutstandingUnderflow, rsp_fire_i |-> (outstanding_q != '0)) @@ -138,29 +159,32 @@ module cachepool_spatz_lock end end - // write 1 for acquire, write 0 for release, read returns the lock status - logic [1:0] is_acquire, is_release, is_read, lock_hit; + // Address-based op decode: a hit on either address is serviced regardless + // of q.write, since only loads carry a usable result back to software. + logic [1:0] acquire_hit, release_hit, lock_hit; for (genvar i = 0; i < 2; i++) begin - assign is_acquire[i] = in_req_i[i].q.write && in_req_i[i].q.data[0]; - assign is_release[i] = in_req_i[i].q.write && !in_req_i[i].q.data[0]; - assign is_read[i] = !in_req_i[i].q.write; - assign lock_hit[i] = in_req_i[i].q_valid && (in_req_i[i].q.addr == lock_addr); + assign acquire_hit[i] = in_req_i[i].q_valid && (in_req_i[i].q.addr == acquire_addr); + assign release_hit[i] = in_req_i[i].q_valid && (in_req_i[i].q.addr == release_addr); + assign lock_hit[i] = acquire_hit[i] || release_hit[i]; end - // The owner's hit always wins arbitration, so a non-owner's (always-illegal-in-Locked) - // request can never starve the owner's legitimate one. + // The owner's hit always wins arbitration when both hosts hit the same + // cycle; the loser simply waits one extra cycle for resp_pending_q to + // clear (both eventually get answered, this is only a tie-break). logic hit_any, winner; assign hit_any = lock_hit[0] || lock_hit[1]; assign winner = lock_hit[owner_q] ? owner_q : ~owner_q; // Pending, not-yet-delivered completion response (one at a time; a new lock op is // only arbitrated once the previous one's response has been taken). - logic resp_pending_d, resp_pending_q; - logic resp_host_d, resp_host_q; - user_t resp_user_d, resp_user_q; - logic resp_read_d, resp_read_q; - logic [1:0] resp_status_d, resp_status_q; + logic resp_pending_d, resp_pending_q; + logic resp_host_d, resp_host_q; + user_t resp_user_d, resp_user_q; + lock_outcome_e resp_outcome_d, resp_outcome_q; + lock_fail_reason_e resp_reason_d, resp_reason_q; + logic resp_owner_d, resp_owner_q; + logic resp_locked_d, resp_locked_q; // Register cuts on the pass-through path (moved here from cachepool_cc.sv's per-core // spill registers), placed at the module's output side. @@ -202,14 +226,14 @@ module cachepool_spatz_lock always_comb begin lock_d = lock_q; owner_d = owner_q; - error_d = error_q; active_d = active_q; - user_d = user_q; resp_pending_d = resp_pending_q; resp_host_d = resp_host_q; resp_user_d = resp_user_q; - resp_read_d = resp_read_q; - resp_status_d = resp_status_q; + resp_outcome_d = resp_outcome_q; + resp_reason_d = resp_reason_q; + resp_owner_d = resp_owner_q; + resp_locked_d = resp_locked_q; // Default: pass through the (registered) memory path; blocked on a lock-address hit. for (int i = 0; i < 2; i++) begin @@ -221,138 +245,148 @@ module cachepool_spatz_lock // Deliver a pending completion response, starting the cycle after it was accepted. if (resp_pending_q) begin - in_rsp_o[resp_host_q] = '0; - in_rsp_o[resp_host_q].p_valid = 1'b1; - in_rsp_o[resp_host_q].p.user = resp_user_q; - if (resp_read_q) begin - in_rsp_o[resp_host_q].p.data[0] = resp_status_q[0]; - in_rsp_o[resp_host_q].p.data[1] = resp_status_q[1]; - end + in_rsp_o[resp_host_q] = '0; + in_rsp_o[resp_host_q].p_valid = 1'b1; + in_rsp_o[resp_host_q].p.user = resp_user_q; + in_rsp_o[resp_host_q].p.data[1:0] = resp_outcome_q; + in_rsp_o[resp_host_q].p.data[4:2] = resp_reason_q; + in_rsp_o[resp_host_q].p.data[5] = resp_owner_q; + in_rsp_o[resp_host_q].p.data[6] = resp_locked_q; if (in_req_i[resp_host_q].p_ready) resp_pending_d = 1'b0; end case (lock_q) // Host 0 is the implicit owner. Its own release is a no-op (immediate grant); - // its own acquire grants immediately only if drained, else waits like host 1's; - // host 1's release is meaningless (nothing to release) and stalled. - Idle: begin + // an acquire from either host grants immediately once drained, else waits. + Free: begin if (hit_any && !resp_pending_q) begin - if (is_acquire[winner]) begin - if (winner == owner_q && drain_done) begin - // host 0 self-acquire, nothing outstanding: immediate grant - lock_d = Locked; - in_rsp_o[winner].q_ready = 1'b1; - resp_pending_d = 1'b1; - resp_host_d = winner; - resp_user_d = in_req_i[winner].q.user; - resp_read_d = 1'b0; + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + + if (acquire_hit[winner]) begin + if (drain_done) begin + lock_d = Locked; + owner_d = winner; + resp_outcome_d = OutcomeSuccess; + resp_owner_d = winner; + resp_locked_d = 1'b1; end else begin - // host 1 wants the lock, or host 0 self-acquires while Idle-mode - // traffic (from either host) is still outstanding -> drain first - active_d = winner; - user_d = in_req_i[winner].q.user; - lock_d = AcqWait; + active_d = winner; + lock_d = AcqWait; + resp_outcome_d = OutcomeSuccessWait; + resp_owner_d = owner_q; + resp_locked_d = 1'b0; end - end else if (is_release[winner]) begin + end else begin // release_hit[winner] if (winner == owner_q) begin - // "self release": nothing to release, immediate no-op grant - in_rsp_o[winner].q_ready = 1'b1; - resp_pending_d = 1'b1; - resp_host_d = winner; - resp_user_d = in_req_i[winner].q.user; - resp_read_d = 1'b0; + // host 0 "self release": nothing to release, immediate no-op grant + resp_outcome_d = OutcomeSuccess; + resp_owner_d = owner_q; + resp_locked_d = 1'b0; end else begin - // host 1 release: stalled, never accepted - error_d = 1'b1; + // host 1 release: never held it + resp_outcome_d = OutcomeFail; + resp_reason_d = ReasonNotOwner; + resp_owner_d = owner_q; + resp_locked_d = 1'b0; end - end else if (is_read[winner]) begin - in_rsp_o[winner].q_ready = 1'b1; - resp_pending_d = 1'b1; - resp_host_d = winner; - resp_user_d = in_req_i[winner].q.user; - resp_read_d = 1'b1; - resp_status_d = {1'b0, owner_q}; end end end - // Only the owner's acquire (no-op) or release (drains, reverts to Idle) is ever - // legal; a non-owner's acquire/release is stalled and flagged. + // Only the owner's acquire (no-op) or release is meaningful; a non-owner's + // acquire/release always fails outright (no queueing for a future handoff). Locked: begin if (hit_any && !resp_pending_q) begin - if (is_acquire[winner]) begin + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + + if (acquire_hit[winner]) begin + resp_outcome_d = (winner == owner_q) ? OutcomeSuccess : OutcomeFail; + resp_reason_d = ReasonBusy; + resp_owner_d = owner_q; + resp_locked_d = 1'b1; + end else begin // release_hit[winner] if (winner == owner_q) begin - // owner self-acquire: no-op, immediate grant - in_rsp_o[winner].q_ready = 1'b1; - resp_pending_d = 1'b1; - resp_host_d = winner; - resp_user_d = in_req_i[winner].q.user; - resp_read_d = 1'b0; + if (drain_done) begin + lock_d = Free; + owner_d = 1'b0; + resp_outcome_d = OutcomeSuccess; + resp_owner_d = 1'b0; + resp_locked_d = 1'b0; + end else begin + active_d = winner; + lock_d = RelWait; + resp_outcome_d = OutcomeSuccessWait; + resp_owner_d = owner_q; + resp_locked_d = 1'b1; + end end else begin - // non-owner acquire: stalled, never accepted - error_d = 1'b1; + resp_outcome_d = OutcomeFail; + resp_reason_d = ReasonNotOwner; + resp_owner_d = owner_q; + resp_locked_d = 1'b1; end - end else if (is_release[winner]) begin - if (winner == owner_q) begin - // owner release -> drain, then revert to Idle - active_d = winner; - user_d = in_req_i[winner].q.user; - lock_d = RelWait; - end else begin - // non-owner release: stalled, never accepted - error_d = 1'b1; - end - end else if (is_read[winner]) begin - in_rsp_o[winner].q_ready = 1'b1; - resp_pending_d = 1'b1; - resp_host_d = winner; - resp_user_d = in_req_i[winner].q.user; - resp_read_d = 1'b1; - resp_status_d = {1'b1, owner_q}; end end end - // No lock op is accepted from anyone while waiting. + // Drain-wait sub-states: any hit (either host, either op) always fails with + // PENDING -- the one already-registered transition (active_q) resolves on + // its own once drain_done, with no response owed for that completion. AcqWait: begin + if (hit_any && !resp_pending_q) begin + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + resp_outcome_d = OutcomeFail; + resp_reason_d = ReasonPending; + resp_owner_d = owner_q; + resp_locked_d = 1'b0; + end if (drain_done) begin - owner_d = active_q; - lock_d = Locked; - in_rsp_o[active_q].q_ready = 1'b1; - resp_pending_d = 1'b1; - resp_host_d = active_q; - resp_user_d = user_q; - resp_read_d = 1'b0; + owner_d = active_q; + lock_d = Locked; end end RelWait: begin + if (hit_any && !resp_pending_q) begin + in_rsp_o[winner].q_ready = 1'b1; + resp_pending_d = 1'b1; + resp_host_d = winner; + resp_user_d = in_req_i[winner].q.user; + resp_outcome_d = OutcomeFail; + resp_reason_d = ReasonPending; + resp_owner_d = owner_q; + resp_locked_d = 1'b1; + end if (drain_done) begin - owner_d = 1'b0; - lock_d = Idle; - in_rsp_o[active_q].q_ready = 1'b1; - resp_pending_d = 1'b1; - resp_host_d = active_q; - resp_user_d = user_q; - resp_read_d = 1'b0; + owner_d = 1'b0; + lock_d = Free; end end - default: lock_d = Idle; + default: lock_d = Free; endcase end - `FF(lock_q, lock_d, Idle, clk_i, rst_ni) - `FF(owner_q, owner_d, 1'b0, clk_i, rst_ni) - `FF(error_q, error_d, 1'b0, clk_i, rst_ni) - `FF(active_q, active_d, 1'b0, clk_i, rst_ni) - `FF(user_q, user_d, '0, clk_i, rst_ni) - `FF(outstanding_q, outstanding_d, '0, clk_i, rst_ni) - `FF(lsu_outstanding_q, lsu_outstanding_d, '0, clk_i, rst_ni) - `FF(resp_pending_q, resp_pending_d, 1'b0, clk_i, rst_ni) - `FF(resp_host_q, resp_host_d, 1'b0, clk_i, rst_ni) - `FF(resp_user_q, resp_user_d, '0, clk_i, rst_ni) - `FF(resp_read_q, resp_read_d, 1'b0, clk_i, rst_ni) - `FF(resp_status_q, resp_status_d, '0, clk_i, rst_ni) + `FF(lock_q, lock_d, Free, clk_i, rst_ni) + `FF(owner_q, owner_d, 1'b0, clk_i, rst_ni) + `FF(active_q, active_d, 1'b0, clk_i, rst_ni) + `FF(outstanding_q, outstanding_d, '0, clk_i, rst_ni) + `FF(lsu_outstanding_q, lsu_outstanding_d, '0, clk_i, rst_ni) + `FF(resp_pending_q, resp_pending_d, 1'b0, clk_i, rst_ni) + `FF(resp_host_q, resp_host_d, 1'b0, clk_i, rst_ni) + `FF(resp_user_q, resp_user_d, '0, clk_i, rst_ni) + `FF(resp_outcome_q, resp_outcome_d, OutcomeFail, clk_i, rst_ni) + `FF(resp_reason_q, resp_reason_d, ReasonNotOwner,clk_i, rst_ni) + `FF(resp_owner_q, resp_owner_d, 1'b0, clk_i, rst_ni) + `FF(resp_locked_q, resp_locked_d, 1'b0, clk_i, rst_ni) endmodule diff --git a/hardware/src/cachepool_tile.sv b/hardware/src/cachepool_tile.sv index 8a275510..f6b83efb 100644 --- a/hardware/src/cachepool_tile.sv +++ b/hardware/src/cachepool_tile.sv @@ -1628,8 +1628,7 @@ module cachepool_tile .tcdm_rsp_ready_o (tcdm_rsp_ready[TcdmPortsOffs +: TcdmPorts] ), .tcdm_addr_base_i (tcdm_start_address ), .cluster_periph_start_address_i (cluster_periph_start_address), - .owner_id_o (/* debug only, unused */ ), - .lock_error_o (/* debug only, unused */ ) + .owner_id_o (/* debug only, unused */ ) ); end else begin : gen_unsupported `ASSERT_INIT(NumScalarPerCCSupported, 1'b0, diff --git a/software/snRuntime/README.md b/software/snRuntime/README.md index b804918d..5d1208ed 100644 --- a/software/snRuntime/README.md +++ b/software/snRuntime/README.md @@ -9,6 +9,7 @@ snRuntime/ ├── include/ # Public headers — include these in application code │ ├── snrt.h # Master header: topology, barriers, DMA, allocation │ ├── l1cache.h # CachePool L1 data cache management API +│ ├── spatz_lock.h # Dual-scalar Spatz ownership lock API │ ├── cachepool_peripheral.h # Register offsets for the cluster peripheral │ ├── perf_cnt.h # Performance counter API │ ├── team.h # Team/cluster descriptor structs @@ -23,6 +24,7 @@ snRuntime/ │ ├── team.c # Team/topology initialisation │ ├── barrier.c # Hardware and software barrier implementations │ ├── l1cache.c # CachePool L1 cache management (flush, partition, xbar) +│ ├── spatz_lock.c # Dual-scalar Spatz ownership lock (see spatz_lock.h) │ ├── alloc.c # L1 TCDM bump allocator + DRAM linked-list allocator │ ├── memcpy.c # Optimised memcpy │ ├── perf_cnt.c # Performance counter helpers @@ -103,6 +105,47 @@ void l1d_wait(); void l1d_init(uint32_t size); ``` +### Dual-Scalar Spatz Lock — CachePool-specific (`spatz_lock.h`) + +Only meaningful on a `num_scalar_per_core=2` build (`cachepool_cc_dual`), where 2 Snitch +harts share 1 Spatz unit. On a single-scalar-per-CC build every call below is a harmless +no-op that always reports success. + +```c +void spatz_lock_acquire(); // blocks (retries in software) until this hart owns Spatz +void spatz_lock_release(); // must be called by the current owner only +``` + +Both are plain, non-blocking-in-hardware retries under the hood — see `spatz_lock_outcome_t` +below — so a hart can never hang in hardware waiting on the other hart to release. After +`spatz_lock_acquire()` returns, it is always safe to issue vector/FP work immediately: if the +ownership switch is still draining, `acc_mux` (RTL) withholds Spatz access until it completes, +so the next vector/FP instruction just blocks there instead, exactly as if the acquire itself +had blocked. + +For finer control (e.g. to do other work instead of retrying), use the non-blocking primitives +directly: + +```c +typedef enum { + SPATZ_LOCK_FAIL = 0, // denied; hardware made no reservation, safe to retry + SPATZ_LOCK_SUCCESS = 1, // granted now + SPATZ_LOCK_SUCCESS_WAIT = 2, // accepted, completes on its own once drained +} spatz_lock_outcome_t; + +uint32_t spatz_lock_try_acquire(); // single, always-immediate attempt +uint32_t spatz_lock_try_release(); +spatz_lock_outcome_t spatz_lock_outcome(uint32_t raw); // decode the above +``` + +Related topology/sync helpers in `snrt.h`: + +```c +int snrt_cluster_is_primary(); // true for the pair's default owner (even cid) +void snrt_cluster_host0_barrier(); // partial barrier over default owners only +void snrt_cluster_host1_barrier(); // partial barrier over their partners only +``` + ### Performance Counters (`perf_cnt.h`) *TODO: REMOVE* ```c diff --git a/software/snRuntime/include/cachepool_peripheral.h b/software/snRuntime/include/cachepool_peripheral.h index f118c267..9eb2b8eb 100644 --- a/software/snRuntime/include/cachepool_peripheral.h +++ b/software/snRuntime/include/cachepool_peripheral.h @@ -24,25 +24,28 @@ extern "C" { // cores have #define CACHEPOOL_PERIPHERAL_HW_BARRIER_REG_OFFSET 0x0 -// Spatz ownership lock for dual-Snitch core complexes. Write 1 to acquire -#define CACHEPOOL_PERIPHERAL_SPATZ_LOCK_REG_OFFSET 0x4 +// Spatz ownership acquire for dual-Snitch core complexes. Load +#define CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_REG_OFFSET 0x4 + +// Spatz ownership release for dual-Snitch core complexes. Load +#define CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_REG_OFFSET 0x8 // Controls prefetching of the instruction cache. -#define CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_REG_OFFSET 0x8 +#define CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_REG_OFFSET 0xc #define CACHEPOOL_PERIPHERAL_ICACHE_PREFETCH_ENABLE_ICACHE_PREFETCH_ENABLE_BIT 0 // Sets the status of the Spatz cluster. -#define CACHEPOOL_PERIPHERAL_SPATZ_STATUS_REG_OFFSET 0xc +#define CACHEPOOL_PERIPHERAL_SPATZ_STATUS_REG_OFFSET 0x10 #define CACHEPOOL_PERIPHERAL_SPATZ_STATUS_SPATZ_CLUSTER_PROBE_BIT 0 // Store cycle counts of kernels -#define CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_REG_OFFSET 0x10 +#define CACHEPOOL_PERIPHERAL_SPATZ_CYCLE_REG_OFFSET 0x14 // Controls the cluster boot process. -#define CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_REG_OFFSET 0x14 +#define CACHEPOOL_PERIPHERAL_CLUSTER_BOOT_CONTROL_REG_OFFSET 0x18 // End of computation and exit status register -#define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_REG_OFFSET 0x18 +#define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_REG_OFFSET 0x1c #define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_MASK 0xf #define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_OFFSET 0 #define CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_FIELD \ @@ -51,7 +54,7 @@ extern "C" { .index = CACHEPOOL_PERIPHERAL_CLUSTER_EOC_EXIT_EOC_EXIT_OFFSET}) // Controls the configurations of L1 DCache SPM size. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_REG_OFFSET 0x1c +#define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_REG_OFFSET 0x20 #define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_MASK 0x3ff #define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_OFFSET 0 #define CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_FIELD \ @@ -60,7 +63,7 @@ extern "C" { .index = CACHEPOOL_PERIPHERAL_CFG_L1D_SPM_SPM_SIZE_OFFSET}) // Controls the L1 DCache flushing and invalidation. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_REG_OFFSET 0x20 +#define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_REG_OFFSET 0x24 #define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_INSN_MASK 0x3 #define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_INSN_OFFSET 0 #define CACHEPOOL_PERIPHERAL_CFG_L1D_INSN_INSN_FIELD \ @@ -74,10 +77,10 @@ extern "C" { #define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_MULTIREG_COUNT 2 // One-hot tile selection mask for private-partition flush. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_REG_OFFSET 0x24 +#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_0_REG_OFFSET 0x28 // One-hot tile selection mask for private-partition flush. -#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_REG_OFFSET 0x28 +#define CACHEPOOL_PERIPHERAL_CFG_L1D_TILE_SEL_1_REG_OFFSET 0x2c // Tile participation mask for the cluster-level hardware barrier. #define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_TILE_FIELD_WIDTH 32 @@ -85,25 +88,25 @@ extern "C" { #define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_MULTIREG_COUNT 2 // Tile participation mask for the cluster-level hardware barrier. -#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_REG_OFFSET 0x2c +#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_0_REG_OFFSET 0x30 // Tile participation mask for the cluster-level hardware barrier. -#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_REG_OFFSET 0x30 +#define CACHEPOOL_PERIPHERAL_HW_BARRIER_PARTICIPATION_MASK_1_REG_OFFSET 0x34 // Controls the L1 DCache flushing and invalidation. -#define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_REG_OFFSET 0x34 +#define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_REG_OFFSET 0x38 #define CACHEPOOL_PERIPHERAL_L1D_SPM_COMMIT_COMMIT_BIT 0 // Controls the L1 DCache flushing and invalidation. -#define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_REG_OFFSET 0x38 +#define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_REG_OFFSET 0x3c #define CACHEPOOL_PERIPHERAL_L1D_INSN_COMMIT_COMMIT_BIT 0 // Indicate the status of flushing -#define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_REG_OFFSET 0x3c +#define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_REG_OFFSET 0x40 #define CACHEPOOL_PERIPHERAL_L1D_FLUSH_STATUS_STATUS_BIT 0 // Number of private banks configured per tile -#define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_REG_OFFSET 0x40 +#define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_REG_OFFSET 0x44 #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_MASK 0xf #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_OFFSET 0 #define CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_FIELD \ @@ -112,10 +115,10 @@ extern "C" { CACHEPOOL_PERIPHERAL_L1D_PRIVATE_NUMBER_OFFSET}) // Starting address of private L1D partition -#define CACHEPOOL_PERIPHERAL_L1D_ADDR_REG_OFFSET 0x44 +#define CACHEPOOL_PERIPHERAL_L1D_ADDR_REG_OFFSET 0x48 // Cache xbar offset setting -#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET 0x48 +#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_REG_OFFSET 0x4c #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_MASK 0x1f #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_OFFSET 0 #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_FIELD \ @@ -124,7 +127,7 @@ extern "C" { CACHEPOOL_PERIPHERAL_XBAR_OFFSET_OFFSET_OFFSET}) // Cache xbar offset setting -#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_REG_OFFSET 0x4c +#define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_REG_OFFSET 0x50 #define CACHEPOOL_PERIPHERAL_XBAR_OFFSET_COMMIT_COMMIT_BIT 0 #ifdef __cplusplus diff --git a/software/snRuntime/include/spatz_lock.h b/software/snRuntime/include/spatz_lock.h index 8f73f5ec..b6985cae 100644 --- a/software/snRuntime/include/spatz_lock.h +++ b/software/snRuntime/include/spatz_lock.h @@ -3,15 +3,39 @@ // SPDX-License-Identifier: Apache-2.0 +#include + #include "cachepool_peripheral.h" #include "team.h" extern __thread struct snrt_team *_snrt_team_current; -// Acquire/release Spatz ownership on a dual-Snitch Core Complex. Each store -// blocks in hardware until granted, so no software polling is needed. Only -// call release after a successful acquire on the same hart, and only from -// harts sharing a Spatz (cachepool_cc_dual) -- on a single-scalar-per-CC -// build these write a harmless, unused peripheral register. +// Outcome of a single spatz_lock_try_acquire()/try_release() attempt -- +// bits [1:0] of the raw register value, see cachepool_spatz_lock.sv. +typedef enum { + SPATZ_LOCK_FAIL = 0, // denied; hardware made no reservation + SPATZ_LOCK_SUCCESS = 1, // granted now + SPATZ_LOCK_SUCCESS_WAIT = 2, // accepted, completes on its own once drained +} spatz_lock_outcome_t; + +// Single, always-immediate attempt (never blocks in hardware). Decode with +// spatz_lock_outcome(). +uint32_t spatz_lock_try_acquire(void); +uint32_t spatz_lock_try_release(void); + +static inline spatz_lock_outcome_t spatz_lock_outcome(uint32_t raw) { + return (spatz_lock_outcome_t)(raw & 0x3); +} + +// Acquire/release Spatz ownership on a dual-Snitch Core Complex. Retries +// spatz_lock_try_acquire()/try_release() until it stops failing: FAIL means +// genuine contention (nothing reserved, must retry); SUCCESS/SUCCESS_WAIT +// both return right away -- on SUCCESS_WAIT, the switch is still draining, +// but acc_mux already refuses any new Spatz issue until it's done, so the +// next real vector/FP instruction naturally blocks in hardware until the +// switch completes, same as before. Only call release after a successful +// acquire on the same hart, and only from harts sharing a Spatz +// (cachepool_cc_dual) -- on a single-scalar-per-CC build these access a +// harmless, unused peripheral register (every attempt returns SUCCESS). void spatz_lock_acquire(void); void spatz_lock_release(void); diff --git a/software/snRuntime/src/spatz_lock.c b/software/snRuntime/src/spatz_lock.c index f3d10a63..6579ba2b 100644 --- a/software/snRuntime/src/spatz_lock.c +++ b/software/snRuntime/src/spatz_lock.c @@ -6,16 +6,41 @@ #include #include -void spatz_lock_acquire(void) { +uint32_t spatz_lock_try_acquire(void) { +#if SNRT_NUM_SCALAR_PER_CORE == 2 volatile uint32_t *lock = (uint32_t *)(_snrt_team_current->root->cluster_mem.end + - CACHEPOOL_PERIPHERAL_SPATZ_LOCK_REG_OFFSET); - *lock = 1; + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_REG_OFFSET); + return *lock; +#else + // No lock module exists to intercept this on a single-scalar-per-CC + // build; report an unconditional grant rather than reading back the + // register's uninitialized reset value (which would decode as FAIL). + return SPATZ_LOCK_SUCCESS; +#endif } -void spatz_lock_release(void) { +uint32_t spatz_lock_try_release(void) { +#if SNRT_NUM_SCALAR_PER_CORE == 2 volatile uint32_t *lock = (uint32_t *)(_snrt_team_current->root->cluster_mem.end + - CACHEPOOL_PERIPHERAL_SPATZ_LOCK_REG_OFFSET); - *lock = 0; + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_REG_OFFSET); + return *lock; +#else + return SPATZ_LOCK_SUCCESS; +#endif +} + +void spatz_lock_acquire(void) { + spatz_lock_outcome_t outcome; + do { + outcome = spatz_lock_outcome(spatz_lock_try_acquire()); + } while (outcome == SPATZ_LOCK_FAIL); +} + +void spatz_lock_release(void) { + spatz_lock_outcome_t outcome; + do { + outcome = spatz_lock_outcome(spatz_lock_try_release()); + } while (outcome == SPATZ_LOCK_FAIL); } From ce4078500a54631ba6f71eedf97148b873d957d7 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Tue, 25 Aug 2026 16:58:31 +0200 Subject: [PATCH 14/17] [SRC] Fix a bug in Spatz related to dual-scalar core muxing. Add synthesis-only debug watchdog in the module --- Bender.lock | 2 +- README.md | 3 +- config/cachepool_dual_fpu_4g.mk | 117 ++++++++++++++++++++++ hardware/src/acc_mux.sv | 100 ++++++++++++++++--- hardware/src/cachepool_spatz_lock.sv | 141 +++++++++++++++++---------- 5 files changed, 295 insertions(+), 68 deletions(-) create mode 100644 config/cachepool_dual_fpu_4g.mk diff --git a/Bender.lock b/Bender.lock index 85d89802..b79fbf04 100644 --- a/Bender.lock +++ b/Bender.lock @@ -150,7 +150,7 @@ packages: dependencies: - tech_cells_generic spatz: - revision: f6ea19fb26c8bf992ae51a3e4acd3f2a36c6659f + revision: f4741b0a5bcc49103bd8a3db8d51fdf76d240df7 version: null source: Git: https://github.com/pulp-platform/spatz.git diff --git a/README.md b/README.md index c0bc4a7e..38e62419 100644 --- a/README.md +++ b/README.md @@ -207,10 +207,11 @@ Configuration names encode the number of groups and whether the FPU is enabled: | `cachepool_fpu_16g` | 16 | 4×4 | Yes | 4 | 4 | 256 | | `cachepool_fpu_16g_tiny` | 16 | 4×4 | Yes | 2 | 2 | 64 | | `cachepool_dual_4g` | 4 | 2×2 | No (IPU only) | 4 | 4 | 64 CCs / 128 harts | +| `cachepool_dual_fpu_4g` | 4 | 2×2 | Yes | 4 | 4 | 64 CCs / 128 harts | `cachepool_fpu_16g_tiny` shrinks tiles/group and cores/tile for a faster-to-build, faster-to-simulate smoke test of the full 16-group mesh topology. -`cachepool_dual_4g` sets `num_scalar_per_core=2`: each Core Complex holds 2 Snitch scalar harts sharing 1 Spatz unit via a hardware ownership lock (`cachepool_spatz_lock.sv`), so "Cores" (Core Complex slots) and hart count diverge — 64 CCs, 128 harts total. The lock never blocks a core's pipeline: every acquire/release attempt (`software/snRuntime/include/spatz_lock.h`) completes immediately with an outcome (granted / denied / granted-but-still-draining), so a hart contending for a lock it doesn't get can always retry or do something else instead of hanging. See `note.md` for the full state-machine design. +`cachepool_dual_4g`/`cachepool_dual_fpu_4g` set `num_scalar_per_core=2`: each Core Complex holds 2 Snitch scalar harts sharing 1 Spatz unit via a hardware ownership lock (`cachepool_spatz_lock.sv`), so "Cores" (Core Complex slots) and hart count diverge — 64 CCs, 128 harts total. `cachepool_dual_fpu_4g` is otherwise identical to `cachepool_fpu_4g` (same FPU/IPU counts), just with the shared-Spatz CC flavor — used to confirm the dual-scalar lock/mux design also works with the FPU enabled, not just IPU-only. The lock never blocks a core's pipeline: every acquire/release attempt (`software/snRuntime/include/spatz_lock.h`) completes immediately with an outcome (granted / denied / granted-but-still-draining), so a hart contending for a lock it doesn't get can always retry or do something else instead of hanging. See `note.md` for the full state-machine design. The Spatz cluster consumes **`config/cachepool.hjson`**, which is **generated** from: - `config/cachepool.hjson.tmpl` (skeleton with comments) diff --git a/config/cachepool_dual_fpu_4g.mk b/config/cachepool_dual_fpu_4g.mk new file mode 100644 index 00000000..bb21a89b --- /dev/null +++ b/config/cachepool_dual_fpu_4g.mk @@ -0,0 +1,117 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Author: Diyou Shen, ETH Zurich + +######################### +## CachePool Cluster ## +######################### + +# Number of groups +num_groups ?= 4 + +# 2×2 mesh +num_groups_x ?= 2 + +# Number of tiles +num_tiles_per_group ?= 4 + +# Number of Core Complex slots (each holding num_scalar_per_core harts) +num_cores_per_tile ?= 4 + +# Core datawidth +data_width ?= 32 + +# Core addrwidth +addr_width ?= 32 + +num_lg_ports_per_core ?= 2 + +num_rg_ports_per_core ?= 1 + +num_noc_ports_per_tile ?= 4 + + +###################### +## CachePool Tile ## +###################### + +##### L1 Data Cache ##### + +# L1 data cacheline width (in Bit) +l1d_cacheline_width ?= 512 + +# L1 data cache banking factor (how many banks per core?) +l1d_bank_factor ?= 1 + +# L1 coalecsing window +l1d_coal_window ?= 2 + +# L1 data cache number of ways per +l1d_num_way ?= 4 + +# L1 data cache size per tile (KiB) +l1d_tile_size ?= 256 + +# L1 data cache tag width (TODO: should be calcualted) +l1d_tag_data_width ?= 92 + +# Use folded (skewed) data banks +l1d_use_folded ?= 1 + +# Fold-way group size for skewed folded banks (0 = auto: min(4, ways)) +l1d_fold_way_group ?= 0 + +# Use hash-based way selection (required by l1d_use_folded / l1d_use_fwd_buf) +l1d_use_hash_way ?= 1 + +# Enable the SRAM forwarding buffer (requires l1d_use_hash_way) +l1d_use_fwd_buf ?= 1 + +#################### +## CachePool CC ## +#################### +# Spatz fpu support? +spatz_fpu_en ?= 1 + +# Spatz number of FPU +spatz_num_fpu ?= 4 + +# Spatz number of IPU +spatz_num_ipu ?= 4 + +# Spatz max outstanding transactions +spatz_max_trans ?= 32 + +# Snitch/FPU max outstanding transactions +snitch_max_trans ?= 16 + +# 2 Snitch scalar harts sharing one Spatz per Core Complex +num_scalar_per_core ?= 2 + + +##################### +## L2 Main Memory ## +##################### +# DRAM type (selects DRAMSys config and sets default refill_data_width) +dram_type ?= HBM2 + +# L2 number of channels +l2_channel ?= 4 + +# L2 bank width (DRAM width, change with care) +l2_bank_width ?= 512 + +# L2 interleaving factor (in order of bank_width) +l2_interleave ?= 16 + + +################## +## Peripherals ## +################## +# Hardware stack size (in Byte) +stack_hw_size ?= 1024 + +# Stack size (total, including share and private, 32'h800) +stack_tot_size ?= 2048 diff --git a/hardware/src/acc_mux.sv b/hardware/src/acc_mux.sv index 9249936c..32d8463b 100644 --- a/hardware/src/acc_mux.sv +++ b/hardware/src/acc_mux.sv @@ -9,16 +9,12 @@ /// Arbitrates 2 Snitch acc interfaces onto 1 shared Spatz acc interface for /// cachepool_cc_dual, gated by cachepool_spatz_lock's owner/locked/waiting -/// state. Locked: only the owner passes through, real access, no -/// arbitration. Idle: both hosts get real round-robin access, one request -/// outstanding at a time -- a FIFO tracks which host is owed the in-flight -/// response (pushed on grant, popped on response), so a new request is only -/// arbitrated once the FIFO is empty. -/// Exclusive-owner-path response draining stays active through owner_active_i -/// (Locked and RelWait, since owner_id_i is valid through both), decoupled -/// from new-issue forwarding (locked_i only, steady-state Locked) -- a -/// response already in flight when release is requested would otherwise -/// have no drain path once locked_i drops for RelWait. +/// state. Locked: only the owner passes through, real access, no arbitration. +/// Idle: both hosts get real round-robin access, one request outstanding at +/// a time -- a FIFO tracks which host is owed the in-flight response. +/// Response draining stays active through owner_active_i (Locked and +/// RelWait); new-issue forwarding is gated separately on locked_i alone, so +/// a response already in flight when release is requested still has a drain path. module acc_mux #( parameter type acc_issue_req_t = logic, parameter type acc_issue_rsp_t = logic, @@ -50,8 +46,7 @@ module acc_mux #( input logic spatz_rsp_valid_i, output logic spatz_rsp_ready_o, - // Drain feed for the lock's outstanding counter (owner path and Idle-mode - // arbitrated path both count, since both represent real Spatz traffic). + // Drain feed for the lock's outstanding counter (owner and Idle-mode paths both count). output logic req_fire_o, output logic rsp_fire_o, @@ -142,9 +137,7 @@ module acc_mux #( acc_snitch_pvalid_o = '0; if (owner_active_i) begin - // Exclusive owner path: response draining stays active through Locked - // and RelWait; new-issue forwarding only happens in steady-state - // Locked, never while a switch is pending. + // Draining runs through Locked/RelWait; new-issue forwarding is steady-state-Locked only. spatz_rsp_ready_o = acc_snitch_pready_i[owner_id_i]; acc_snitch_prsp_o[owner_id_i] = spatz_rsp_i; acc_snitch_pvalid_o[owner_id_i] = spatz_rsp_valid_i; @@ -171,4 +164,81 @@ module acc_mux #( end end +`ifndef TARGET_SYNTHESIS + // Debug-only: verbose per-event log (+acc_mux_verbose plusarg) and a + // stuck-transaction watchdog. Added to debug the FPU-enabled dual-CC + // boot hang; temporary instrumentation, safe to strip once resolved. + bit acc_mux_verbose = 1'b0; + initial begin + // verilog_lint: waive plusarg-assignment + acc_mux_verbose = $test$plusargs("acc_mux_verbose"); + end + + logic spatz_rsp_valid_q; + `FF(spatz_rsp_valid_q, spatz_rsp_valid_i, 1'b0, clk_i, rst_ni) + + always_ff @(posedge clk_i) begin + if (rst_ni && acc_mux_verbose) begin + if (idle_req_o && idle_gnt_i) begin + $display({"[ACC-MUX %0t %m] IDLE-GRANT host=%0d accept=%0b writeback=%0b ", + "loadstore=%0b isfloat=%0b"}, + $time, idle_winner, spatz_issue_rsp_i.accept, spatz_issue_rsp_i.writeback, + spatz_issue_rsp_i.loadstore, spatz_issue_rsp_i.isfloat); + end + if (route_fifo_push) begin + $display("[ACC-MUX %0t %m] ROUTE-FIFO PUSH host=%0d", $time, idle_winner); + end + if (route_fifo_pop) begin + $display("[ACC-MUX %0t %m] ROUTE-FIFO POP host=%0d", $time, route_fifo_route_q); + end + if (req_fire_any) begin + $display({"[ACC-MUX %0t %m] REQ-FIRE owner_active=%0b locked=%0b owner=%0d ", + "writeback=%0b"}, + $time, owner_active_i, locked_i, owner_id_i, spatz_issue_rsp_i.writeback); + end + if (rsp_fire_o) begin + $display("[ACC-MUX %0t %m] RSP-FIRE owner_active=%0b route_host=%0d", + $time, owner_active_i, route_fifo_route_q); + end + // Edge-triggered: which side of the p-channel handshake (valid vs. + // ready) is actually stuck once route_fifo has something owed. + if (spatz_rsp_valid_i != spatz_rsp_valid_q) begin + $display({"[ACC-MUX %0t %m] SPATZ-RSP-VALID -> %0b route_fifo_empty=%0b ", + "route_host=%0d owner_active=%0b spatz_rsp_ready=%0b"}, + $time, spatz_rsp_valid_i, route_fifo_empty, route_fifo_route_q, + owner_active_i, spatz_rsp_ready_o); + end + end + end + + // Watchdog: warn if a round-robin (Idle-mode) transaction has been + // accepted by Spatz but its response hasn't drained (route_fifo stuck + // non-empty) for more than AccMuxWdogPs, re-warning every AccMuxWdogPs + // while still stuck. + localparam longint unsigned AccMuxWdogPs = 5_000_000; + logic [63:0] acc_mux_last_progress_q, acc_mux_last_warn_q; + + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) begin + acc_mux_last_progress_q <= '0; + acc_mux_last_warn_q <= '0; + end else begin + if (route_fifo_empty || route_fifo_push || route_fifo_pop) begin + acc_mux_last_progress_q <= 64'($time); + acc_mux_last_warn_q <= 64'($time); + end + if (!route_fifo_empty && + (64'($time) - acc_mux_last_progress_q) > AccMuxWdogPs && + (64'($time) - acc_mux_last_warn_q) > AccMuxWdogPs) begin + acc_mux_last_warn_q <= 64'($time); + $display({"[%0t] [ACC-MUX %m] STUCK: route_fifo non-empty, owed host=%0d ", + "locked=%0b waiting=%0b owner_active=%0b idle_req_i=%0b ", + "spatz_rsp_valid=%0b spatz_rsp_ready=%0b"}, + $time, route_fifo_route_q, locked_i, waiting_i, owner_active_i, + idle_req_i, spatz_rsp_valid_i, spatz_rsp_ready_o); + end + end + end +`endif // TARGET_SYNTHESIS + endmodule diff --git a/hardware/src/cachepool_spatz_lock.sv b/hardware/src/cachepool_spatz_lock.sv index 623eef9d..c3cd96a1 100644 --- a/hardware/src/cachepool_spatz_lock.sv +++ b/hardware/src/cachepool_spatz_lock.sv @@ -7,23 +7,14 @@ `include "common_cells/assertions.svh" `include "common_cells/registers.svh" -/// Spatz ownership lock/switch for a dual-Snitch cachepool_cc_dual. -/// Host 0 is the implicit owner while Free. Two dedicated addresses are -/// intercepted: a load to ACQUIRE attempts to acquire, a load to RELEASE -/// attempts to release -- both always complete immediately (q_ready is -/// never withheld), and the loaded word encodes the outcome (fail/ -/// success/success-wait) plus current owner/locked status, so software is -/// never blocked in hardware and can always retry or bail out on its own. -/// Pure arbiter: does not route the acc/data traffic itself (that lives in -/// acc_mux.sv/cachepool_cc_dual.sv) -- only decides ownership and exposes -/// owner_id_o/locked_o/waiting_o/owner_active_o for the CC to apply, and -/// counts real Spatz acc handshakes (via req_fire_i/rsp_fire_i, fed back -/// by acc_mux) to gate a switch until fully drained. Separately tracks -/// outstanding vle/vse ops (invisible to req_fire_i/rsp_fire_i, since they -/// never produce an acc_rsp_t writeback) via Spatz's own -/// spatz_mem_finished_i/spatz_st_rsp_done_i. -/// Also hosts the pass-through memory-path register cuts (moved here from -/// the per-core spill registers in cachepool_cc.sv). +/// Spatz ownership lock/switch for a dual-Snitch cachepool_cc_dual. Two +/// dedicated addresses (ACQUIRE/RELEASE) are intercepted as loads that +/// always complete immediately, encoding the outcome and current +/// owner/locked status in the returned word (see note.md). Pure arbiter: +/// does not route acc/data traffic itself (see acc_mux.sv); only decides +/// ownership and counts real Spatz acc handshakes to gate a switch until +/// fully drained. Also hosts the pass-through memory-path register cuts +/// (moved here from the per-core spill registers in cachepool_cc.sv). module cachepool_spatz_lock import cachepool_peripheral_reg_pkg::*; #( @@ -52,9 +43,7 @@ module cachepool_spatz_lock input logic req_fire_i, input logic rsp_fire_i, - // Spatz LSU drain tracking: vle/vse never produce an acc_rsp_t completion - // (writeback=0), so they are invisible to req_fire_i/rsp_fire_i above and - // need their own outstanding count, gated by Spatz's own signals. + // Spatz LSU drain tracking: vle/vse have no acc_rsp_t completion, so they need their own count. input logic spatz_lsu_issue_fire_i, input logic [1:0] spatz_mem_finished_i, input logic spatz_st_rsp_done_i, @@ -63,9 +52,7 @@ module cachepool_spatz_lock output logic owner_id_o, output logic locked_o, output logic waiting_o, - // True while the exclusive owner path may still have in-flight traffic to - // drain (Locked and RelWait); acc_mux uses this to keep response draining - // active through RelWait even though new-issue forwarding must stop. + // True during Locked/RelWait; lets acc_mux keep draining through RelWait after new issues stop. output logic owner_active_o, // peripheral base address @@ -76,22 +63,15 @@ module cachepool_spatz_lock assign acquire_addr = cluster_periph_start_address_i + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_ACQUIRE_OFFSET; assign release_addr = cluster_periph_start_address_i + CACHEPOOL_PERIPHERAL_SPATZ_LOCK_RELEASE_OFFSET; - // Single lock FSM. Free/Locked are the two "real" states (host 0 is the - // implicit owner while Free); AcqWait/RelWait are drain-wait sub-states - // on the way to an actual ownership switch, resolved autonomously once - // drain_done -- no response is owed for that internal transition, since - // the triggering request already got a SUCCESS-WAIT answer up front. - // Free -(acquire, other host)-> AcqWait -(drain_done)-> Locked (new owner) - // Locked -(release, owner) -> RelWait -(drain_done)-> Free (owner = host 0) - // Every hit always completes immediately (q_ready is never withheld); - // the response encodes an outcome (FAIL/SUCCESS/SUCCESS-WAIT) instead of - // stalling the requester. A hit during AcqWait/RelWait, from either host, - // always gets FAIL(PENDING) -- only the one already-in-flight transition - // is ever active at a time. - // - // Handshake timing: the q-channel is accepted (q_ready=1) exactly on the - // deciding cycle; the response (p_valid=1) is then held starting the - // following cycle until the host takes it via p_ready. + // Single lock FSM (host 0 is the implicit owner while Free). AcqWait/RelWait + // are drain-wait sub-states, resolved autonomously once drain_done -- no + // response is owed then, since the triggering request already got SUCCESS-WAIT. + // Free -(acquire, other host)-> AcqWait -(drain_done)-> Locked (new owner) + // Locked -(release, owner) -> RelWait -(drain_done)-> Free (owner = host 0) + // Every hit completes immediately with an outcome (FAIL/SUCCESS/SUCCESS-WAIT) + // instead of stalling; a hit during AcqWait/RelWait always gets FAIL(PENDING). + // Handshake timing: q_ready fires on the deciding cycle; p_valid is held + // starting the next cycle until the host takes it via p_ready. typedef enum logic [1:0] { Free, AcqWait, @@ -119,8 +99,7 @@ module cachepool_spatz_lock // Outstanding acc handshakes on the owner's path; must reach 0 before a wait can complete. logic [7:0] outstanding_d, outstanding_q; - // Outstanding Spatz vle/vse ops; mirrors snitch.sv's own acc_mem_cnt_q so a - // switch can't happen while a load/store is still draining through the cache. + // Outstanding Spatz vle/vse ops; mirrors snitch.sv's own acc_mem_cnt_q. logic [7:0] lsu_outstanding_d, lsu_outstanding_q; logic drain_done; @@ -169,9 +148,7 @@ module cachepool_spatz_lock assign lock_hit[i] = acquire_hit[i] || release_hit[i]; end - // The owner's hit always wins arbitration when both hosts hit the same - // cycle; the loser simply waits one extra cycle for resp_pending_q to - // clear (both eventually get answered, this is only a tie-break). + // Owner's hit wins on a same-cycle tie; the loser just waits one cycle for resp_pending_q to clear. logic hit_any, winner; assign hit_any = lock_hit[0] || lock_hit[1]; assign winner = lock_hit[owner_q] ? owner_q : ~owner_q; @@ -279,7 +256,8 @@ module cachepool_spatz_lock resp_owner_d = owner_q; resp_locked_d = 1'b0; end - end else begin // release_hit[winner] + end else begin + // release_hit[winner] if (winner == owner_q) begin // host 0 "self release": nothing to release, immediate no-op grant resp_outcome_d = OutcomeSuccess; @@ -296,8 +274,7 @@ module cachepool_spatz_lock end end - // Only the owner's acquire (no-op) or release is meaningful; a non-owner's - // acquire/release always fails outright (no queueing for a future handoff). + // Non-owner acquire/release always fails outright, no queueing for a future handoff. Locked: begin if (hit_any && !resp_pending_q) begin in_rsp_o[winner].q_ready = 1'b1; @@ -310,7 +287,8 @@ module cachepool_spatz_lock resp_reason_d = ReasonBusy; resp_owner_d = owner_q; resp_locked_d = 1'b1; - end else begin // release_hit[winner] + end else begin + // release_hit[winner] if (winner == owner_q) begin if (drain_done) begin lock_d = Free; @@ -335,9 +313,7 @@ module cachepool_spatz_lock end end - // Drain-wait sub-states: any hit (either host, either op) always fails with - // PENDING -- the one already-registered transition (active_q) resolves on - // its own once drain_done, with no response owed for that completion. + // Any hit here fails with PENDING; active_q's transition resolves on its own, no response owed. AcqWait: begin if (hit_any && !resp_pending_q) begin in_rsp_o[winner].q_ready = 1'b1; @@ -389,4 +365,67 @@ module cachepool_spatz_lock `FF(resp_owner_q, resp_owner_d, 1'b0, clk_i, rst_ni) `FF(resp_locked_q, resp_locked_d, 1'b0, clk_i, rst_ni) +`ifndef TARGET_SYNTHESIS + // Debug-only: verbose state-transition/response log (+spatz_lock_verbose + // plusarg) and a stuck-in-wait watchdog. Added to debug the FPU-enabled + // dual-CC boot hang; temporary instrumentation, safe to strip once resolved. + bit spatz_lock_verbose = 1'b0; + initial begin + // verilog_lint: waive plusarg-assignment + spatz_lock_verbose = $test$plusargs("spatz_lock_verbose"); + end + + always_ff @(posedge clk_i) begin + if (rst_ni && spatz_lock_verbose) begin + if (lock_d != lock_q) begin + $display({"[SPATZ-LOCK %0t %m] STATE %s -> %s owner_d=%0d active_d=%0d ", + "outstanding_q=%0d lsu_outstanding_q=%0d"}, + $time, lock_q.name(), lock_d.name(), owner_d, active_d, + outstanding_q, lsu_outstanding_q); + end + if (hit_any && !resp_pending_q) begin + $display({"[SPATZ-LOCK %0t %m] HIT host=%0d acquire=%0b release=%0b ", + "state=%s outstanding_q=%0d lsu_outstanding_q=%0d ", + "spatz_st_rsp_done=%0b"}, + $time, winner, acquire_hit[winner], release_hit[winner], + lock_q.name(), outstanding_q, lsu_outstanding_q, spatz_st_rsp_done_i); + end + if (resp_pending_d && !resp_pending_q) begin + $display({"[SPATZ-LOCK %0t %m] RESP host=%0d outcome=%s reason=%s owner=%0d ", + "locked=%0b"}, + $time, resp_host_d, resp_outcome_d.name(), resp_reason_d.name(), + resp_owner_d, resp_locked_d); + end + end + end + + // Watchdog: warn if the FSM has been stuck in AcqWait/RelWait (waiting on + // drain_done) for more than SpatzLockWdogPs, re-warning every + // SpatzLockWdogPs while still stuck. + localparam longint unsigned SpatzLockWdogPs = 5_000_000; + logic [63:0] spatz_lock_last_progress_q, spatz_lock_last_warn_q; + + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) begin + spatz_lock_last_progress_q <= '0; + spatz_lock_last_warn_q <= '0; + end else begin + if (lock_q != AcqWait && lock_q != RelWait) begin + spatz_lock_last_progress_q <= 64'($time); + spatz_lock_last_warn_q <= 64'($time); + end + if ((lock_q == AcqWait || lock_q == RelWait) && + (64'($time) - spatz_lock_last_progress_q) > SpatzLockWdogPs && + (64'($time) - spatz_lock_last_warn_q) > SpatzLockWdogPs) begin + spatz_lock_last_warn_q <= 64'($time); + $display({"[%0t] [SPATZ-LOCK %m] STUCK: state=%s active_q=%0d owner_q=%0d ", + "outstanding_q=%0d lsu_outstanding_q=%0d spatz_st_rsp_done=%0b ", + "drain_done=%0b"}, + $time, lock_q.name(), active_q, owner_q, outstanding_q, + lsu_outstanding_q, spatz_st_rsp_done_i, drain_done); + end + end + end +`endif // TARGET_SYNTHESIS + endmodule From 5b241592d4c6da95531d5141bda4fc3e71758577 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Wed, 26 Aug 2026 13:18:03 +0200 Subject: [PATCH 15/17] [SRC] Fix a bug in loading instruction handling for dual-scalar core --- hardware/src/acc_mux.sv | 110 +++++++++++++++++---------- hardware/src/cachepool_cc_dual.sv | 72 +++++++++--------- hardware/src/cachepool_spatz_lock.sv | 5 +- 3 files changed, 110 insertions(+), 77 deletions(-) diff --git a/hardware/src/acc_mux.sv b/hardware/src/acc_mux.sv index 32d8463b..48e23ba1 100644 --- a/hardware/src/acc_mux.sv +++ b/hardware/src/acc_mux.sv @@ -10,11 +10,14 @@ /// Arbitrates 2 Snitch acc interfaces onto 1 shared Spatz acc interface for /// cachepool_cc_dual, gated by cachepool_spatz_lock's owner/locked/waiting /// state. Locked: only the owner passes through, real access, no arbitration. -/// Idle: both hosts get real round-robin access, one request outstanding at +/// Free: both hosts get real round-robin access, one request outstanding at /// a time -- a FIFO tracks which host is owed the in-flight response. /// Response draining stays active through owner_active_i (Locked and /// RelWait); new-issue forwarding is gated separately on locked_i alone, so /// a response already in flight when release is requested still has a drain path. +/// Also demuxes Spatz's LSU-consistency signals (mem_finished/mem_str_finished/ +/// st_rsp_done) to the issuing host, attributed by current-cycle owner so a +/// same-cycle issue+finish (e.g. stores) is never misrouted to a stale owner. module acc_mux #( parameter type acc_issue_req_t = logic, parameter type acc_issue_rsp_t = logic, @@ -46,12 +49,20 @@ module acc_mux #( input logic spatz_rsp_valid_i, output logic spatz_rsp_ready_o, - // Drain feed for the lock's outstanding counter (owner and Idle-mode paths both count). + // Spatz's raw LSU-consistency signals, demuxed to the per-host outputs below. + input logic [1:0] spatz_mem_finished_i, + input logic [1:0] spatz_mem_str_finished_i, + input logic spatz_st_rsp_done_i, + + // Drain feed for the lock's outstanding counter (owner and Free-mode paths both count). output logic req_fire_o, output logic rsp_fire_o, - // Host most recently issued to Spatz; gates spatz_mem_finished etc. in cachepool_cc_dual.sv. - output logic last_req_host_o + // Per-host demuxed copies of the signals above, attributed to the actual current-cycle + // owner (not a registered value that can go stale by one cycle on same-cycle completions). + output logic [1:0][1:0] acc_mem_finished_o, + output logic [1:0][1:0] acc_mem_str_finished_o, + output logic [1:0] acc_st_rsp_done_o ); logic req_fire_any; @@ -62,40 +73,52 @@ module acc_mux #( assign req_fire_o = req_fire_any & spatz_issue_rsp_i.writeback; assign rsp_fire_o = spatz_rsp_valid_i & spatz_rsp_ready_o; - // Idle-mode round-robin arbitration, offered only while nothing is already outstanding. - logic idle_gnt_i, idle_req_o; - logic idle_winner; - logic [1:0] idle_req_i; - acc_issue_req_t idle_data_o; - logic [1:0] idle_gnt_o; + // Free-mode round-robin arbitration, offered only while nothing is already outstanding. + logic free_gnt_i, free_req_o; + logic free_winner; + logic [1:0] free_req_i; + acc_issue_req_t free_data_o; + logic [1:0] free_gnt_o; logic route_fifo_empty, route_fifo_push, route_fifo_pop, route_fifo_route_q; logic route_fifo_full; - assign idle_req_i = (!locked_i && !waiting_i && route_fifo_empty) ? acc_snitch_qvalid_i : 2'b00; - assign idle_gnt_i = spatz_issue_ready_i; + // Holds off the next Free-mode grant until the current LSU op has actually + // drained (not just accepted), so last_req_host_q can't go stale mid-drain. + logic lsu_busy_d, lsu_busy_q; + + always_comb begin + lsu_busy_d = lsu_busy_q; + if (req_fire_any && spatz_issue_rsp_i.loadstore) lsu_busy_d = 1'b1; + if (spatz_mem_finished_i[0] || spatz_mem_finished_i[1]) lsu_busy_d = 1'b0; + end + + `FF(lsu_busy_q, lsu_busy_d, 1'b0, clk_i, rst_ni) + + assign free_req_i = (!locked_i && !waiting_i && route_fifo_empty && !lsu_busy_q) ? acc_snitch_qvalid_i : 2'b00; + assign free_gnt_i = spatz_issue_ready_i; rr_arb_tree #( .NumIn (2 ), .DataType (acc_issue_req_t), .AxiVldRdy (1'b1 ), .LockIn (1'b1 ) - ) i_idle_arb ( + ) i_free_arb ( .clk_i, .rst_ni, .flush_i (1'b0 ), .rr_i ('0 ), - .req_i (idle_req_i ), - .gnt_o (idle_gnt_o ), + .req_i (free_req_i ), + .gnt_o (free_gnt_o ), .data_i (acc_snitch_req_i), - .req_o (idle_req_o ), - .gnt_i (idle_gnt_i ), - .data_o (idle_data_o ), - .idx_o (idle_winner ) + .req_o (free_req_o ), + .gnt_i (free_gnt_i ), + .data_o (free_data_o ), + .idx_o (free_winner ) ); // Only track requests with a real completion coming: non-writeback ops never produce a later acc_rsp_t. - assign route_fifo_push = idle_req_o && idle_gnt_i && spatz_issue_rsp_i.writeback; + assign route_fifo_push = free_req_o && free_gnt_i && spatz_issue_rsp_i.writeback; assign route_fifo_pop = rsp_fire_o && !owner_active_i; // Depth 2 for safety margin; only 1 entry is ever pushed at a time. @@ -110,21 +133,26 @@ module acc_mux #( .full_o (route_fifo_full ), .empty_o (route_fifo_empty ), .usage_o ( ), - .data_i (idle_winner ), + .data_i (free_winner ), .push_i (route_fifo_push ), .data_o (route_fifo_route_q), .pop_i (route_fifo_pop ) ); - logic last_req_host_d, last_req_host_q; + // Current owner for this cycle's LSU-consistency demux: the just-issued host when a + // grant is firing right now (covers same-cycle issue+finish, e.g. stores), else the + // latched host from whichever grant is still draining (covers delayed completions). + logic last_req_host_q; + logic cur_owner; + assign cur_owner = req_fire_any ? (locked_i ? owner_id_i : free_winner) : last_req_host_q; - always_comb begin - last_req_host_d = last_req_host_q; - if (req_fire_any) last_req_host_d = locked_i ? owner_id_i : idle_winner; - end + `FF(last_req_host_q, cur_owner, 1'b0, clk_i, rst_ni) - `FF(last_req_host_q, last_req_host_d, 1'b0, clk_i, rst_ni) - assign last_req_host_o = last_req_host_q; + for (genvar h = 0; h < 2; h++) begin : gen_lsu_consistency_demux + assign acc_mem_finished_o[h] = (cur_owner == h[0]) ? spatz_mem_finished_i : '0; + assign acc_mem_str_finished_o[h] = (cur_owner == h[0]) ? spatz_mem_str_finished_i : '0; + assign acc_st_rsp_done_o[h] = (cur_owner == h[0]) ? spatz_st_rsp_done_i : 1'b1; + end always_comb begin spatz_issue_req_o = acc_issue_req_t'('0); @@ -156,10 +184,10 @@ module acc_mux #( acc_snitch_prsp_o[route_fifo_route_q] = spatz_rsp_i; acc_snitch_pvalid_o[route_fifo_route_q] = spatz_rsp_valid_i; end else if (!waiting_i) begin - spatz_issue_req_o = idle_data_o; - spatz_issue_valid_o = idle_req_o; - acc_snitch_qready_o[idle_winner] = idle_req_o && spatz_issue_ready_i; - acc_snitch_rsp_o[idle_winner] = spatz_issue_rsp_i; + spatz_issue_req_o = free_data_o; + spatz_issue_valid_o = free_req_o; + acc_snitch_qready_o[free_winner] = free_req_o && spatz_issue_ready_i; + acc_snitch_rsp_o[free_winner] = spatz_issue_rsp_i; end end end @@ -179,14 +207,14 @@ module acc_mux #( always_ff @(posedge clk_i) begin if (rst_ni && acc_mux_verbose) begin - if (idle_req_o && idle_gnt_i) begin - $display({"[ACC-MUX %0t %m] IDLE-GRANT host=%0d accept=%0b writeback=%0b ", + if (free_req_o && free_gnt_i) begin + $display({"[ACC-MUX %0t %m] FREE-GRANT host=%0d accept=%0b writeback=%0b ", "loadstore=%0b isfloat=%0b"}, - $time, idle_winner, spatz_issue_rsp_i.accept, spatz_issue_rsp_i.writeback, + $time, free_winner, spatz_issue_rsp_i.accept, spatz_issue_rsp_i.writeback, spatz_issue_rsp_i.loadstore, spatz_issue_rsp_i.isfloat); end if (route_fifo_push) begin - $display("[ACC-MUX %0t %m] ROUTE-FIFO PUSH host=%0d", $time, idle_winner); + $display("[ACC-MUX %0t %m] ROUTE-FIFO PUSH host=%0d", $time, free_winner); end if (route_fifo_pop) begin $display("[ACC-MUX %0t %m] ROUTE-FIFO POP host=%0d", $time, route_fifo_route_q); @@ -200,6 +228,10 @@ module acc_mux #( $display("[ACC-MUX %0t %m] RSP-FIRE owner_active=%0b route_host=%0d", $time, owner_active_i, route_fifo_route_q); end + if (spatz_mem_finished_i[0] || spatz_mem_finished_i[1]) begin + $display("[ACC-MUX %0t %m] LSU-FINISH cur_owner=%0d mem_finished=%0b", + $time, cur_owner, spatz_mem_finished_i); + end // Edge-triggered: which side of the p-channel handshake (valid vs. // ready) is actually stuck once route_fifo has something owed. if (spatz_rsp_valid_i != spatz_rsp_valid_q) begin @@ -211,7 +243,7 @@ module acc_mux #( end end - // Watchdog: warn if a round-robin (Idle-mode) transaction has been + // Watchdog: warn if a round-robin (Free-mode) transaction has been // accepted by Spatz but its response hasn't drained (route_fifo stuck // non-empty) for more than AccMuxWdogPs, re-warning every AccMuxWdogPs // while still stuck. @@ -232,10 +264,10 @@ module acc_mux #( (64'($time) - acc_mux_last_warn_q) > AccMuxWdogPs) begin acc_mux_last_warn_q <= 64'($time); $display({"[%0t] [ACC-MUX %m] STUCK: route_fifo non-empty, owed host=%0d ", - "locked=%0b waiting=%0b owner_active=%0b idle_req_i=%0b ", + "locked=%0b waiting=%0b owner_active=%0b free_req_i=%0b ", "spatz_rsp_valid=%0b spatz_rsp_ready=%0b"}, $time, route_fifo_route_q, locked_i, waiting_i, owner_active_i, - idle_req_i, spatz_rsp_valid_i, spatz_rsp_ready_o); + free_req_i, spatz_rsp_valid_i, spatz_rsp_ready_o); end end end diff --git a/hardware/src/cachepool_cc_dual.sv b/hardware/src/cachepool_cc_dual.sv index ade2d2e6..c306d1d7 100644 --- a/hardware/src/cachepool_cc_dual.sv +++ b/hardware/src/cachepool_cc_dual.sv @@ -126,7 +126,6 @@ module cachepool_cc_dual logic owner_id, locked, waiting, owner_active; logic req_fire, rsp_fire; - logic last_req_host; assign owner_id_o = owner_id; // Raw per-host Snitch data port, and its acc issue/response bundle. @@ -143,20 +142,14 @@ module cachepool_cc_dual fpnew_pkg::fmt_mode_t [1:0] fpu_fmt_mode_h; fpnew_pkg::status_t fpu_status; - // Spatz memory-consistency signals, forwarded only to last_req_host (not owner_id, which is stale in Idle mode). + // Spatz's raw LSU-consistency signals; acc_mux demuxes these to the issuing host. logic [1:0] spatz_mem_finished; logic [1:0] spatz_mem_str_finished; logic spatz_st_rsp_done; - logic [1:0][1:0] spatz_mem_finished_h; - logic [1:0][1:0] spatz_mem_str_finished_h; - logic [1:0] spatz_st_rsp_done_h; - - for (genvar h = 0; h < 2; h++) begin : gen_spatz_mem_consistency_gate - assign spatz_mem_finished_h[h] = (last_req_host == h[0]) ? spatz_mem_finished : '0; - assign spatz_mem_str_finished_h[h] = (last_req_host == h[0]) ? spatz_mem_str_finished : '0; - assign spatz_st_rsp_done_h[h] = (last_req_host == h[0]) ? spatz_st_rsp_done : 1'b1; - end + logic [1:0][1:0] acc_mem_finished_h; + logic [1:0][1:0] acc_mem_str_finished_h; + logic [1:0] acc_st_rsp_done_h; for (genvar h = 0; h < 2; h++) begin : gen_snitch snitch #( @@ -206,9 +199,9 @@ module cachepool_cc_dual .acc_prsp_i (acc_snitch_prsp[h] ), .acc_pvalid_i (acc_snitch_pvalid[h] ), .acc_pready_o (acc_snitch_pready[h] ), - .acc_mem_finished_i (spatz_mem_finished_h[h] ), - .acc_mem_str_finished_i(spatz_mem_str_finished_h[h] ), - .acc_st_rsp_done_i (spatz_st_rsp_done_h[h] ), + .acc_mem_finished_i (acc_mem_finished_h[h] ), + .acc_mem_str_finished_i(acc_mem_str_finished_h[h] ), + .acc_st_rsp_done_i (acc_st_rsp_done_h[h] ), .data_req_o (snitch_dreq_d[h] ), .data_rsp_i (snitch_drsp_d[h] ), .ptw_valid_o (hive_req_o[h].ptw_valid ), @@ -282,29 +275,34 @@ module cachepool_cc_dual .acc_issue_rsp_t (acc_issue_rsp_t), .acc_rsp_t (acc_rsp_t ) ) i_acc_mux ( - .clk_i (clk_i ), - .rst_ni (rst_ni ), - .owner_id_i (owner_id ), - .locked_i (locked ), - .waiting_i (waiting ), - .owner_active_i (owner_active ), - .acc_snitch_req_i (acc_snitch_req ), - .acc_snitch_rsp_o (acc_snitch_rsp ), - .acc_snitch_qvalid_i (acc_snitch_qvalid ), - .acc_snitch_qready_o (acc_snitch_qready ), - .acc_snitch_prsp_o (acc_snitch_prsp ), - .acc_snitch_pvalid_o (acc_snitch_pvalid ), - .acc_snitch_pready_i (acc_snitch_pready ), - .spatz_issue_req_o (spatz_issue_req ), - .spatz_issue_rsp_i (spatz_issue_rsp ), - .spatz_issue_valid_o (spatz_issue_valid ), - .spatz_issue_ready_i (spatz_issue_ready ), - .spatz_rsp_i (spatz_rsp ), - .spatz_rsp_valid_i (spatz_rsp_valid ), - .spatz_rsp_ready_o (spatz_rsp_ready ), - .req_fire_o (req_fire ), - .rsp_fire_o (rsp_fire ), - .last_req_host_o (last_req_host ) + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .owner_id_i (owner_id ), + .locked_i (locked ), + .waiting_i (waiting ), + .owner_active_i (owner_active ), + .acc_snitch_req_i (acc_snitch_req ), + .acc_snitch_rsp_o (acc_snitch_rsp ), + .acc_snitch_qvalid_i (acc_snitch_qvalid ), + .acc_snitch_qready_o (acc_snitch_qready ), + .acc_snitch_prsp_o (acc_snitch_prsp ), + .acc_snitch_pvalid_o (acc_snitch_pvalid ), + .acc_snitch_pready_i (acc_snitch_pready ), + .spatz_issue_req_o (spatz_issue_req ), + .spatz_issue_rsp_i (spatz_issue_rsp ), + .spatz_issue_valid_o (spatz_issue_valid ), + .spatz_issue_ready_i (spatz_issue_ready ), + .spatz_rsp_i (spatz_rsp ), + .spatz_rsp_valid_i (spatz_rsp_valid ), + .spatz_rsp_ready_o (spatz_rsp_ready ), + .spatz_mem_finished_i (spatz_mem_finished ), + .spatz_mem_str_finished_i(spatz_mem_str_finished ), + .spatz_st_rsp_done_i (spatz_st_rsp_done ), + .req_fire_o (req_fire ), + .rsp_fire_o (rsp_fire ), + .acc_mem_finished_o (acc_mem_finished_h ), + .acc_mem_str_finished_o (acc_mem_str_finished_h ), + .acc_st_rsp_done_o (acc_st_rsp_done_h ) ); // FPU rounding-mode/format CSRs: Spatz has one FPU, so it only ever sees the diff --git a/hardware/src/cachepool_spatz_lock.sv b/hardware/src/cachepool_spatz_lock.sv index c3cd96a1..3899f405 100644 --- a/hardware/src/cachepool_spatz_lock.sv +++ b/hardware/src/cachepool_spatz_lock.sv @@ -111,8 +111,11 @@ module cachepool_spatz_lock assign drain_done = (outstanding_q == '0) && (lsu_outstanding_q == '0) && spatz_st_rsp_done_i; `ASSERT(NoOutstandingUnderflow, rsp_fire_i |-> (outstanding_q != '0)) + // Checked against the net balance for this cycle, not the pre-update register alone: + // a scalar FP store's finish can land in the same cycle as its own issue. `ASSERT(NoLsuOutstandingUnderflow, - (spatz_mem_finished_i[0] || spatz_mem_finished_i[1]) |-> (lsu_outstanding_q != '0)) + (spatz_mem_finished_i[0] || spatz_mem_finished_i[1]) |-> + (lsu_outstanding_q + spatz_lsu_issue_fire_i >= (spatz_mem_finished_i[0] + spatz_mem_finished_i[1]))) always_comb begin outstanding_d = outstanding_q; From 2c8b4f5499e8a16b143977f88578a8c8652484a9 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Thu, 27 Aug 2026 10:30:09 +0200 Subject: [PATCH 16/17] [SW] Adapt the kernels for dual scalar core configuration. --- README.md | 2 ++ software/kernels/fp/fdotp-32b/main.c | 49 +++++++++++++------------- software/kernels/fp/fft-32b/main.c | 23 ++++++------ software/kernels/fp/fmatmul-32b/main.c | 39 ++++++++++---------- software/kernels/fp/gemv-opt/main.c | 33 +++++++++-------- software/kernels/fp/gemv/main.c | 31 ++++++++-------- software/snRuntime/include/snrt.h | 6 ++++ software/snRuntime/src/team.c | 24 +++++++++++++ 8 files changed, 125 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 38e62419..dd4b82eb 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,8 @@ Set `num_scalar_per_core=2` (e.g. `cachepool_dual_4g`, see [Configurations](#con Ownership is arbitrated by `cachepool_spatz_lock.sv`, a small hardware FSM (`Free`/`Locked`/`AcqWait`/`RelWait`) intercepting two dedicated peripheral addresses. It never blocks a core's pipeline: every acquire/release attempt is a plain **load** that always completes immediately, returning an outcome (`FAIL`/`SUCCESS`/`SUCCESS-WAIT`) instead of stalling the hart — so a hart that doesn't get the lock can retry, back off, or do other work instead of hanging. See `software/snRuntime/README.md` for the software API and `note.md` for the full state-machine design. +**Free vs. Locked performance**: while unlocked (`Free` state), `acc_mux.sv` shares Spatz between both harts via real round-robin arbitration, but every loadstore op must fully drain (real memory completion, not just issue-accept) before the next grant is offered — this holds even for a single hart with no contention from its partner, since the mux has no way to know a sequence of ops belongs to an uncontended hart until it tries to arbitrate again. Loadstore-heavy code therefore runs serialized on real memory latency in `Free` mode. Acquiring the lock (`Locked` state) removes this: the owner's issue path bypasses round-robin arbitration entirely and pipelines back-to-back LSU ops at full throughput. A kernel that does sustained vector/FP loadstore work should hold the lock around that section even if its partner hart never contends for Spatz at all. + Two per-role partial-barrier helpers, `snrt_cluster_host0_barrier()`/`snrt_cluster_host1_barrier()` (`snrt.h`), let a kernel synchronize only the pair's default owners (or only their partners) without hand-writing a participant mask; both degrade to an ordinary full barrier on a single-scalar-per-CC build, so kernels using them don't need a config-specific `#if`. ## Stack diff --git a/software/kernels/fp/fdotp-32b/main.c b/software/kernels/fp/fdotp-32b/main.c index 5492ff31..1f10da72 100644 --- a/software/kernels/fp/fdotp-32b/main.c +++ b/software/kernels/fp/fdotp-32b/main.c @@ -24,8 +24,9 @@ #include "kernel/fdotp.c" int main() { - const uint32_t num_cores = snrt_cluster_core_num(); - const uint32_t cid = snrt_cluster_core_idx(); + const uint32_t num_cores = snrt_cluster_vpu_num(); + const uint32_t cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); const uint32_t measure_iter = 3; @@ -54,7 +55,7 @@ int main() { } else if (lmul_max == 1) { lmul = 1; } else { - if (cid == 0) { + if (is_primary && cid == 0) { printf("FATAL: Problem size too small!\n"); return -2; } @@ -67,7 +68,7 @@ int main() { if ((elem_per_round * num_cores) < (l2_block_elem * l2_channel)) { - if (cid == 0) { + if (is_primary && cid == 0) { printf("Warning: Current scheme cannot utilize all bandwidth!\n"); } } @@ -93,14 +94,14 @@ int main() { float *a_int = dotp_A_dram + cid * elem_per_round; float *b_int = dotp_B_dram + cid * elem_per_round; - if (cid == 0) { + if (is_primary && cid == 0) { printf("lmul:%u, elem:%u, offs:%u, iter:%u\n", lmul, elem_per_round, elem_jump_per_round, rounds); } for (int iter = 0; iter < measure_iter; iter ++) { // Start dump - if (cid == 0) + if (is_primary && cid == 0) start_kernel(); snrt_cluster_hw_barrier(); @@ -111,18 +112,20 @@ int main() { // Calculate dotp float acc; - if (lmul >= 8) - acc = fdotp_v32b_lmul8(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else if (lmul >= 4) - acc = fdotp_v32b_lmul4(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else if (lmul >= 2) - acc = fdotp_v32b_lmul2(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else if (lmul >= 1) - acc = fdotp_v32b_lmul1(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); - else - return -3; + if (is_primary) { + if (lmul >= 8) + acc = fdotp_v32b_lmul8(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); + else if (lmul >= 4) + acc = fdotp_v32b_lmul4(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); + else if (lmul >= 2) + acc = fdotp_v32b_lmul2(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); + else if (lmul >= 1) + acc = fdotp_v32b_lmul1(a_int, b_int, elem_jump_per_round, elem_per_round, rounds); + else + return -3; - result[cid] = acc; + result[cid] = acc; + } // Make sure spatz has finished writing snrt_fence_spatz(); @@ -131,7 +134,7 @@ int main() { snrt_cluster_hw_barrier(); // End timer and check if new best runtime - if (cid == 0) { + if (is_primary && cid == 0) { timer_tmp = benchmark_get_cycle() - timer_tmp; timer = (timer < timer_tmp) ? timer : timer_tmp; if (iter == 0) @@ -144,7 +147,7 @@ int main() { const uint32_t red_group = 4; // Level 1: lead core of each group accumulates its group - if (cid % red_group == 0) { + if (is_primary && cid % red_group == 0) { for (uint32_t i = 1; i < red_group && (cid + i) < num_cores; ++i) acc += result[cid + i]; result[cid] = acc; @@ -153,7 +156,7 @@ int main() { snrt_cluster_hw_barrier(); // Level 2: core 0 sums all group results - if (cid == 0) { + if (is_primary && cid == 0) { for (uint32_t g = red_group; g < num_cores; g += red_group) acc += result[g]; result[0] = acc; @@ -161,10 +164,8 @@ int main() { } - snrt_cluster_hw_barrier(); - // Check and display results - if (cid == 0) { + if (is_primary && cid == 0) { // The timer did not count the reduction time uint32_t performance = 1000 * 2 * dotp_l.M / timer; uint32_t perf_iter1 = 1000 * 2 * dotp_l.M / timer_iter1; @@ -181,7 +182,7 @@ int main() { performance, utilization); } - if (cid == 0) { + if (is_primary && cid == 0) { if (fp_check(result[0], dotp_result*measure_iter)) { printf("Check Failed!\n"); printf("Calc:"); diff --git a/software/kernels/fp/fft-32b/main.c b/software/kernels/fp/fft-32b/main.c index 8976a9ce..233ee261 100644 --- a/software/kernels/fp/fft-32b/main.c +++ b/software/kernels/fp/fft-32b/main.c @@ -48,9 +48,10 @@ int main() { const int measure_iter = 3; // twiddle layout: [re_p1, im_p1, re_p2, im_p2] - const uint32_t num_cores = snrt_cluster_core_num(); - const uint32_t cid = snrt_cluster_core_idx(); - + const uint32_t num_cores = snrt_cluster_vpu_num(); + const uint32_t cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); + snrt_cluster_hw_barrier(); const uint32_t NFFTpc = NFFT / active_cores; // 32-bit floating, 4 byte distance in memory @@ -89,7 +90,7 @@ int main() { uint32_t ierror = 0; for (int iter = 0; iter < measure_iter; iter++) { - if (cid == 0) { + if (is_primary && cid == 0) { start_kernel(); } @@ -97,12 +98,12 @@ int main() { snrt_cluster_hw_barrier(); // Start timer - if (cid == 0) { + if (is_primary && cid == 0) { timer_tmp = benchmark_get_cycle(); } for (uint32_t i = 0; i < log2_nfft1; i ++) { - if (cid < active_cores) { + if (is_primary && cid < active_cores) { fft_p1(src_p1, buf_p1, twi_p1, NFFT, NTWI_P1, cid, active_cores, i, len); // each round will use half the twiddle than previous round // the first round needs re/im NFFT/2 twiddles @@ -116,7 +117,7 @@ int main() { snrt_cluster_hw_barrier(); } - if (cid < active_cores) { + if (is_primary && cid < active_cores) { // Fall back into the single-core case // Each core just do a FFT on (NFFT >> stage_in_P1) data if (p2_switch) { @@ -131,7 +132,7 @@ int main() { snrt_cluster_hw_barrier(); // End timer and check if new best runtime - if (cid == 0) { + if (is_primary && cid == 0) { timer_tmp = benchmark_get_cycle() - timer_tmp; timer = (timer < timer_tmp) ? timer : timer_tmp; if (iter == 0) @@ -145,7 +146,7 @@ int main() { l1d_cluster_flush(); } - if (cid == 0) { + if (is_primary && cid == 0) { if ((iter == 0) && CHECK) { // Verify the real part for (unsigned int i = 0; i < NFFT; i++) { @@ -168,9 +169,9 @@ int main() { snrt_cluster_hw_barrier(); } - + // Display runtime - if (cid == 0) { + if (is_primary && cid == 0) { // Each stage requires: // 2 add, 2 sub, 2 mul, 2 macc/msac // in total 10 operations on NFFT/2 real and NFFT/2 im elements diff --git a/software/kernels/fp/fmatmul-32b/main.c b/software/kernels/fp/fmatmul-32b/main.c index c59db8ca..fa308a0b 100644 --- a/software/kernels/fp/fmatmul-32b/main.c +++ b/software/kernels/fp/fmatmul-32b/main.c @@ -59,9 +59,10 @@ int verify_matrix(float *matrix, const float *checksum, } int main() { - const unsigned int num_cores = snrt_cluster_core_num(); - const unsigned int num_cores_per_tile = snrt_cluster_core_per_tile(); - const unsigned int cid = snrt_cluster_core_idx(); + const unsigned int num_cores = snrt_cluster_vpu_num(); + const unsigned int num_cores_per_tile = snrt_cluster_vpu_per_tile(); + const unsigned int cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); #if MEAS_1ITER == 1 const int measure_iter = 1; @@ -83,7 +84,7 @@ int main() { // Allocate and zero the error array while the cache is still in shared mode, // so the pointer write and slot initialisation are visible to all cores. - if (cid == 0) { + if (is_primary && cid == 0) { error_arr = (int *)snrt_malloc(active_cores * sizeof(int)); for (unsigned int i = 0; i < active_cores; i++) error_arr[i] = 0; @@ -117,7 +118,7 @@ int main() { // Initialize matrices #ifdef DEBUG - if (cid == 0) { + if (is_primary && cid == 0) { printf ("a:%x\n", a); printf ("b:%x\n", b); printf ("c:%x\n", c); @@ -134,21 +135,23 @@ int main() { // Calculate matmul for (unsigned int i = 0; i < measure_iter; ++i) { // Start dump - if (cid == 0) { + if (is_primary && cid == 0) { start_kernel(); } // Start timer timer_start = benchmark_get_cycle(); - if (kernel_size == 2) { - matmul_2xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); - } else if (kernel_size == 4) { - matmul_4xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); - } else if (kernel_size == 8) { - matmul_8xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); - } else { - return -1; + if (is_primary) { + if (kernel_size == 2) { + matmul_2xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); + } else if (kernel_size == 4) { + matmul_4xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); + } else if (kernel_size == 8) { + matmul_8xVL(gemm_C_dram, gemm_A_dram, gemm_B_dram, m_start, m_end, gemm_l.K, gemm_l.N, p_start, p_end); + } else { + return -1; + } } // Wait for all cores to finish @@ -157,7 +160,7 @@ int main() { // End timer and check if new best runtime timer_end = benchmark_get_cycle(); unsigned int timer_temp = timer_end - timer_start; - if (cid == 0) { + if (is_primary && cid == 0) { if (timer_temp < timer) { timer = timer_temp; if (i == 0) @@ -167,7 +170,7 @@ int main() { } if (i == 0) { - if (cid < active_cores) { + if (is_primary && cid < active_cores) { float *check_C = gemm_C_dram + cid * (gemm_l.M / active_cores) * gemm_l.N; float *check_gold = (float *)gemm_checksum + cid * (gemm_l.M / active_cores); @@ -177,7 +180,7 @@ int main() { snrt_cluster_hw_barrier(); - if (cid == 0) { + if (is_primary && cid == 0) { if (error_arr[0] != 0) printf("Core 0 error %d\n", error_arr[0]); @@ -196,7 +199,7 @@ int main() { } // Check and display results - if (cid == 0) { + if (is_primary && cid == 0) { long unsigned int performance = 1000 * 2 * gemm_l.M * gemm_l.N * gemm_l.K / timer; long unsigned int utilization = performance / (2 * active_cores * 4); diff --git a/software/kernels/fp/gemv-opt/main.c b/software/kernels/fp/gemv-opt/main.c index 7136b597..ae1b2dc3 100644 --- a/software/kernels/fp/gemv-opt/main.c +++ b/software/kernels/fp/gemv-opt/main.c @@ -65,8 +65,9 @@ int main() { T *b; T *result; - const uint32_t num_cores = snrt_cluster_core_num(); - const uint32_t cid = snrt_cluster_core_idx(); + const uint32_t num_cores = snrt_cluster_vpu_num(); + const uint32_t cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); // How many column of data each core will work on // This will determine the vlen of the calculation @@ -90,7 +91,7 @@ int main() { } else if (lmul_max == 1) { lmul = 1; } else { - if (cid == 0) { + if (is_primary && cid == 0) { printf("FATAL: Problem size too small!\n"); } snrt_cluster_hw_barrier(); @@ -103,7 +104,7 @@ int main() { // offset bits in unit of byte (address) uint32_t offset = 31 - __builtin_clz(m_core * elem_width/8); - if (cid == 0) { + if (is_primary && cid == 0) { result_dram = (T *)snrt_malloc(gemv_l.M * sizeof(T)); } @@ -131,7 +132,7 @@ int main() { // If our block size is a 4-times multiple of l2 block size // Then we will always visiting the same L2 channel if (block_elem_core < l2_block_elem) { - if (cid == 0) { + if (is_primary && cid == 0) { printf("FATAL: Current scheme cannot utilize all bandwidth!\n"); printf("Core block size:%u, DRAM block size:%u\n", block_elem_core, l2_block_elem); } @@ -166,7 +167,7 @@ int main() { uint32_t n_core = gemv_l.N - col_shift; // first segment length uint32_t comp_size = col_shift; // second segment length - if (cid == 0) { + if (is_primary && cid == 0) { printf("lmul:%u, mcore:%u\n", lmul, m_core); } @@ -177,26 +178,28 @@ int main() { for (int i = 0; i < 3; i++) { // Start dump - if (cid == 0) { + if (is_primary && cid == 0) { start_kernel(); // Start timer timer_start = benchmark_get_cycle(); } // Calculate gemv - if (lmul == 4) { - gemv_v32b_m4(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); - } else if (lmul == 2) { - gemv_v32b_m2(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); - } else if (lmul == 1) { - gemv_v32b_m1(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); + if (is_primary) { + if (lmul == 4) { + gemv_v32b_m4(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); + } else if (lmul == 2) { + gemv_v32b_m2(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); + } else if (lmul == 1) { + gemv_v32b_m1(a_core, b_core, r_core, a_offset, b_offset, gemv_l.M, n_core, m_core, comp_size); + } } // Wait for all cores to finish snrt_cluster_hw_barrier(); - if (cid == 0) { + if (is_primary && cid == 0) { // End timer and check if new best runtime timer_end = benchmark_get_cycle(); unsigned int timer_temp = timer_end - timer_start; @@ -226,7 +229,7 @@ int main() { } // Check and display results - if (cid == 0) { + if (is_primary && cid == 0) { long unsigned int performance = 1000 * 2 * gemv_l.M * gemv_l.N / timer; long unsigned int utilization = performance / (2 * num_cores * 4 * (4 / sizeof(T))); diff --git a/software/kernels/fp/gemv/main.c b/software/kernels/fp/gemv/main.c index 1cb5b89b..36a63280 100644 --- a/software/kernels/fp/gemv/main.c +++ b/software/kernels/fp/gemv/main.c @@ -55,8 +55,9 @@ int main() { T *b; T *result; - const unsigned int num_cores = snrt_cluster_core_num(); - const unsigned int cid = snrt_cluster_core_idx(); + const unsigned int num_cores = snrt_cluster_vpu_num(); + const unsigned int cid = snrt_cluster_vpu_idx(); + const int is_primary = snrt_cluster_is_primary(); unsigned int m_core = gemv_l.M / num_cores; @@ -71,7 +72,7 @@ int main() { a = gemv_A_dram; b = gemv_B_dram; - if (cid == 0) { + if (is_primary && cid == 0) { result_dram = (T *)snrt_malloc(gemv_l.M * sizeof(T)); } @@ -96,26 +97,28 @@ int main() { for (int i = 0; i < 3; i++) { // Start dump - if (cid == 0) { + if (is_primary && cid == 0) { start_kernel(); // Start timer timer_start = benchmark_get_cycle(); } // Calculate gemv - if (sizeof(T) == 8) - // does not support 64b - return -2; - else if (sizeof(T) == 4) - gemv_v32b_m4(a_core, b, result_core, gemv_l.M, m_core, gemv_l.N); - else - gemv_v16b_m4(a_core, b, result_core, gemv_l.M, m_core, gemv_l.N); + if (is_primary) { + if (sizeof(T) == 8) + // does not support 64b + return -2; + else if (sizeof(T) == 4) + gemv_v32b_m4(a_core, b, result_core, gemv_l.M, m_core, gemv_l.N); + else + gemv_v16b_m4(a_core, b, result_core, gemv_l.M, m_core, gemv_l.N); + } // Wait for all cores to finish snrt_cluster_hw_barrier(); - if (cid == 0) { + if (is_primary && cid == 0) { // End timer and check if new best runtime timer_end = benchmark_get_cycle(); unsigned int timer_temp = timer_end - timer_start; @@ -137,7 +140,7 @@ int main() { l1d_cluster_flush(); } - if (cid == 0) { + if (is_primary && cid == 0) { if (i == 0) { for (uint32_t j = 0; j < gemv_l.M; j++) { if (fp_check(&result[j], &gemv_result[j])) { @@ -156,7 +159,7 @@ int main() { } // Check and display results - if (cid == 0) { + if (is_primary && cid == 0) { long unsigned int performance = 1000 * 2 * gemv_l.M * gemv_l.N / timer; long unsigned int utilization = performance / (2 * num_cores * 4 * (4 / sizeof(T))); diff --git a/software/snRuntime/include/snrt.h b/software/snRuntime/include/snrt.h index fb0cd952..d76c7c70 100644 --- a/software/snRuntime/include/snrt.h +++ b/software/snRuntime/include/snrt.h @@ -108,6 +108,12 @@ extern uint32_t snrt_cluster_num(); /// whether this hart is host 0 of its Core Complex pair extern int snrt_cluster_is_primary(); +/// Physical vector-unit count/index/per-tile count, halved vs. the hart-based +/// counterparts in dual-scalar configs (one Spatz per CC pair), unchanged otherwise. +extern uint32_t snrt_cluster_vpu_num(); +extern uint32_t snrt_cluster_vpu_idx(); +extern uint32_t snrt_cluster_vpu_per_tile(); + /// get pointer to barrier register extern uint32_t _snrt_barrier_reg_ptr(); diff --git a/software/snRuntime/src/team.c b/software/snRuntime/src/team.c index d5940491..0b423722 100644 --- a/software/snRuntime/src/team.c +++ b/software/snRuntime/src/team.c @@ -66,6 +66,30 @@ uint32_t snrt_cluster_core_num() { return _snrt_team_current->root->cluster_core_num; } +uint32_t snrt_cluster_vpu_num() { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + return snrt_cluster_core_num() / 2; +#else + return snrt_cluster_core_num(); +#endif +} + +uint32_t snrt_cluster_vpu_idx() { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + return snrt_cluster_core_idx() / 2; +#else + return snrt_cluster_core_idx(); +#endif +} + +uint32_t snrt_cluster_vpu_per_tile() { +#if SNRT_NUM_SCALAR_PER_CORE == 2 + return snrt_cluster_core_per_tile() / 2; +#else + return snrt_cluster_core_per_tile(); +#endif +} + uint32_t snrt_cluster_core_per_tile() { uint32_t c = _snrt_team_current->root->cluster_core_num; uint32_t t = SNRT_BOOT_TILE_COUNT; From 45b99f9cae464375e12f888b7891f1ae6d54d006 Mon Sep 17 00:00:00 2001 From: Diyou Shen Date: Thu, 27 Aug 2026 13:11:18 +0200 Subject: [PATCH 17/17] [CI] Add dual-scalar core configuration into CI and limit the thread in building DRAM due to CI server limits --- .gitlab-ci.yml | 59 +++++++++++++++++++++++++++++++++++++++++++++++++- Makefile | 4 ++-- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 04a9c647..8c52cb6b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,7 @@ variables: python3: 'python3' # Config to build and test CI_CONFIG: 'cachepool_fpu_4g' + CI_CONFIG_DUAL: 'cachepool_dual_fpu_4g' SW_PREFIX: 'test-' default: @@ -60,6 +61,33 @@ build: - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/build/lib/ - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/configs/ +# --------------------------------------------------------------------------- +# Build stage: same as `build`, for the dual-scalar-per-CC config. Only the +# kernels adapted for dual-scalar (see `test-dual`) are exercised against it. +# --------------------------------------------------------------------------- +build-dual: + stage: build + script: + - echo "Using CC=$CC" + - echo "Using CXX=$CXX" + - test -x "$CC" + - test -x "$CXX" + - source iis-env.sh + - make init + - make dram-build + - make clean generate update-floonoc bootrom vsim sw config=$CI_CONFIG_DUAL DEBUG=0 noc_profiling=0 + artifacts: + when: always + expire_in: 2h + paths: + - sim/work/ + - sim/bin/cachepool_cluster.vsim + - sim/bin/cachepool_cluster.vsim.gui + - sim/work-dpi/ + - software/build/CachePoolTests/ + - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/build/lib/ + - hardware/deps/dram_rtl_sim/dramsys_lib/DRAMSys/configs/ + # --------------------------------------------------------------------------- # Test stage: run each kernel in parallel on a separate runner. # Each job downloads the build artifacts, runs one simulation, and checks @@ -90,9 +118,38 @@ test: - python3 util/auto-benchmark/check-ci.py test_${KERNEL}.log artifacts: when: always - expire_in: 1 week + expire_in: 3 days paths: # Full simulation log - test_*.log # Performance-monitor trace files written by the simulator - sim/bin/logs/ + +# --------------------------------------------------------------------------- +# Test stage: dual-scalar-per-CC config. Limited to the kernels adapted and +# confirmed working under this config (host0-only Spatz access via +# snrt_cluster_vpu_*/snrt_cluster_is_primary); other tests are not yet +# verified for dual-scalar and are left to the default-config matrix above. +# --------------------------------------------------------------------------- +test-dual: + stage: test + needs: [build-dual] + parallel: + matrix: + - KERNEL: + - fdotp-32b_M32768 + - gemv_M512_N128_K32 + - fmatmul-32b_M64_N64_K64 + - fft-32b_M1024_N16 + script: + - mkdir -p sim/bin/logs sim/bin/logs/core sim/bin/logs/noc sim/bin/logs/others + - chmod +x sim/bin/cachepool_cluster.vsim + - BIN="${SW_PREFIX}${KERNEL}" + - sim/bin/cachepool_cluster.vsim software/build/CachePoolTests/$BIN 2>&1 | tee test_dual_${KERNEL}.log + - python3 util/auto-benchmark/check-ci.py test_dual_${KERNEL}.log + artifacts: + when: always + expire_in: 3 days + paths: + - test_dual_*.log + - sim/bin/logs/ diff --git a/Makefile b/Makefile index d8a252ee..89a9b4cd 100644 --- a/Makefile +++ b/Makefile @@ -213,7 +213,7 @@ dram-build: $(DRAMSYS_PATH)/README.md dram-config if [ ! -d "build" ]; then \ mkdir build && cd build; \ $(CMAKE) -DCMAKE_CXX_FLAGS=-fPIC -DCMAKE_C_FLAGS=-fPIC -D DRAMSYS_WITH_DRAMPOWER=ON .. ; \ - make -j; \ + make -j4; \ fi $(DRAMSYS_PATH)/README.md: dram-init @@ -242,7 +242,7 @@ dram-clean: fi dram-init: - make -C ${DRAMSYS_DIR} -j8 dramsys CXX=$(CXX) CC=$(CC) + make -C ${DRAMSYS_DIR} -j4 dramsys CXX=$(CXX) CC=$(CC) ############ # Modelsim #