From c0a7fc0f949e82ff6f191619726abeaeaba4f980 Mon Sep 17 00:00:00 2001 From: Thomas Weatherly Date: Tue, 1 Sep 2026 17:07:33 -0400 Subject: [PATCH 1/5] vm: banked L1 TLB storage as a config-selectable alternative (VX_CFG_L1_TLB_NUM_BANKS) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds VX_tlb_l1_banked behind VX_tlb_l1's parent contract: the entry array splits into power-of-two single-ported banks selected by the low VPN bits (a losing lane holds for a cycle via bank_conflict), and each bank holds one parked miss — a miss blocks only its bank while the others keep hitting; a walk in flight across a flush is discarded and re-walked. The knob defaults to 0, which elaborates the baseline multi-ported CAM + MSHR unchanged; the banked organization from #392 becomes a reachable configuration for same-base comparison (cycle A/B by config flip, and the synthesis area/timing comparison banking exists for). Kill/fault, flush, drain, and perf contracts match the baseline. Co-Authored-By: Claude Fable 5 --- VX_config.toml | 5 + hw/rtl/vm/VX_mmu.sv | 58 +++++- hw/rtl/vm/VX_tlb_l1_banked.sv | 372 ++++++++++++++++++++++++++++++++++ 3 files changed, 432 insertions(+), 3 deletions(-) create mode 100644 hw/rtl/vm/VX_tlb_l1_banked.sv diff --git a/VX_config.toml b/VX_config.toml index c7206c6ed5..184dd049bc 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -141,6 +141,11 @@ VX_CFG_NUM_VPU_BLOCKS = 1 # VM page-table format is a HW<->SW contract, moved to VX_types.toml [vm]; TLB depth stays here. VX_CFG_TLB_SIZE = 32 +# L1 TLB organization: 0 selects the baseline multi-ported CAM + MSHR +# (hit-under-miss); a power of two selects the banked storage — per-bank +# single lookup ports with one parked miss per bank (VX_tlb_l1_banked). +VX_CFG_L1_TLB_NUM_BANKS = 0 + # L1 TLB stage (per core): D-side / I-side entries and the per-instance # miss station depth (distinct outstanding VPN misses). VX_CFG_DTLB_SIZE = 16 diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index d663381245..136dbf0daf 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -145,6 +145,56 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire flush_clear; + wire [NUM_REQS-1:0] bank_conflict; + + // L1 storage organization: banked (per-bank lookup port + one parked + // miss per bank, losing lanes hold via bank_conflict) or the baseline + // multi-ported CAM + MSHR. + if (`VX_CFG_L1_TLB_NUM_BANKS != 0) begin : g_tlb_banked + VX_tlb_l1_banked #( + .NUM_REQS (NUM_REQS), + .TLB_SIZE (TLB_SIZE), + .NUM_BANKS (`VX_CFG_L1_TLB_NUM_BANKS), + .PAYLOAD_W (PAYLOAD_W), + .ID_WIDTH (ID_WIDTH) + ) tlb ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .mmu_perf (mmu_perf), + `endif + .lookup_vpn (cam_vpn), + .lookup_valid (req_valid & ~req_bypass), + .lookup_hit (cam_hit), + .lookup_ppn (cam_ppn), + .lookup_flags (cam_flags), + .bank_conflict (bank_conflict), + .access_hit (cam_access_hit), + .mshr_match (mshr_match), + .park_valid (park_valid), + .park_vpn (park_vpn), + .park_access (park_access), + .park_amo (park_amo), + .park_lane (park_lane), + .park_payload (park_payload), + .park_ready (park_ready), + .replay_valid (replay_valid), + .replay_payload(replay_payload), + .replay_ppn (replay_ppn), + .replay_level (replay_level), + .replay_flags (replay_flags), + .replay_ready (replay_ready), + .kill_valid (kill_valid), + .kill_ready (kill_ready), + .mshr_fault_valid (mshr_fault_valid), + .mshr_fault_vpn (mshr_fault_vpn), + .mshr_fault_access (mshr_fault_access), + .tlb_bus_if (tlb_bus_if), + .flush (flush_clear), + .empty (tlb_empty) + ); + end else begin : g_tlb_baseline + assign bank_conflict = '0; VX_tlb_l1 #( .NUM_REQS (NUM_REQS), .TLB_SIZE (TLB_SIZE), @@ -186,6 +236,8 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .flush (flush_clear), .empty (tlb_empty) ); + end + // --------------------------------------------------------------------- // Per-lane request category (mutually exclusive, by priority) @@ -199,9 +251,9 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( for (genvar l = 0; l < NUM_REQS; ++l) begin : g_cat assign perm_hit[l] = tlb_perm_ok(cam_flags[l], req_acc[l], req_amo[l]); assign cat_bypass[l] = req_valid[l] && req_bypass[l]; - assign cat_park[l] = req_valid[l] && !req_bypass[l] && (mshr_match[l] || !cam_hit[l]); - assign cat_hit[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && perm_hit[l]; - assign cat_pfault[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && !perm_hit[l]; + assign cat_park[l] = req_valid[l] && !req_bypass[l] && !bank_conflict[l] && (mshr_match[l] || !cam_hit[l]); + assign cat_hit[l] = req_valid[l] && !req_bypass[l] && !bank_conflict[l] && !mshr_match[l] && cam_hit[l] && perm_hit[l]; + assign cat_pfault[l] = req_valid[l] && !req_bypass[l] && !bank_conflict[l] && !mshr_match[l] && cam_hit[l] && !perm_hit[l]; end // Park arbitration: at most one lane parks a miss per cycle (lowest lane). diff --git a/hw/rtl/vm/VX_tlb_l1_banked.sv b/hw/rtl/vm/VX_tlb_l1_banked.sv new file mode 100644 index 0000000000..d4a73af226 --- /dev/null +++ b/hw/rtl/vm/VX_tlb_l1_banked.sv @@ -0,0 +1,372 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_define.vh" + +// Banked L1 TLB storage + miss station, an alternative to VX_tlb_l1 behind +// the same parent (VX_mmu) contract. The entry array is split into +// NUM_BANKS single-ported banks selected by the low VPN bits: each bank +// answers at most one lane per cycle (bank_conflict tells the parent to +// hold the other lanes), and each bank holds at most one parked miss — +// a miss blocks its bank for the walk's duration while the other banks +// keep hitting. A same-VPN request waits on its busk's in-flight walk +// (mshr_match) instead of joining a queue. Trades the baseline's +// full multi-port CAM + MSHR for per-bank ports and slots: cheaper +// lookup hardware at scale, one outstanding walk per bank. +module VX_tlb_l1_banked import VX_gpu_pkg::*, VX_tlb_pkg::*; #( + parameter NUM_REQS = DCACHE_NUM_REQS, + parameter TLB_SIZE = `VX_CFG_DTLB_SIZE, + parameter NUM_BANKS = 4, + parameter PAYLOAD_W = 1, + parameter ID_WIDTH = `CLOG2(NUM_BANKS) +) ( + input wire clk, + input wire reset, + +`ifdef PERF_ENABLE + output mmu_perf_t mmu_perf, +`endif + + // Per-lane lookup. A lane that loses its bank's port this cycle gets + // bank_conflict (parent must hold it); hit/miss is only meaningful for + // lanes with bank_conflict == 0. + input wire [NUM_REQS-1:0][TLB_VPN_WIDTH-1:0] lookup_vpn, + input wire [NUM_REQS-1:0] lookup_valid, + output wire [NUM_REQS-1:0] lookup_hit, + output wire [NUM_REQS-1:0][TLB_PPN_WIDTH-1:0] lookup_ppn, + output wire [NUM_REQS-1:0][TLB_FLAGS_WIDTH-1:0] lookup_flags, + output wire [NUM_REQS-1:0] bank_conflict, + input wire [NUM_REQS-1:0] access_hit, + output wire [NUM_REQS-1:0] mshr_match, + + // Park a miss (payload is opaque; the parent splices on replay). + input wire park_valid, + input wire [TLB_VPN_WIDTH-1:0] park_vpn, + input tlb_access_e park_access, + input wire park_amo, + input wire [`UP(`CLOG2(NUM_REQS))-1:0] park_lane, + input wire [PAYLOAD_W-1:0] park_payload, + output wire park_ready, + + // Replay a parked request once its fill lands. + output wire replay_valid, + output wire [PAYLOAD_W-1:0] replay_payload, + output wire [TLB_PPN_WIDTH-1:0] replay_ppn, + output wire [TLB_LEVEL_WIDTH-1:0] replay_level, + output wire [TLB_FLAGS_WIDTH-1:0] replay_flags, + input wire replay_ready, + + // Kill a parked request whose walk faulted. + output wire kill_valid, + input wire kill_ready, + + // Structural-fault sideband. + output wire mshr_fault_valid, + output wire [TLB_VPN_WIDTH-1:0] mshr_fault_vpn, + output tlb_access_e mshr_fault_access, + + // Miss/fill fabric to the shared walker complex (id = bank index). + VX_tlb_bus_if.master tlb_bus_if, + + input wire flush, + output wire empty +); + `STATIC_ASSERT(`IS_POW2(NUM_BANKS), ("NUM_BANKS must be a power of 2")) + `STATIC_ASSERT((TLB_SIZE % NUM_BANKS) == 0, ("NUM_BANKS must divide TLB_SIZE")) + `STATIC_ASSERT(ID_WIDTH >= `CLOG2(NUM_BANKS), ("bank index must fit the bus id")) + + localparam ENTRIES_PER_BANK = TLB_SIZE / NUM_BANKS; + localparam BANK_W = `UP(`CLOG2(NUM_BANKS)); + localparam LANE_W = `UP(`CLOG2(NUM_REQS)); + + function automatic logic [BANK_W-1:0] bank_of(input logic [BANK_W-1:0] vpn_lo); + if (NUM_BANKS == 1) bank_of = '0; + else bank_of = vpn_lo; + endfunction + + // --------------------------------------------------------------------- + // Per-bank parked-miss slot + // --------------------------------------------------------------------- + typedef enum logic [1:0] { + B_IDLE, B_WALK_REQ, B_WALK_WAIT, B_DRAIN + } bank_state_e; + + bank_state_e bk_state [NUM_BANKS]; + logic [TLB_VPN_WIDTH-1:0] bk_vpn [NUM_BANKS]; + tlb_access_e bk_access [NUM_BANKS]; + logic bk_amo [NUM_BANKS]; + logic [PAYLOAD_W-1:0] bk_payload [NUM_BANKS]; + logic bk_fault [NUM_BANKS]; + logic [TLB_PPN_WIDTH-1:0] bk_ppn [NUM_BANKS]; + logic [TLB_LEVEL_WIDTH-1:0] bk_level [NUM_BANKS]; + logic [TLB_FLAGS_WIDTH-1:0] bk_flags [NUM_BANKS]; + // A walk in flight when the flush arrived resolved against the old page + // table: drop its fill and re-walk (see the shared walker's discipline). + logic bk_stale [NUM_BANKS]; + + // --------------------------------------------------------------------- + // Bank lookup port arbitration: lowest contending lane wins the bank. + // --------------------------------------------------------------------- + wire [NUM_REQS-1:0][BANK_W-1:0] lane_bank; + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lane_bank + assign lane_bank[l] = bank_of(lookup_vpn[l][BANK_W-1:0]); + end + + wire [NUM_REQS-1:0] lane_grant; + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_grant + logic older_same_bank; + always @(*) begin + older_same_bank = 1'b0; + for (int k = 0; k < l; ++k) begin + if (lookup_valid[k] && (lane_bank[k] == lane_bank[l])) begin + older_same_bank = 1'b1; + end + end + end + assign lane_grant[l] = lookup_valid[l] && !older_same_bank; + assign bank_conflict[l] = lookup_valid[l] && older_same_bank; + end + + // --------------------------------------------------------------------- + // Entry storage: one CAM per bank, granted lane only. + // --------------------------------------------------------------------- + wire [NUM_BANKS-1:0] bank_install_valid; + tlb_entry_t install_entry; + wire [NUM_BANKS-1:0] bank_lookup_hit; + wire [NUM_BANKS-1:0][TLB_PPN_WIDTH-1:0] bank_lookup_ppn; + wire [NUM_BANKS-1:0][TLB_FLAGS_WIDTH-1:0] bank_lookup_flags; + wire [NUM_BANKS-1:0][TLB_VPN_WIDTH-1:0] bank_lookup_vpn; + wire [NUM_BANKS-1:0] bank_access_hit; + wire [NUM_BANKS-1:0] bank_install_evict; +`ifndef PERF_ENABLE + `UNUSED_VAR (bank_install_evict) +`endif + + for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_banks + logic [LANE_W-1:0] owner; + always @(*) begin + owner = '0; + for (int l = NUM_REQS-1; l >= 0; --l) begin + if (lane_grant[l] && (lane_bank[l] == BANK_W'(b))) begin + owner = LANE_W'(l); + end + end + end + assign bank_lookup_vpn[b] = lookup_vpn[owner]; + assign bank_access_hit[b] = access_hit[owner] && (lane_bank[owner] == BANK_W'(b)); + + VX_tlb_cam #( + .NUM_REQS (1), + .TLB_SIZE (ENTRIES_PER_BANK) + ) cam ( + .clk (clk), + .reset (reset), + .lookup_vpn (bank_lookup_vpn[b]), + .lookup_hit (bank_lookup_hit[b]), + .lookup_ppn (bank_lookup_ppn[b]), + .lookup_flags (bank_lookup_flags[b]), + `UNUSED_PIN (lookup_ppn_raw), + `UNUSED_PIN (lookup_level), + .access_hit (bank_access_hit[b]), + .install_valid (bank_install_valid[b]), + .install_entry (install_entry), + .install_evict (bank_install_evict[b]), + .flush (flush) + ); + end + + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lane_out + assign lookup_hit[l] = lane_grant[l] && bank_lookup_hit[lane_bank[l]]; + assign lookup_ppn[l] = bank_lookup_ppn[lane_bank[l]]; + assign lookup_flags[l] = bank_lookup_flags[lane_bank[l]]; + // A lane whose VPN matches its bank's in-flight walk waits for that + // fill rather than re-walking (categorized cat_park; park_ready holds + // it off until the bank drains and the entry installs). + assign mshr_match[l] = lane_grant[l] + && (bk_state[lane_bank[l]] != B_IDLE) + && (bk_vpn[lane_bank[l]] == lookup_vpn[l]); + end + + // --------------------------------------------------------------------- + // Park: one slot per bank; the bank must be idle. + // --------------------------------------------------------------------- + wire [BANK_W-1:0] park_bank = bank_of(park_vpn[BANK_W-1:0]); + assign park_ready = !flush && (bk_state[park_bank] == B_IDLE); + wire park_fire = park_valid && park_ready; + `UNUSED_VAR (park_lane) + + // --------------------------------------------------------------------- + // Walk issue: round-robin over banks in B_WALK_REQ. + // --------------------------------------------------------------------- + reg [BANK_W-1:0] issue_rr; + logic [BANK_W-1:0] issue_sel; + logic issue_any; + always @(*) begin + issue_sel = '0; + issue_any = 1'b0; + for (int i = NUM_BANKS-1; i >= 0; --i) begin + automatic logic [BANK_W-1:0] b = BANK_W'(int'(issue_rr) + i + 1); + if (bk_state[b] == B_WALK_REQ) begin + issue_sel = b; + issue_any = 1'b1; + end + end + end + + assign tlb_bus_if.req_valid = issue_any; + assign tlb_bus_if.req_data = '{ + id: `UP(ID_WIDTH)'(issue_sel), + access: bk_access[issue_sel], + amo: bk_amo[issue_sel], + vpn: bk_vpn[issue_sel] + }; + wire issue_fire = tlb_bus_if.req_valid && tlb_bus_if.req_ready; + + // --------------------------------------------------------------------- + // Fill: install (unless stale/faulted) and stage the bank for drain. + // --------------------------------------------------------------------- + assign tlb_bus_if.rsp_ready = 1'b1; + wire fill_fire = tlb_bus_if.rsp_valid && tlb_bus_if.rsp_ready; + wire [BANK_W-1:0] fill_bank = BANK_W'(tlb_bus_if.rsp_data.id); + wire fill_ok = fill_fire && !tlb_bus_if.rsp_data.fault + && !bk_stale[fill_bank] && !flush; + + assign install_entry = '{ + level: tlb_bus_if.rsp_data.level, + vpn: bk_vpn[fill_bank], + ppn: tlb_bus_if.rsp_data.ppn, + flags: tlb_bus_if.rsp_data.flags + }; + for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_install + assign bank_install_valid[b] = fill_ok && (fill_bank == BANK_W'(b)); + end + + // --------------------------------------------------------------------- + // Drain: one bank per cycle, round-robin; faulted walks kill. + // --------------------------------------------------------------------- + reg [BANK_W-1:0] drain_rr; + logic [BANK_W-1:0] drain_sel; + logic drain_any; + always @(*) begin + drain_sel = '0; + drain_any = 1'b0; + for (int i = NUM_BANKS-1; i >= 0; --i) begin + automatic logic [BANK_W-1:0] b = BANK_W'(int'(drain_rr) + i + 1); + if (bk_state[b] == B_DRAIN) begin + drain_sel = b; + drain_any = 1'b1; + end + end + end + + assign replay_valid = drain_any && !bk_fault[drain_sel]; + assign replay_payload = bk_payload[drain_sel]; + assign replay_ppn = bk_ppn[drain_sel]; + assign replay_level = bk_level[drain_sel]; + assign replay_flags = bk_flags[drain_sel]; + assign kill_valid = drain_any && bk_fault[drain_sel]; + wire drain_fire = drain_any && (bk_fault[drain_sel] ? kill_ready : replay_ready); + + // Structural faults surface as the kill drains. + assign mshr_fault_valid = kill_valid && kill_ready; + assign mshr_fault_vpn = bk_vpn[drain_sel]; + assign mshr_fault_access = bk_access[drain_sel]; + + // --------------------------------------------------------------------- + // Bank state + // --------------------------------------------------------------------- + always @(posedge clk) begin + if (reset) begin + for (int b = 0; b < NUM_BANKS; ++b) begin + bk_state[b] <= B_IDLE; + bk_stale[b] <= 1'b0; + end + issue_rr <= '0; + drain_rr <= '0; + end else begin + if (park_fire) begin + bk_state [park_bank] <= B_WALK_REQ; + bk_vpn [park_bank] <= park_vpn; + bk_access [park_bank] <= park_access; + bk_amo [park_bank] <= park_amo; + bk_payload[park_bank] <= park_payload; + bk_stale [park_bank] <= 1'b0; + end + if (issue_fire) begin + bk_state[issue_sel] <= B_WALK_WAIT; + issue_rr <= issue_sel; + end + if (fill_fire) begin + if (bk_stale[fill_bank]) begin + // stale walk: discard the result and walk again + bk_state[fill_bank] <= B_WALK_REQ; + bk_stale[fill_bank] <= 1'b0; + end else begin + bk_state[fill_bank] <= B_DRAIN; + bk_fault[fill_bank] <= tlb_bus_if.rsp_data.fault; + bk_ppn [fill_bank] <= tlb_bus_if.rsp_data.ppn; + bk_level[fill_bank] <= tlb_bus_if.rsp_data.level; + bk_flags[fill_bank] <= tlb_bus_if.rsp_data.flags; + end + end + if (drain_fire) begin + bk_state[drain_sel] <= B_IDLE; + drain_rr <= drain_sel; + end + if (flush) begin + for (int b = 0; b < NUM_BANKS; ++b) begin + if (bk_state[b] == B_WALK_WAIT) begin + bk_stale[b] <= 1'b1; + end + end + end + end + end + + logic any_busy; + always @(*) begin + any_busy = 1'b0; + for (int b = 0; b < NUM_BANKS; ++b) begin + if (bk_state[b] != B_IDLE) any_busy = 1'b1; + end + end + assign empty = !any_busy; + + // --------------------------------------------------------------------- + // Performance counters + // --------------------------------------------------------------------- +`ifdef PERF_ENABLE + reg [PERF_CTR_BITS-1:0] perf_reads, perf_hits, perf_misses, perf_evicts; + logic [`CLOG2(NUM_REQS+1)-1:0] reads_now, hits_now; + always @(*) begin + reads_now = '0; + hits_now = '0; + for (int l = 0; l < NUM_REQS; ++l) begin + if (lane_grant[l]) reads_now = reads_now + 1; + if (lookup_hit[l] && access_hit[l]) hits_now = hits_now + 1; + end + end + always @(posedge clk) begin + if (reset) begin + perf_reads <= '0; + perf_hits <= '0; + perf_misses <= '0; + perf_evicts <= '0; + end else begin + perf_reads <= perf_reads + PERF_CTR_BITS'(reads_now); + perf_hits <= perf_hits + PERF_CTR_BITS'(hits_now); + perf_misses <= perf_misses + PERF_CTR_BITS'(park_fire); + perf_evicts <= perf_evicts + PERF_CTR_BITS'((| (bank_install_valid & bank_install_evict))); + end + end + assign mmu_perf.tlb_reads = perf_reads; + assign mmu_perf.tlb_hits = perf_hits; + assign mmu_perf.tlb_misses = perf_misses; + assign mmu_perf.tlb_evictions = perf_evicts; + // Every parked miss issues exactly one walk (re-walks after a flush are + // counted again on issue). + assign mmu_perf.ptw_walks = perf_misses; + assign mmu_perf.ptw_latency = '0; +`endif + +endmodule From 7e65f817386ec050bf1bfbb82c8c71c53b48e0e1 Mon Sep 17 00:00:00 2001 From: Thomas Weatherly Date: Tue, 1 Sep 2026 18:40:28 -0400 Subject: [PATCH 2/5] hw: adopt the banked L1 TLB Replace the multi-ported CAM + shared-MSHR L1 TLB with the banked organization in place: VX_tlb_l1.sv now holds the banked storage + miss station (previously VX_tlb_l1_banked.sv behind a config select), and VX_mmu instantiates it unconditionally. The entry array splits into VX_CFG_L1_TLB_NUM_BANKS single-ported banks (low VPN bits select); each bank answers one lane per cycle and holds one parked miss, so a miss blocks only its bank while the others keep hitting. Trades the full multi-port CAM + shared MSHR for per-bank ports and slots: cheaper lookup hardware at scale, one outstanding walk per bank. Validated (banked, 4 banks, with the device-level walker): full vm catalog green on rtlsim-32 (23), simx-32 (21, incl. model parity) and rtlsim-64 (23); demo at 1 and 2 clusters; -Wall clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DJk9ipw4g1ZP3zZShiYaMq --- VX_config.toml | 7 +- hw/rtl/vm/VX_mmu.sv | 55 +---- hw/rtl/vm/VX_tlb_l1.sv | 435 +++++++++++++++++++++++----------- hw/rtl/vm/VX_tlb_l1_banked.sv | 372 ----------------------------- 4 files changed, 298 insertions(+), 571 deletions(-) delete mode 100644 hw/rtl/vm/VX_tlb_l1_banked.sv diff --git a/VX_config.toml b/VX_config.toml index 184dd049bc..888b83a1b7 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -141,10 +141,9 @@ VX_CFG_NUM_VPU_BLOCKS = 1 # VM page-table format is a HW<->SW contract, moved to VX_types.toml [vm]; TLB depth stays here. VX_CFG_TLB_SIZE = 32 -# L1 TLB organization: 0 selects the baseline multi-ported CAM + MSHR -# (hit-under-miss); a power of two selects the banked storage — per-bank -# single lookup ports with one parked miss per bank (VX_tlb_l1_banked). -VX_CFG_L1_TLB_NUM_BANKS = 0 +# L1 TLB banks (power of two): per-bank single lookup ports with one +# parked miss per bank; hits in other banks proceed under a miss. +VX_CFG_L1_TLB_NUM_BANKS = 4 # L1 TLB stage (per core): D-side / I-side entries and the per-instance # miss station depth (distinct outstanding VPN misses). diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index 136dbf0daf..27acc4df48 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -25,8 +25,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter NUM_REQS = DCACHE_NUM_REQS, parameter TLB_SIZE = `VX_CFG_DTLB_SIZE, - parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, - parameter REPLAY_DEPTH = 2, + parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, // walk-ID space (>= NUM_BANKS) parameter EXEC_SIDE = 0, parameter DATA_SIZE = DCACHE_WORD_SIZE, parameter TAG_WIDTH = DCACHE_TAG_WIDTH_BASE, @@ -147,11 +146,9 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [NUM_REQS-1:0] bank_conflict; - // L1 storage organization: banked (per-bank lookup port + one parked - // miss per bank, losing lanes hold via bank_conflict) or the baseline - // multi-ported CAM + MSHR. - if (`VX_CFG_L1_TLB_NUM_BANKS != 0) begin : g_tlb_banked - VX_tlb_l1_banked #( + // L1 storage: banked — per-bank lookup port + one parked miss per bank; + // losing lanes hold via bank_conflict. + VX_tlb_l1 #( .NUM_REQS (NUM_REQS), .TLB_SIZE (TLB_SIZE), .NUM_BANKS (`VX_CFG_L1_TLB_NUM_BANKS), @@ -193,50 +190,6 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .flush (flush_clear), .empty (tlb_empty) ); - end else begin : g_tlb_baseline - assign bank_conflict = '0; - VX_tlb_l1 #( - .NUM_REQS (NUM_REQS), - .TLB_SIZE (TLB_SIZE), - .MSHR_SIZE (MSHR_SIZE), - .REPLAY_DEPTH (REPLAY_DEPTH), - .PAYLOAD_W (PAYLOAD_W), - .ID_WIDTH (ID_WIDTH) - ) tlb ( - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .mmu_perf (mmu_perf), - `endif - .lookup_vpn (cam_vpn), - .lookup_hit (cam_hit), - .lookup_ppn (cam_ppn), - .lookup_flags (cam_flags), - .access_hit (cam_access_hit), - .mshr_match (mshr_match), - .park_valid (park_valid), - .park_vpn (park_vpn), - .park_access (park_access), - .park_amo (park_amo), - .park_lane (park_lane), - .park_payload (park_payload), - .park_ready (park_ready), - .replay_valid (replay_valid), - .replay_payload(replay_payload), - .replay_ppn (replay_ppn), - .replay_level (replay_level), - .replay_flags (replay_flags), - .replay_ready (replay_ready), - .kill_valid (kill_valid), - .kill_ready (kill_ready), - .mshr_fault_valid (mshr_fault_valid), - .mshr_fault_vpn (mshr_fault_vpn), - .mshr_fault_access (mshr_fault_access), - .tlb_bus_if (tlb_bus_if), - .flush (flush_clear), - .empty (tlb_empty) - ); - end // --------------------------------------------------------------------- diff --git a/hw/rtl/vm/VX_tlb_l1.sv b/hw/rtl/vm/VX_tlb_l1.sv index 705a8812c1..10ab8b54bc 100644 --- a/hw/rtl/vm/VX_tlb_l1.sv +++ b/hw/rtl/vm/VX_tlb_l1.sv @@ -1,33 +1,23 @@ // Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Licensed under the Apache License, Version 2.0. `include "VX_define.vh" -// L1 TLB storage: the pure lookup + miss-handling core, with no address -// translation of its own. It holds the fully-associative entry array -// (parallel per-lane read), the non-blocking miss station (park / replay / -// kill), and the fill path from the shared walker over `tlb_bus`. The parent -// `VX_mmu` drives the VPN probes and consumes the raw lookup results (PPN, -// flags) plus the replay/kill streams, doing the VA→PA splice and permission -// checks itself. Splitting the storage out keeps "TLB = lookup" explicit and -// lets the banked variant live entirely here. +// Banked L1 TLB storage + miss station behind the VX_mmu parent contract. +// The entry array is split into NUM_BANKS single-ported banks selected by +// the low VPN bits: each bank answers at most one lane per cycle +// (bank_conflict tells the parent to hold the other lanes), and each bank +// holds at most one parked miss — a miss blocks its bank for the walk's +// duration while the other banks keep hitting. A same-VPN request waits on +// its bank's in-flight walk (mshr_match) instead of joining a queue. +// Trades a full multi-port CAM + shared MSHR for per-bank ports and slots: +// cheaper lookup hardware at scale, one outstanding walk per bank. module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter NUM_REQS = DCACHE_NUM_REQS, - parameter TLB_SIZE = `VX_CFG_DTLB_SIZE, - parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, - parameter REPLAY_DEPTH = 2, - parameter PAYLOAD_W = 1, - parameter ID_WIDTH = `CLOG2(MSHR_SIZE) + parameter NUM_REQS = DCACHE_NUM_REQS, + parameter TLB_SIZE = `VX_CFG_DTLB_SIZE, + parameter NUM_BANKS = 4, + parameter PAYLOAD_W = 1, + parameter ID_WIDTH = `CLOG2(NUM_BANKS) ) ( input wire clk, input wire reset, @@ -36,11 +26,15 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( output mmu_perf_t mmu_perf, `endif - // Per-lane combinational lookup (VPN in, raw translation out). + // Per-lane lookup. A lane that loses its bank's port this cycle gets + // bank_conflict (parent must hold it); hit/miss is only meaningful for + // lanes with bank_conflict == 0. input wire [NUM_REQS-1:0][TLB_VPN_WIDTH-1:0] lookup_vpn, + input wire [NUM_REQS-1:0] lookup_valid, output wire [NUM_REQS-1:0] lookup_hit, output wire [NUM_REQS-1:0][TLB_PPN_WIDTH-1:0] lookup_ppn, output wire [NUM_REQS-1:0][TLB_FLAGS_WIDTH-1:0] lookup_flags, + output wire [NUM_REQS-1:0] bank_conflict, input wire [NUM_REQS-1:0] access_hit, output wire [NUM_REQS-1:0] mshr_match, @@ -65,159 +59,312 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( output wire kill_valid, input wire kill_ready, - // Structural-fault sideband (from the miss station). + // Structural-fault sideband. output wire mshr_fault_valid, output wire [TLB_VPN_WIDTH-1:0] mshr_fault_vpn, output tlb_access_e mshr_fault_access, - // Miss/fill fabric to the shared walker complex. + // Miss/fill fabric to the shared walker complex (id = bank index). VX_tlb_bus_if.master tlb_bus_if, input wire flush, output wire empty ); + `STATIC_ASSERT(`IS_POW2(NUM_BANKS), ("NUM_BANKS must be a power of 2")) + `STATIC_ASSERT((TLB_SIZE % NUM_BANKS) == 0, ("NUM_BANKS must divide TLB_SIZE")) + `STATIC_ASSERT(ID_WIDTH >= `CLOG2(NUM_BANKS), ("bank index must fit the bus id")) + + localparam ENTRIES_PER_BANK = TLB_SIZE / NUM_BANKS; + localparam BANK_W = `UP(`CLOG2(NUM_BANKS)); + localparam LANE_W = `UP(`CLOG2(NUM_REQS)); + + function automatic logic [BANK_W-1:0] bank_of(input logic [BANK_W-1:0] vpn_lo); + if (NUM_BANKS == 1) bank_of = '0; + else bank_of = vpn_lo; + endfunction + // --------------------------------------------------------------------- - // Entry array (fully-associative, parallel per-lane read) - // --------------------------------------------------------------------- - wire install_valid; - tlb_entry_t install_entry; - wire install_evict; - - VX_tlb_cam #( - .NUM_REQS (NUM_REQS), - .TLB_SIZE (TLB_SIZE) - ) cam ( - .clk (clk), - .reset (reset), - .lookup_vpn (lookup_vpn), - .lookup_hit (lookup_hit), - .lookup_ppn (lookup_ppn), - .lookup_flags (lookup_flags), - `UNUSED_PIN (lookup_ppn_raw), - `UNUSED_PIN (lookup_level), - .access_hit (access_hit), - .install_valid (install_valid), - .install_entry (install_entry), - .install_evict (install_evict), - .flush (flush) - ); + // Per-bank parked-miss slot + // --------------------------------------------------------------------- + typedef enum logic [1:0] { + B_IDLE, B_WALK_REQ, B_WALK_WAIT, B_DRAIN + } bank_state_e; + + bank_state_e bk_state [NUM_BANKS]; + logic [TLB_VPN_WIDTH-1:0] bk_vpn [NUM_BANKS]; + tlb_access_e bk_access [NUM_BANKS]; + logic bk_amo [NUM_BANKS]; + logic [PAYLOAD_W-1:0] bk_payload [NUM_BANKS]; + logic bk_fault [NUM_BANKS]; + logic [TLB_PPN_WIDTH-1:0] bk_ppn [NUM_BANKS]; + logic [TLB_LEVEL_WIDTH-1:0] bk_level [NUM_BANKS]; + logic [TLB_FLAGS_WIDTH-1:0] bk_flags [NUM_BANKS]; + // A walk in flight when the flush arrived resolved against the old page + // table: drop its fill and re-walk (see the shared walker's discipline). + logic bk_stale [NUM_BANKS]; + + // --------------------------------------------------------------------- + // Bank lookup port arbitration: lowest contending lane wins the bank. + // --------------------------------------------------------------------- + wire [NUM_REQS-1:0][BANK_W-1:0] lane_bank; + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lane_bank + assign lane_bank[l] = bank_of(lookup_vpn[l][BANK_W-1:0]); + end + + wire [NUM_REQS-1:0] lane_grant; + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_grant + logic older_same_bank; + always @(*) begin + older_same_bank = 1'b0; + for (int k = 0; k < l; ++k) begin + if (lookup_valid[k] && (lane_bank[k] == lane_bank[l])) begin + older_same_bank = 1'b1; + end + end + end + assign lane_grant[l] = lookup_valid[l] && !older_same_bank; + assign bank_conflict[l] = lookup_valid[l] && older_same_bank; + end + + // --------------------------------------------------------------------- + // Entry storage: one CAM per bank, granted lane only. + // --------------------------------------------------------------------- + wire [NUM_BANKS-1:0] bank_install_valid; + tlb_entry_t install_entry; + wire [NUM_BANKS-1:0] bank_lookup_hit; + wire [NUM_BANKS-1:0][TLB_PPN_WIDTH-1:0] bank_lookup_ppn; + wire [NUM_BANKS-1:0][TLB_FLAGS_WIDTH-1:0] bank_lookup_flags; + wire [NUM_BANKS-1:0][TLB_VPN_WIDTH-1:0] bank_lookup_vpn; + wire [NUM_BANKS-1:0] bank_access_hit; + wire [NUM_BANKS-1:0] bank_install_evict; `ifndef PERF_ENABLE - `UNUSED_VAR (install_evict) + `UNUSED_VAR (bank_install_evict) `endif + for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_banks + logic [LANE_W-1:0] owner; + always @(*) begin + owner = '0; + for (int l = NUM_REQS-1; l >= 0; --l) begin + if (lane_grant[l] && (lane_bank[l] == BANK_W'(b))) begin + owner = LANE_W'(l); + end + end + end + assign bank_lookup_vpn[b] = lookup_vpn[owner]; + assign bank_access_hit[b] = access_hit[owner] && (lane_bank[owner] == BANK_W'(b)); + + VX_tlb_cam #( + .NUM_REQS (1), + .TLB_SIZE (ENTRIES_PER_BANK) + ) cam ( + .clk (clk), + .reset (reset), + .lookup_vpn (bank_lookup_vpn[b]), + .lookup_hit (bank_lookup_hit[b]), + .lookup_ppn (bank_lookup_ppn[b]), + .lookup_flags (bank_lookup_flags[b]), + `UNUSED_PIN (lookup_ppn_raw), + `UNUSED_PIN (lookup_level), + .access_hit (bank_access_hit[b]), + .install_valid (bank_install_valid[b]), + .install_entry (install_entry), + .install_evict (bank_install_evict[b]), + .flush (flush) + ); + end + + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lane_out + assign lookup_hit[l] = lane_grant[l] && bank_lookup_hit[lane_bank[l]]; + assign lookup_ppn[l] = bank_lookup_ppn[lane_bank[l]]; + assign lookup_flags[l] = bank_lookup_flags[lane_bank[l]]; + // A lane whose VPN matches its bank's in-flight walk waits for that + // fill rather than re-walking (categorized cat_park; park_ready holds + // it off until the bank drains and the entry installs). + assign mshr_match[l] = lane_grant[l] + && (bk_state[lane_bank[l]] != B_IDLE) + && (bk_vpn[lane_bank[l]] == lookup_vpn[l]); + end + + // --------------------------------------------------------------------- + // Park: one slot per bank; the bank must be idle. // --------------------------------------------------------------------- - // Miss station - // --------------------------------------------------------------------- - wire [`UP(ID_WIDTH)-1:0] tlb_req_id; - tlb_access_e tlb_req_access; - wire tlb_req_amo; - wire [TLB_VPN_WIDTH-1:0] tlb_req_vpn; - - // L1 drain glue: faulted entries drain as kills, live ones as replays. - wire drain_valid; - wire [PAYLOAD_W-1:0] drain_qdata; - wire drain_fault; - wire [TLB_PPN_WIDTH-1:0] drain_ppn; - wire [TLB_LEVEL_WIDTH-1:0] drain_level; - wire [TLB_FLAGS_WIDTH-1:0] drain_flags; - - assign replay_valid = drain_valid && !drain_fault; - assign replay_payload = drain_qdata; - assign replay_ppn = drain_ppn; - assign replay_level = drain_level; - assign replay_flags = drain_flags; - assign kill_valid = drain_valid && drain_fault; - wire drain_ready = drain_fault ? kill_ready : replay_ready; - - VX_tlb_mshr #( - .NUM_REQS (NUM_REQS), - .MSHR_SIZE (MSHR_SIZE), - .QDATA_W (PAYLOAD_W), - .QDEPTH (REPLAY_DEPTH), - .DEDUP_LIVE_EXCLUDES_FAULT (1), - .ID_WIDTH (ID_WIDTH) - ) mshr ( - .clk (clk), - .reset (reset), - .probe_vpn (lookup_vpn), - .probe_match (mshr_match), - .alloc_valid (park_valid), - .alloc_vpn (park_vpn), - .alloc_access (park_access), - .alloc_amo (park_amo), - .alloc_lane (park_lane), - .alloc_qdata (park_payload), - .alloc_ready (park_ready), - .issue_valid (tlb_bus_if.req_valid), - .issue_slot (tlb_req_id), - .issue_access (tlb_req_access), - .issue_amo (tlb_req_amo), - .issue_vpn (tlb_req_vpn), - .issue_ready (tlb_bus_if.req_ready), - .fill_valid (tlb_bus_if.rsp_valid), - .fill_slot (tlb_bus_if.rsp_data.id), - .fill_fault (tlb_bus_if.rsp_data.fault), - .fill_level (tlb_bus_if.rsp_data.level), - .fill_ppn (tlb_bus_if.rsp_data.ppn), - .fill_flags (tlb_bus_if.rsp_data.flags), - .fill_ready (tlb_bus_if.rsp_ready), - .install_valid (install_valid), - .install_entry (install_entry), - .fault_valid (mshr_fault_valid), - .fault_vpn (mshr_fault_vpn), - .fault_access (mshr_fault_access), - .drain_valid (drain_valid), - .drain_qdata (drain_qdata), - .drain_fault (drain_fault), - .drain_ppn (drain_ppn), - .drain_level (drain_level), - .drain_flags (drain_flags), - .drain_ready (drain_ready), - .flush (flush), - .empty (empty) - ); + wire [BANK_W-1:0] park_bank = bank_of(park_vpn[BANK_W-1:0]); + assign park_ready = !flush && (bk_state[park_bank] == B_IDLE); + wire park_fire = park_valid && park_ready; + `UNUSED_VAR (park_lane) + // --------------------------------------------------------------------- + // Walk issue: round-robin over banks in B_WALK_REQ. + // --------------------------------------------------------------------- + reg [BANK_W-1:0] issue_rr; + logic [BANK_W-1:0] issue_sel; + logic issue_any; + always @(*) begin + issue_sel = '0; + issue_any = 1'b0; + for (int i = NUM_BANKS-1; i >= 0; --i) begin + automatic logic [BANK_W-1:0] b = BANK_W'(int'(issue_rr) + i + 1); + if (bk_state[b] == B_WALK_REQ) begin + issue_sel = b; + issue_any = 1'b1; + end + end + end + + assign tlb_bus_if.req_valid = issue_any; assign tlb_bus_if.req_data = '{ - id: tlb_req_id, - access: tlb_req_access, - amo: tlb_req_amo, - vpn: tlb_req_vpn + id: `UP(ID_WIDTH)'(issue_sel), + access: bk_access[issue_sel], + amo: bk_amo[issue_sel], + vpn: bk_vpn[issue_sel] }; + wire issue_fire = tlb_bus_if.req_valid && tlb_bus_if.req_ready; + + // --------------------------------------------------------------------- + // Fill: install (unless stale/faulted) and stage the bank for drain. + // --------------------------------------------------------------------- + assign tlb_bus_if.rsp_ready = 1'b1; + wire fill_fire = tlb_bus_if.rsp_valid && tlb_bus_if.rsp_ready; + wire [BANK_W-1:0] fill_bank = BANK_W'(tlb_bus_if.rsp_data.id); + wire fill_ok = fill_fire && !tlb_bus_if.rsp_data.fault + && !bk_stale[fill_bank] && !flush; + + assign install_entry = '{ + level: tlb_bus_if.rsp_data.level, + vpn: bk_vpn[fill_bank], + ppn: tlb_bus_if.rsp_data.ppn, + flags: tlb_bus_if.rsp_data.flags + }; + for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_install + assign bank_install_valid[b] = fill_ok && (fill_bank == BANK_W'(b)); + end + + // --------------------------------------------------------------------- + // Drain: one bank per cycle, round-robin; faulted walks kill. + // --------------------------------------------------------------------- + reg [BANK_W-1:0] drain_rr; + logic [BANK_W-1:0] drain_sel; + logic drain_any; + always @(*) begin + drain_sel = '0; + drain_any = 1'b0; + for (int i = NUM_BANKS-1; i >= 0; --i) begin + automatic logic [BANK_W-1:0] b = BANK_W'(int'(drain_rr) + i + 1); + if (bk_state[b] == B_DRAIN) begin + drain_sel = b; + drain_any = 1'b1; + end + end + end + + assign replay_valid = drain_any && !bk_fault[drain_sel]; + assign replay_payload = bk_payload[drain_sel]; + assign replay_ppn = bk_ppn[drain_sel]; + assign replay_level = bk_level[drain_sel]; + assign replay_flags = bk_flags[drain_sel]; + assign kill_valid = drain_any && bk_fault[drain_sel]; + wire drain_fire = drain_any && (bk_fault[drain_sel] ? kill_ready : replay_ready); + + // Structural faults surface as the kill drains. + assign mshr_fault_valid = kill_valid && kill_ready; + assign mshr_fault_vpn = bk_vpn[drain_sel]; + assign mshr_fault_access = bk_access[drain_sel]; + + // --------------------------------------------------------------------- + // Bank state + // --------------------------------------------------------------------- + always @(posedge clk) begin + if (reset) begin + for (int b = 0; b < NUM_BANKS; ++b) begin + bk_state[b] <= B_IDLE; + bk_stale[b] <= 1'b0; + end + issue_rr <= '0; + drain_rr <= '0; + end else begin + if (park_fire) begin + bk_state [park_bank] <= B_WALK_REQ; + bk_vpn [park_bank] <= park_vpn; + bk_access [park_bank] <= park_access; + bk_amo [park_bank] <= park_amo; + bk_payload[park_bank] <= park_payload; + bk_stale [park_bank] <= 1'b0; + end + if (issue_fire) begin + bk_state[issue_sel] <= B_WALK_WAIT; + issue_rr <= issue_sel; + end + if (fill_fire) begin + if (bk_stale[fill_bank]) begin + // stale walk: discard the result and walk again + bk_state[fill_bank] <= B_WALK_REQ; + bk_stale[fill_bank] <= 1'b0; + end else begin + bk_state[fill_bank] <= B_DRAIN; + bk_fault[fill_bank] <= tlb_bus_if.rsp_data.fault; + bk_ppn [fill_bank] <= tlb_bus_if.rsp_data.ppn; + bk_level[fill_bank] <= tlb_bus_if.rsp_data.level; + bk_flags[fill_bank] <= tlb_bus_if.rsp_data.flags; + end + end + if (drain_fire) begin + bk_state[drain_sel] <= B_IDLE; + drain_rr <= drain_sel; + end + if (flush) begin + for (int b = 0; b < NUM_BANKS; ++b) begin + if (bk_state[b] == B_WALK_WAIT) begin + bk_stale[b] <= 1'b1; + end + end + end + end + end + + logic any_busy; + always @(*) begin + any_busy = 1'b0; + for (int b = 0; b < NUM_BANKS; ++b) begin + if (bk_state[b] != B_IDLE) any_busy = 1'b1; + end + end + assign empty = !any_busy; // --------------------------------------------------------------------- // Performance counters // --------------------------------------------------------------------- `ifdef PERF_ENABLE - wire [`CLOG2(NUM_REQS+1)-1:0] n_hits; - `POP_COUNT(n_hits, access_hit); - wire miss_ev = park_valid && park_ready; - - reg [PERF_CTR_BITS-1:0] perf_reads, perf_hits, perf_misses, perf_evicts, perf_walks; + reg [PERF_CTR_BITS-1:0] perf_reads, perf_hits, perf_misses, perf_evicts; + logic [`CLOG2(NUM_REQS+1)-1:0] reads_now, hits_now; + always @(*) begin + reads_now = '0; + hits_now = '0; + for (int l = 0; l < NUM_REQS; ++l) begin + if (lane_grant[l]) reads_now = reads_now + 1; + if (lookup_hit[l] && access_hit[l]) hits_now = hits_now + 1; + end + end always @(posedge clk) begin if (reset) begin perf_reads <= '0; perf_hits <= '0; perf_misses <= '0; perf_evicts <= '0; - perf_walks <= '0; end else begin - perf_reads <= perf_reads + PERF_CTR_BITS'(n_hits) + PERF_CTR_BITS'(miss_ev); - perf_hits <= perf_hits + PERF_CTR_BITS'(n_hits); - perf_misses <= perf_misses + PERF_CTR_BITS'(miss_ev); - if (install_valid && install_evict) begin - perf_evicts <= perf_evicts + PERF_CTR_BITS'(1); - end - if (tlb_bus_if.req_valid && tlb_bus_if.req_ready) begin - perf_walks <= perf_walks + PERF_CTR_BITS'(1); - end + perf_reads <= perf_reads + PERF_CTR_BITS'(reads_now); + perf_hits <= perf_hits + PERF_CTR_BITS'(hits_now); + perf_misses <= perf_misses + PERF_CTR_BITS'(park_fire); + perf_evicts <= perf_evicts + PERF_CTR_BITS'((| (bank_install_valid & bank_install_evict))); end end - assign mmu_perf.tlb_reads = perf_reads; assign mmu_perf.tlb_hits = perf_hits; assign mmu_perf.tlb_misses = perf_misses; assign mmu_perf.tlb_evictions = perf_evicts; - assign mmu_perf.ptw_walks = perf_walks; + // Every parked miss issues exactly one walk (re-walks after a flush are + // counted again on issue). + assign mmu_perf.ptw_walks = perf_misses; assign mmu_perf.ptw_latency = '0; `endif diff --git a/hw/rtl/vm/VX_tlb_l1_banked.sv b/hw/rtl/vm/VX_tlb_l1_banked.sv deleted file mode 100644 index d4a73af226..0000000000 --- a/hw/rtl/vm/VX_tlb_l1_banked.sv +++ /dev/null @@ -1,372 +0,0 @@ -// Copyright © 2019-2023 -// Licensed under the Apache License, Version 2.0. - -`include "VX_define.vh" - -// Banked L1 TLB storage + miss station, an alternative to VX_tlb_l1 behind -// the same parent (VX_mmu) contract. The entry array is split into -// NUM_BANKS single-ported banks selected by the low VPN bits: each bank -// answers at most one lane per cycle (bank_conflict tells the parent to -// hold the other lanes), and each bank holds at most one parked miss — -// a miss blocks its bank for the walk's duration while the other banks -// keep hitting. A same-VPN request waits on its busk's in-flight walk -// (mshr_match) instead of joining a queue. Trades the baseline's -// full multi-port CAM + MSHR for per-bank ports and slots: cheaper -// lookup hardware at scale, one outstanding walk per bank. -module VX_tlb_l1_banked import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter NUM_REQS = DCACHE_NUM_REQS, - parameter TLB_SIZE = `VX_CFG_DTLB_SIZE, - parameter NUM_BANKS = 4, - parameter PAYLOAD_W = 1, - parameter ID_WIDTH = `CLOG2(NUM_BANKS) -) ( - input wire clk, - input wire reset, - -`ifdef PERF_ENABLE - output mmu_perf_t mmu_perf, -`endif - - // Per-lane lookup. A lane that loses its bank's port this cycle gets - // bank_conflict (parent must hold it); hit/miss is only meaningful for - // lanes with bank_conflict == 0. - input wire [NUM_REQS-1:0][TLB_VPN_WIDTH-1:0] lookup_vpn, - input wire [NUM_REQS-1:0] lookup_valid, - output wire [NUM_REQS-1:0] lookup_hit, - output wire [NUM_REQS-1:0][TLB_PPN_WIDTH-1:0] lookup_ppn, - output wire [NUM_REQS-1:0][TLB_FLAGS_WIDTH-1:0] lookup_flags, - output wire [NUM_REQS-1:0] bank_conflict, - input wire [NUM_REQS-1:0] access_hit, - output wire [NUM_REQS-1:0] mshr_match, - - // Park a miss (payload is opaque; the parent splices on replay). - input wire park_valid, - input wire [TLB_VPN_WIDTH-1:0] park_vpn, - input tlb_access_e park_access, - input wire park_amo, - input wire [`UP(`CLOG2(NUM_REQS))-1:0] park_lane, - input wire [PAYLOAD_W-1:0] park_payload, - output wire park_ready, - - // Replay a parked request once its fill lands. - output wire replay_valid, - output wire [PAYLOAD_W-1:0] replay_payload, - output wire [TLB_PPN_WIDTH-1:0] replay_ppn, - output wire [TLB_LEVEL_WIDTH-1:0] replay_level, - output wire [TLB_FLAGS_WIDTH-1:0] replay_flags, - input wire replay_ready, - - // Kill a parked request whose walk faulted. - output wire kill_valid, - input wire kill_ready, - - // Structural-fault sideband. - output wire mshr_fault_valid, - output wire [TLB_VPN_WIDTH-1:0] mshr_fault_vpn, - output tlb_access_e mshr_fault_access, - - // Miss/fill fabric to the shared walker complex (id = bank index). - VX_tlb_bus_if.master tlb_bus_if, - - input wire flush, - output wire empty -); - `STATIC_ASSERT(`IS_POW2(NUM_BANKS), ("NUM_BANKS must be a power of 2")) - `STATIC_ASSERT((TLB_SIZE % NUM_BANKS) == 0, ("NUM_BANKS must divide TLB_SIZE")) - `STATIC_ASSERT(ID_WIDTH >= `CLOG2(NUM_BANKS), ("bank index must fit the bus id")) - - localparam ENTRIES_PER_BANK = TLB_SIZE / NUM_BANKS; - localparam BANK_W = `UP(`CLOG2(NUM_BANKS)); - localparam LANE_W = `UP(`CLOG2(NUM_REQS)); - - function automatic logic [BANK_W-1:0] bank_of(input logic [BANK_W-1:0] vpn_lo); - if (NUM_BANKS == 1) bank_of = '0; - else bank_of = vpn_lo; - endfunction - - // --------------------------------------------------------------------- - // Per-bank parked-miss slot - // --------------------------------------------------------------------- - typedef enum logic [1:0] { - B_IDLE, B_WALK_REQ, B_WALK_WAIT, B_DRAIN - } bank_state_e; - - bank_state_e bk_state [NUM_BANKS]; - logic [TLB_VPN_WIDTH-1:0] bk_vpn [NUM_BANKS]; - tlb_access_e bk_access [NUM_BANKS]; - logic bk_amo [NUM_BANKS]; - logic [PAYLOAD_W-1:0] bk_payload [NUM_BANKS]; - logic bk_fault [NUM_BANKS]; - logic [TLB_PPN_WIDTH-1:0] bk_ppn [NUM_BANKS]; - logic [TLB_LEVEL_WIDTH-1:0] bk_level [NUM_BANKS]; - logic [TLB_FLAGS_WIDTH-1:0] bk_flags [NUM_BANKS]; - // A walk in flight when the flush arrived resolved against the old page - // table: drop its fill and re-walk (see the shared walker's discipline). - logic bk_stale [NUM_BANKS]; - - // --------------------------------------------------------------------- - // Bank lookup port arbitration: lowest contending lane wins the bank. - // --------------------------------------------------------------------- - wire [NUM_REQS-1:0][BANK_W-1:0] lane_bank; - for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lane_bank - assign lane_bank[l] = bank_of(lookup_vpn[l][BANK_W-1:0]); - end - - wire [NUM_REQS-1:0] lane_grant; - for (genvar l = 0; l < NUM_REQS; ++l) begin : g_grant - logic older_same_bank; - always @(*) begin - older_same_bank = 1'b0; - for (int k = 0; k < l; ++k) begin - if (lookup_valid[k] && (lane_bank[k] == lane_bank[l])) begin - older_same_bank = 1'b1; - end - end - end - assign lane_grant[l] = lookup_valid[l] && !older_same_bank; - assign bank_conflict[l] = lookup_valid[l] && older_same_bank; - end - - // --------------------------------------------------------------------- - // Entry storage: one CAM per bank, granted lane only. - // --------------------------------------------------------------------- - wire [NUM_BANKS-1:0] bank_install_valid; - tlb_entry_t install_entry; - wire [NUM_BANKS-1:0] bank_lookup_hit; - wire [NUM_BANKS-1:0][TLB_PPN_WIDTH-1:0] bank_lookup_ppn; - wire [NUM_BANKS-1:0][TLB_FLAGS_WIDTH-1:0] bank_lookup_flags; - wire [NUM_BANKS-1:0][TLB_VPN_WIDTH-1:0] bank_lookup_vpn; - wire [NUM_BANKS-1:0] bank_access_hit; - wire [NUM_BANKS-1:0] bank_install_evict; -`ifndef PERF_ENABLE - `UNUSED_VAR (bank_install_evict) -`endif - - for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_banks - logic [LANE_W-1:0] owner; - always @(*) begin - owner = '0; - for (int l = NUM_REQS-1; l >= 0; --l) begin - if (lane_grant[l] && (lane_bank[l] == BANK_W'(b))) begin - owner = LANE_W'(l); - end - end - end - assign bank_lookup_vpn[b] = lookup_vpn[owner]; - assign bank_access_hit[b] = access_hit[owner] && (lane_bank[owner] == BANK_W'(b)); - - VX_tlb_cam #( - .NUM_REQS (1), - .TLB_SIZE (ENTRIES_PER_BANK) - ) cam ( - .clk (clk), - .reset (reset), - .lookup_vpn (bank_lookup_vpn[b]), - .lookup_hit (bank_lookup_hit[b]), - .lookup_ppn (bank_lookup_ppn[b]), - .lookup_flags (bank_lookup_flags[b]), - `UNUSED_PIN (lookup_ppn_raw), - `UNUSED_PIN (lookup_level), - .access_hit (bank_access_hit[b]), - .install_valid (bank_install_valid[b]), - .install_entry (install_entry), - .install_evict (bank_install_evict[b]), - .flush (flush) - ); - end - - for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lane_out - assign lookup_hit[l] = lane_grant[l] && bank_lookup_hit[lane_bank[l]]; - assign lookup_ppn[l] = bank_lookup_ppn[lane_bank[l]]; - assign lookup_flags[l] = bank_lookup_flags[lane_bank[l]]; - // A lane whose VPN matches its bank's in-flight walk waits for that - // fill rather than re-walking (categorized cat_park; park_ready holds - // it off until the bank drains and the entry installs). - assign mshr_match[l] = lane_grant[l] - && (bk_state[lane_bank[l]] != B_IDLE) - && (bk_vpn[lane_bank[l]] == lookup_vpn[l]); - end - - // --------------------------------------------------------------------- - // Park: one slot per bank; the bank must be idle. - // --------------------------------------------------------------------- - wire [BANK_W-1:0] park_bank = bank_of(park_vpn[BANK_W-1:0]); - assign park_ready = !flush && (bk_state[park_bank] == B_IDLE); - wire park_fire = park_valid && park_ready; - `UNUSED_VAR (park_lane) - - // --------------------------------------------------------------------- - // Walk issue: round-robin over banks in B_WALK_REQ. - // --------------------------------------------------------------------- - reg [BANK_W-1:0] issue_rr; - logic [BANK_W-1:0] issue_sel; - logic issue_any; - always @(*) begin - issue_sel = '0; - issue_any = 1'b0; - for (int i = NUM_BANKS-1; i >= 0; --i) begin - automatic logic [BANK_W-1:0] b = BANK_W'(int'(issue_rr) + i + 1); - if (bk_state[b] == B_WALK_REQ) begin - issue_sel = b; - issue_any = 1'b1; - end - end - end - - assign tlb_bus_if.req_valid = issue_any; - assign tlb_bus_if.req_data = '{ - id: `UP(ID_WIDTH)'(issue_sel), - access: bk_access[issue_sel], - amo: bk_amo[issue_sel], - vpn: bk_vpn[issue_sel] - }; - wire issue_fire = tlb_bus_if.req_valid && tlb_bus_if.req_ready; - - // --------------------------------------------------------------------- - // Fill: install (unless stale/faulted) and stage the bank for drain. - // --------------------------------------------------------------------- - assign tlb_bus_if.rsp_ready = 1'b1; - wire fill_fire = tlb_bus_if.rsp_valid && tlb_bus_if.rsp_ready; - wire [BANK_W-1:0] fill_bank = BANK_W'(tlb_bus_if.rsp_data.id); - wire fill_ok = fill_fire && !tlb_bus_if.rsp_data.fault - && !bk_stale[fill_bank] && !flush; - - assign install_entry = '{ - level: tlb_bus_if.rsp_data.level, - vpn: bk_vpn[fill_bank], - ppn: tlb_bus_if.rsp_data.ppn, - flags: tlb_bus_if.rsp_data.flags - }; - for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_install - assign bank_install_valid[b] = fill_ok && (fill_bank == BANK_W'(b)); - end - - // --------------------------------------------------------------------- - // Drain: one bank per cycle, round-robin; faulted walks kill. - // --------------------------------------------------------------------- - reg [BANK_W-1:0] drain_rr; - logic [BANK_W-1:0] drain_sel; - logic drain_any; - always @(*) begin - drain_sel = '0; - drain_any = 1'b0; - for (int i = NUM_BANKS-1; i >= 0; --i) begin - automatic logic [BANK_W-1:0] b = BANK_W'(int'(drain_rr) + i + 1); - if (bk_state[b] == B_DRAIN) begin - drain_sel = b; - drain_any = 1'b1; - end - end - end - - assign replay_valid = drain_any && !bk_fault[drain_sel]; - assign replay_payload = bk_payload[drain_sel]; - assign replay_ppn = bk_ppn[drain_sel]; - assign replay_level = bk_level[drain_sel]; - assign replay_flags = bk_flags[drain_sel]; - assign kill_valid = drain_any && bk_fault[drain_sel]; - wire drain_fire = drain_any && (bk_fault[drain_sel] ? kill_ready : replay_ready); - - // Structural faults surface as the kill drains. - assign mshr_fault_valid = kill_valid && kill_ready; - assign mshr_fault_vpn = bk_vpn[drain_sel]; - assign mshr_fault_access = bk_access[drain_sel]; - - // --------------------------------------------------------------------- - // Bank state - // --------------------------------------------------------------------- - always @(posedge clk) begin - if (reset) begin - for (int b = 0; b < NUM_BANKS; ++b) begin - bk_state[b] <= B_IDLE; - bk_stale[b] <= 1'b0; - end - issue_rr <= '0; - drain_rr <= '0; - end else begin - if (park_fire) begin - bk_state [park_bank] <= B_WALK_REQ; - bk_vpn [park_bank] <= park_vpn; - bk_access [park_bank] <= park_access; - bk_amo [park_bank] <= park_amo; - bk_payload[park_bank] <= park_payload; - bk_stale [park_bank] <= 1'b0; - end - if (issue_fire) begin - bk_state[issue_sel] <= B_WALK_WAIT; - issue_rr <= issue_sel; - end - if (fill_fire) begin - if (bk_stale[fill_bank]) begin - // stale walk: discard the result and walk again - bk_state[fill_bank] <= B_WALK_REQ; - bk_stale[fill_bank] <= 1'b0; - end else begin - bk_state[fill_bank] <= B_DRAIN; - bk_fault[fill_bank] <= tlb_bus_if.rsp_data.fault; - bk_ppn [fill_bank] <= tlb_bus_if.rsp_data.ppn; - bk_level[fill_bank] <= tlb_bus_if.rsp_data.level; - bk_flags[fill_bank] <= tlb_bus_if.rsp_data.flags; - end - end - if (drain_fire) begin - bk_state[drain_sel] <= B_IDLE; - drain_rr <= drain_sel; - end - if (flush) begin - for (int b = 0; b < NUM_BANKS; ++b) begin - if (bk_state[b] == B_WALK_WAIT) begin - bk_stale[b] <= 1'b1; - end - end - end - end - end - - logic any_busy; - always @(*) begin - any_busy = 1'b0; - for (int b = 0; b < NUM_BANKS; ++b) begin - if (bk_state[b] != B_IDLE) any_busy = 1'b1; - end - end - assign empty = !any_busy; - - // --------------------------------------------------------------------- - // Performance counters - // --------------------------------------------------------------------- -`ifdef PERF_ENABLE - reg [PERF_CTR_BITS-1:0] perf_reads, perf_hits, perf_misses, perf_evicts; - logic [`CLOG2(NUM_REQS+1)-1:0] reads_now, hits_now; - always @(*) begin - reads_now = '0; - hits_now = '0; - for (int l = 0; l < NUM_REQS; ++l) begin - if (lane_grant[l]) reads_now = reads_now + 1; - if (lookup_hit[l] && access_hit[l]) hits_now = hits_now + 1; - end - end - always @(posedge clk) begin - if (reset) begin - perf_reads <= '0; - perf_hits <= '0; - perf_misses <= '0; - perf_evicts <= '0; - end else begin - perf_reads <= perf_reads + PERF_CTR_BITS'(reads_now); - perf_hits <= perf_hits + PERF_CTR_BITS'(hits_now); - perf_misses <= perf_misses + PERF_CTR_BITS'(park_fire); - perf_evicts <= perf_evicts + PERF_CTR_BITS'((| (bank_install_valid & bank_install_evict))); - end - end - assign mmu_perf.tlb_reads = perf_reads; - assign mmu_perf.tlb_hits = perf_hits; - assign mmu_perf.tlb_misses = perf_misses; - assign mmu_perf.tlb_evictions = perf_evicts; - // Every parked miss issues exactly one walk (re-walks after a flush are - // counted again on issue). - assign mmu_perf.ptw_walks = perf_misses; - assign mmu_perf.ptw_latency = '0; -`endif - -endmodule From a06d26b998bc5d7056f23490bcef0f16fbbf019b Mon Sep 17 00:00:00 2001 From: Thomas Weatherly Date: Tue, 1 Sep 2026 18:40:38 -0400 Subject: [PATCH 3/5] hw: move the page-table walker to the device level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lift the per-cluster VX_ptw to one shared walker at the device, the shared-walker organization from the GPU-MMU literature. Each cluster exports its L2 TLB's miss bus (new dev_ptw_if port); a VX_tlb_bus_arb folds the clusters into the walker, whose PTE fetches ride a dedicated LLC client port (per_cluster_mem_bus_if[L3_PTW_IDX]) instead of competing with the sockets inside each cluster's L2 cache — the L2 PTE-fetch client slot is removed outright (L2_PTW_REQS = 0), returning that arbitration bandwidth to the cores. The walker's flush-done leg joins the DCR done-tree through cluster 0's slot, and its structural faults land on cluster 0's fault lines (the DCR fault latch is device-global, so attribution is preserved). One walk cache now serves all clusters. Topologically identical to the per-cluster walker at NUM_CLUSTERS=1; the organizations diverge at 2+ clusters. The MMU synthesis sandbox (hw/unittest/vm) follows: walker behind the device arb, PTE fetches exiting at a dedicated top-level port at the production LLC-client boundary. Validated together with the banked L1: full vm catalog green on rtlsim-32 (23), simx-32 (21, incl. model parity) and rtlsim-64 (23); demo at 1 and 2 clusters; -Wall clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DJk9ipw4g1ZP3zZShiYaMq --- hw/rtl/VX_cluster.sv | 62 +++++++++++---------------- hw/rtl/VX_gpu_pkg.sv | 11 +++-- hw/rtl/Vortex.sv | 85 ++++++++++++++++++++++++++++++++++--- hw/unittest/vm/VX_vm_top.sv | 84 ++++++++++++++++++++++++++++-------- 4 files changed, 178 insertions(+), 64 deletions(-) diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 0f6e001ece..4a323439c3 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -45,6 +45,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_VM_ENABLE // Device MMU sideband from the top DCR surface. input wire [`VX_CFG_XLEN-1:0] mmu_satp, + // Device-level walker: the L2 TLB's miss bus, exported to the device. + VX_tlb_bus_if.master dev_ptw_if, input wire mmu_flush_req, output wire mmu_flush_done, output wire mmu_fault_valid, @@ -328,7 +330,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_tlb_flush_if l2_flush_if (); VX_tlb_flush_if ptw_flush_if (); - wire l2_empty, ptw_empty; + wire l2_empty; VX_tlb_l2 #( .INSTANCE_ID (`SFORMATF(("%s-l2tlb", INSTANCE_ID))), @@ -342,44 +344,30 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .empty (l2_empty) ); - VX_mmu_fault_if ptw_fault_if (); - VX_mem_bus_if #( - .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), - .TAG_WIDTH (L2_TAG_WIDTH) - ) ptw_mem_if (); - - VX_ptw #( - .ID_WIDTH (L2_TLB_SLOT_WIDTH), - .MEM_TAG_WIDTH (L2_TAG_WIDTH) - ) ptw ( - .clk (clk), - .reset (reset), - .satp (mmu_satp), - .miss_if (l2_ptw_if), - .mem_bus_if (ptw_mem_if), - .flush_if (ptw_flush_if), - .fault_if (ptw_fault_if), - .empty (ptw_empty) - ); + // The walker lives at the device: export the L2 TLB's miss bus. The + // L2 cache carries no PTE-fetch client; its bandwidth is all sockets'. + assign dev_ptw_if.req_valid = l2_ptw_if.req_valid; + assign dev_ptw_if.req_data = l2_ptw_if.req_data; + assign l2_ptw_if.req_ready = dev_ptw_if.req_ready; + assign l2_ptw_if.rsp_valid = dev_ptw_if.rsp_valid; + assign l2_ptw_if.rsp_data = dev_ptw_if.rsp_data; + assign dev_ptw_if.rsp_ready = l2_ptw_if.rsp_ready; + + // Flush root fans to the cluster L2; each socket self-times its own L1 + // TLB flush off the SATP DCR write, and the device walker reports its + // own done leg at the top. + assign l2_flush_if.req = mmu_flush_req; + assign mmu_flush_done = l2_flush_if.done; - // PTE fetches attach as one more L2-cache client (like ocache/rcache). - `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_PTW_IDX], ptw_mem_if); + // Structural faults surface at the device walker; L1 permission faults + // are not reported here — cluster_tlb_bus_if stays a pure translation + // fabric across the socket boundary. + assign mmu_fault_valid = 1'b0; + assign mmu_fault_va = '0; + assign mmu_fault_access = 2'b0; + assign mmu_fault_amo = 1'b0; - // Flush root fans to the cluster L2 + walker; each socket self-times its own - // L1 TLB flush off the SATP DCR write, so only these two legs report done. - assign l2_flush_if.req = mmu_flush_req; - assign ptw_flush_if.req = mmu_flush_req; - assign mmu_flush_done = l2_flush_if.done && ptw_flush_if.done; - - // Only the shared walker's structural faults are surfaced. L1 permission - // faults are not reported: cluster_tlb_bus_if is the sole MMU signal - // crossing the socket boundary, and it stays a pure translation fabric. - assign mmu_fault_valid = ptw_fault_if.valid; - assign mmu_fault_va = ptw_fault_if.va; - assign mmu_fault_access = ptw_fault_if.access; - assign mmu_fault_amo = ptw_fault_if.amo; - - wire mmu_busy = ~l2_empty || ~ptw_empty; + wire mmu_busy = ~l2_empty; `else wire mmu_busy = 1'b0; `endif diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 978e902505..3a0327eedc 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1704,8 +1704,7 @@ package VX_gpu_pkg; // The shared page-table walker attaches one PTE-fetch port under VM, right // after the socket and graphics ports (like ocache/rcache). - localparam L2_PTW_REQS = `VX_CFG_VM_ENABLED; - localparam L2_PTW_IDX = L2_SOCKET_REQS + L2_GFX_REQS; + localparam L2_PTW_REQS = 0; // walker is device-level (LLC client) localparam L2_NUM_REQS = L2_SOCKET_REQS + L2_GFX_REQS + L2_PTW_REQS; @@ -1720,6 +1719,8 @@ package VX_gpu_pkg; localparam TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + `ARB_SEL_BITS(2, 1); localparam TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + `ARB_SEL_BITS(NUM_SOCKETS, 1); localparam L2_TLB_SLOT_WIDTH = `CLOG2(`VX_CFG_L2_TLB_MSHR_SIZE); + // Device-level walker: cluster L2-TLB miss buses arb into one walker. + localparam TLB_DEV_ID_WIDTH = L2_TLB_SLOT_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_CLUSTERS, 1); // Memory request data bits (mem transacts in sectors) localparam L2_MEM_DATA_WIDTH = (L2_SECTOR_SIZE * 8); @@ -1743,7 +1744,11 @@ package VX_gpu_pkg; localparam L3_SECTOR_SIZE = `VX_CFG_L3_SECTOR_SIZE; // Input request size - localparam L3_NUM_REQS = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS; + // The device-level walker attaches its PTE fetches as one more LLC + // client on the last requestor slot. + localparam L3_PTW_IDX = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS; + localparam L3_NUM_REQS = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS + + `VX_CFG_VM_ENABLED; // Core request tag bits localparam L3_TAG_WIDTH = L2_MEM_TAG_WIDTH; diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index abf7c8dadd..7602d0bde6 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -117,7 +117,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( VX_mem_bus_if #( .DATA_SIZE (L2_SECTOR_SIZE), .TAG_WIDTH (L3_TAG_WIDTH) - ) per_cluster_mem_bus_if[`VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS](); + ) per_cluster_mem_bus_if[L3_NUM_REQS](); VX_mem_bus_if #( .DATA_SIZE (L3_SECTOR_SIZE), @@ -219,6 +219,78 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( wire [`VX_CFG_NUM_CLUSTERS-1:0][1:0] cl_mmu_fault_access; wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_fault_amo; + // Per-cluster L2-TLB miss export buses (used by the device-level walker). + VX_tlb_bus_if #(.ID_WIDTH (L2_TLB_SLOT_WIDTH)) per_cluster_dev_ptw_if [`VX_CFG_NUM_CLUSTERS] (); + wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_flush_done_in; + + // One shared walker at the device: the clusters' L2-TLB miss buses arb + // into it, PTE fetches ride a dedicated LLC client port, and the flush + // done-tree gains the walker's leg. Structural faults surface here. + VX_tlb_bus_if #(.ID_WIDTH (TLB_DEV_ID_WIDTH)) dev_ptw_bus_if (); + + VX_tlb_bus_arb #( + .NUM_INPUTS (`VX_CFG_NUM_CLUSTERS), + .ID_WIDTH_IN (L2_TLB_SLOT_WIDTH), + .OUT_BUF (3) + ) tlb_dev_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (per_cluster_dev_ptw_if), + .bus_out_if (dev_ptw_bus_if) + ); + + VX_tlb_flush_if dev_ptw_flush_if (); + VX_mmu_fault_if dev_ptw_fault_if (); + wire dev_ptw_empty; + `UNUSED_VAR (dev_ptw_empty) + + VX_mem_bus_if #( + .DATA_SIZE (L2_SECTOR_SIZE), + .TAG_WIDTH (L3_TAG_WIDTH) + ) dev_ptw_mem_if (); + + VX_ptw #( + .ID_WIDTH (TLB_DEV_ID_WIDTH), + .DATA_SIZE (L2_SECTOR_SIZE), + .MEM_TAG_WIDTH (L3_TAG_WIDTH) + ) dev_ptw ( + .clk (clk), + .reset (reset), + .satp (mmu_satp), + .miss_if (dev_ptw_bus_if), + .mem_bus_if (dev_ptw_mem_if), + .flush_if (dev_ptw_flush_if), + .fault_if (dev_ptw_fault_if), + .empty (dev_ptw_empty) + ); + + `ASSIGN_VX_MEM_BUS_IF (per_cluster_mem_bus_if[L3_PTW_IDX], dev_ptw_mem_if); + + assign dev_ptw_flush_if.req = mmu_flush_req; + // The walker's done leg joins cluster 0's slot of the done-tree; the + // fault report likewise lands on cluster 0's lines (the DCR fault latch + // is device-global, so attribution is not lost). + for (genvar c = 0; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_dev_flush_done + assign cl_mmu_flush_done_in[c] = (c == 0) + ? (cl_mmu_flush_done[c] && dev_ptw_flush_if.done) + : cl_mmu_flush_done[c]; + end + wire mmu_fault_valid_in = dev_ptw_fault_if.valid; + wire [`VX_CFG_XLEN-1:0] mmu_fault_va_in = dev_ptw_fault_if.va; + wire [1:0] mmu_fault_access_in = dev_ptw_fault_if.access; + wire mmu_fault_amo_in = dev_ptw_fault_if.amo; + + wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_fault_valid_in; + wire [`VX_CFG_NUM_CLUSTERS-1:0][`VX_CFG_XLEN-1:0] cl_mmu_fault_va_in; + wire [`VX_CFG_NUM_CLUSTERS-1:0][1:0] cl_mmu_fault_access_in; + wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_fault_amo_in; + for (genvar c = 0; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_dev_fault_mux + assign cl_mmu_fault_valid_in[c] = cl_mmu_fault_valid[c] || ((c == 0) && mmu_fault_valid_in); + assign cl_mmu_fault_va_in[c] = ((c == 0) && mmu_fault_valid_in) ? mmu_fault_va_in : cl_mmu_fault_va[c]; + assign cl_mmu_fault_access_in[c] = ((c == 0) && mmu_fault_valid_in) ? mmu_fault_access_in : cl_mmu_fault_access[c]; + assign cl_mmu_fault_amo_in[c] = ((c == 0) && mmu_fault_valid_in) ? mmu_fault_amo_in : cl_mmu_fault_amo[c]; + end + VX_mmu_dcr mmu_dcr ( .clk (clk), .reset (reset), @@ -226,11 +298,11 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .dcr_bus_out_if (dcr_cluster_src_if), .satp (mmu_satp), .flush_req (mmu_flush_req), - .cluster_flush_done (cl_mmu_flush_done), - .cluster_fault_valid (cl_mmu_fault_valid), - .cluster_fault_va (cl_mmu_fault_va), - .cluster_fault_access (cl_mmu_fault_access), - .cluster_fault_amo (cl_mmu_fault_amo) + .cluster_flush_done (cl_mmu_flush_done_in), + .cluster_fault_valid (cl_mmu_fault_valid_in), + .cluster_fault_va (cl_mmu_fault_va_in), + .cluster_fault_access (cl_mmu_fault_access_in), + .cluster_fault_amo (cl_mmu_fault_amo_in) ); `else assign dcr_cluster_src_if.req_valid = dcr_bus_if.req_valid; @@ -278,6 +350,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `ifdef VX_CFG_VM_ENABLE .mmu_satp (mmu_satp), + .dev_ptw_if (per_cluster_dev_ptw_if[cluster_id]), .mmu_flush_req (mmu_flush_req), .mmu_flush_done (cl_mmu_flush_done[cluster_id]), .mmu_fault_valid (cl_mmu_fault_valid[cluster_id]), diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 89991d3e8f..7ac7aec7f0 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -27,13 +27,16 @@ // per cluster: // VX_tlb_bus_arb : folds all sockets' miss ports -> the L2 TLB client bus // VX_tlb_l2 : shared L2 TLB -// VX_ptw : page-table walker (fed satp; PTE reads through the L2) -// VX_cache_wrap : the shared L2, fed by the sockets' L1 mem ports + the PTW +// at the device (topologically identical at one cluster): +// VX_tlb_bus_arb : folds the clusters' L2-TLB miss buses -> the walker +// VX_ptw : device-level page-table walker (fed satp; PTE reads on +// a dedicated LLC-client port, NOT through the L2) +// VX_cache_wrap : the shared L2, fed by the sockets' L1 mem ports // -// Memory hierarchy: the L1 dcaches and the PTW are clients of the shared L2 -// (the real L2_NUM_REQS client set, minus graphics), so the L2 input-arb and -// its PTE-fetch boundary carry realistic timing. The icaches' memory side and -// the L2's memory side exit to top-level ports. +// Memory hierarchy: the L1 dcaches are clients of the shared L2 (the real +// L2_NUM_REQS client set, minus graphics), so the L2 input-arb carries +// realistic timing. The icaches' memory side, the L2's memory side, and the +// walker's LLC-client port exit to top-level ports. // // Fidelity: every MMU/PTW timing boundary is present and loaded by its real // neighbour (MMU->real L1 cache, PTW->real L2). What is removed (cores/FPU/ @@ -118,7 +121,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( output wire [IC_MEM_PORTS-1:0] ic_mem_rsp_ready, // ----------------------------------------------------------------------- - // L2 memory side (master): L1 dcache + PTE traffic reach memory here + // L2 memory side (master): L1 dcache traffic reaches memory here // ----------------------------------------------------------------------- output wire [L2_MEM_PORTS-1:0] l2_mem_req_valid, output wire [L2_MEM_PORTS-1:0] l2_mem_req_rw, @@ -133,6 +136,23 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( input wire [L2_MEM_PORTS-1:0][L2_MEM_TAG_WIDTH-1:0] l2_mem_rsp_tag, output wire [L2_MEM_PORTS-1:0] l2_mem_rsp_ready, + // ----------------------------------------------------------------------- + // Device walker LLC-client port (master): PTE fetches exit here, at the + // boundary where production attaches per_cluster_mem_bus_if[L3_PTW_IDX] + // ----------------------------------------------------------------------- + output wire ptw_mem_req_valid, + output wire ptw_mem_req_rw, + output wire [L2_SECTOR_SIZE-1:0] ptw_mem_req_byteen, + output wire [L2_MEM_ADDR_W-1:0] ptw_mem_req_addr, + output wire [L3_TAG_WIDTH-1:0] ptw_mem_req_tag, + output wire [L2_SECTOR_SIZE*8-1:0] ptw_mem_req_data, + input wire ptw_mem_req_ready, + + input wire ptw_mem_rsp_valid, + input wire [L2_SECTOR_SIZE*8-1:0] ptw_mem_rsp_data, + input wire [L3_TAG_WIDTH-1:0] ptw_mem_rsp_tag, + output wire ptw_mem_rsp_ready, + // ----------------------------------------------------------------------- // Page-fault sideband (from the walker) + drain // ----------------------------------------------------------------------- @@ -448,7 +468,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // Cluster TLB + walker tail // ======================================================================== VX_tlb_bus_if #(.ID_WIDTH (TLB_CLUSTER_ID_WIDTH)) l2_client_if (); - VX_tlb_bus_if #(.ID_WIDTH (L2_TLB_SLOT_WIDTH)) l2_ptw_if (); + // one cluster's L2-TLB miss bus, as the device arb's input array + VX_tlb_bus_if #(.ID_WIDTH (L2_TLB_SLOT_WIDTH)) dev_ptw_in_if [1] (); VX_tlb_bus_arb #( .NUM_INPUTS (NS), @@ -470,39 +491,68 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .client_if (l2_client_if), - .ptw_if (l2_ptw_if), + .ptw_if (dev_ptw_in_if[0]), .flush_if (l2_flush_if), .empty (l2tlb_empty) ); + // Device-level walker tail: the arb is a passthrough at one cluster but + // keeps the production boundary (and its ID growth) in the netlist. + VX_tlb_bus_if #(.ID_WIDTH (TLB_DEV_ID_WIDTH)) dev_ptw_bus_if (); + + VX_tlb_bus_arb #( + .NUM_INPUTS (1), + .ID_WIDTH_IN (L2_TLB_SLOT_WIDTH), + .OUT_BUF (3) + ) tlb_dev_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (dev_ptw_in_if), + .bus_out_if (dev_ptw_bus_if) + ); + VX_mmu_fault_if ptw_fault_if (); VX_mem_bus_if #( - .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), - .TAG_WIDTH (L2_TAG_WIDTH) + .DATA_SIZE (L2_SECTOR_SIZE), + .TAG_WIDTH (L3_TAG_WIDTH) ) ptw_mem_if (); VX_ptw #( - .ID_WIDTH (L2_TLB_SLOT_WIDTH), - .MEM_TAG_WIDTH (L2_TAG_WIDTH) + .ID_WIDTH (TLB_DEV_ID_WIDTH), + .DATA_SIZE (L2_SECTOR_SIZE), + .MEM_TAG_WIDTH (L3_TAG_WIDTH) ) ptw ( .clk (clk), .reset (reset), .satp (satp), - .miss_if (l2_ptw_if), + .miss_if (dev_ptw_bus_if), .mem_bus_if (ptw_mem_if), .flush_if (ptw_flush_if), .fault_if (ptw_fault_if), .empty (ptw_empty) ); + assign ptw_mem_req_valid = ptw_mem_if.req_valid; + assign ptw_mem_req_rw = ptw_mem_if.req_data.rw; + assign ptw_mem_req_byteen = ptw_mem_if.req_data.byteen; + assign ptw_mem_req_addr = ptw_mem_if.req_data.addr; + assign ptw_mem_req_tag = ptw_mem_if.req_data.tag; + assign ptw_mem_req_data = ptw_mem_if.req_data.data; + assign ptw_mem_if.req_ready = ptw_mem_req_ready; + assign ptw_mem_if.rsp_valid = ptw_mem_rsp_valid; + assign ptw_mem_if.rsp_data.data = ptw_mem_rsp_data; + assign ptw_mem_if.rsp_data.tag = ptw_mem_rsp_tag; + assign ptw_mem_rsp_ready = ptw_mem_if.rsp_ready; + `UNUSED_VAR (ptw_mem_if.req_data.attr) + assign fault_valid = ptw_fault_if.valid; assign fault_va = ptw_fault_if.va; assign fault_access = ptw_fault_if.access; assign fault_amo = ptw_fault_if.amo; // ======================================================================== - // Shared L2: sockets' L1 dcache mem ports + the PTW (the real client set, - // minus graphics) -> memory ports + // Shared L2: sockets' L1 dcache mem ports (the real client set, minus + // graphics; the walker is an LLC client, not an L2 client) -> memory ports // ======================================================================== VX_mem_bus_if #( .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), @@ -513,8 +563,6 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( if (i < L2_SOCKET_REQS) begin : g_socket // socket L1 mem port (dcache), widened to the L2 client tag width `ASSIGN_VX_MEM_BUS_IF_EX (l2_core_if[i], all_dc_mem_if[i], L2_TAG_WIDTH, DCACHE_MEM_TAG_WIDTH, UUID_WIDTH); - end else if (i == L2_PTW_IDX) begin : g_ptw - `ASSIGN_VX_MEM_BUS_IF (l2_core_if[i], ptw_mem_if); end else begin : g_tie // graphics client slots (unused in this DUT) assign l2_core_if[i].req_valid = 1'b0; From d38cdf9bf65fb49b367fc91a5334c297c6c5d82c Mon Sep 17 00:00:00 2001 From: Thomas Weatherly Date: Tue, 1 Sep 2026 20:26:22 -0400 Subject: [PATCH 4/5] hw: banked L1 keeps the shared miss station The banked VX_tlb_l1 had ported the old per-bank parked-miss slot, which dropped MMU v2's non-blocking miss handling (multiple outstanding walks, same-VPN dedup, hit-under-miss inside a bank). Put VX_tlb_mshr back behind the banked CAMs, wired as master wires it; the MSHR install routes to the bank of the walked VPN. MSHR_SIZE/REPLAY_DEPTH return as VX_mmu parameters and the walk id is the MSHR slot again. The L1 proposal is now only the banked entry array: single-lookup-port banks selected by low VPN bits (bank_conflict holds losing lanes) in front of an unchanged miss station. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DJk9ipw4g1ZP3zZShiYaMq --- VX_config.toml | 5 +- hw/rtl/vm/VX_mmu.sv | 20 +-- hw/rtl/vm/VX_tlb_l1.sv | 342 ++++++++++++++++------------------------- 3 files changed, 148 insertions(+), 219 deletions(-) diff --git a/VX_config.toml b/VX_config.toml index 888b83a1b7..7834c35ac1 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -141,8 +141,9 @@ VX_CFG_NUM_VPU_BLOCKS = 1 # VM page-table format is a HW<->SW contract, moved to VX_types.toml [vm]; TLB depth stays here. VX_CFG_TLB_SIZE = 32 -# L1 TLB banks (power of two): per-bank single lookup ports with one -# parked miss per bank; hits in other banks proceed under a miss. +# L1 TLB banks (power of two, dividing the TLB sizes): the entry array is +# split by low VPN bits into single-lookup-port banks; lanes contending for +# a bank serialize, misses go to the shared miss station below. VX_CFG_L1_TLB_NUM_BANKS = 4 # L1 TLB stage (per core): D-side / I-side entries and the per-instance diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index 27acc4df48..1967b2a127 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -25,7 +25,8 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( parameter `STRING INSTANCE_ID = "", parameter NUM_REQS = DCACHE_NUM_REQS, parameter TLB_SIZE = `VX_CFG_DTLB_SIZE, - parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, // walk-ID space (>= NUM_BANKS) + parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, + parameter REPLAY_DEPTH = 2, parameter EXEC_SIDE = 0, parameter DATA_SIZE = DCACHE_WORD_SIZE, parameter TAG_WIDTH = DCACHE_TAG_WIDTH_BASE, @@ -146,14 +147,16 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [NUM_REQS-1:0] bank_conflict; - // L1 storage: banked — per-bank lookup port + one parked miss per bank; - // losing lanes hold via bank_conflict. + // L1 storage: banked single-port CAMs (losing lanes hold via + // bank_conflict) in front of the shared non-blocking miss station. VX_tlb_l1 #( - .NUM_REQS (NUM_REQS), - .TLB_SIZE (TLB_SIZE), - .NUM_BANKS (`VX_CFG_L1_TLB_NUM_BANKS), - .PAYLOAD_W (PAYLOAD_W), - .ID_WIDTH (ID_WIDTH) + .NUM_REQS (NUM_REQS), + .TLB_SIZE (TLB_SIZE), + .NUM_BANKS (`VX_CFG_L1_TLB_NUM_BANKS), + .MSHR_SIZE (MSHR_SIZE), + .REPLAY_DEPTH (REPLAY_DEPTH), + .PAYLOAD_W (PAYLOAD_W), + .ID_WIDTH (ID_WIDTH) ) tlb ( .clk (clk), .reset (reset), @@ -191,7 +194,6 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .empty (tlb_empty) ); - // --------------------------------------------------------------------- // Per-lane request category (mutually exclusive, by priority) // --------------------------------------------------------------------- diff --git a/hw/rtl/vm/VX_tlb_l1.sv b/hw/rtl/vm/VX_tlb_l1.sv index 10ab8b54bc..a640b86e1e 100644 --- a/hw/rtl/vm/VX_tlb_l1.sv +++ b/hw/rtl/vm/VX_tlb_l1.sv @@ -1,23 +1,38 @@ // Copyright © 2019-2023 -// Licensed under the Apache License, Version 2.0. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. `include "VX_define.vh" -// Banked L1 TLB storage + miss station behind the VX_mmu parent contract. -// The entry array is split into NUM_BANKS single-ported banks selected by -// the low VPN bits: each bank answers at most one lane per cycle -// (bank_conflict tells the parent to hold the other lanes), and each bank -// holds at most one parked miss — a miss blocks its bank for the walk's -// duration while the other banks keep hitting. A same-VPN request waits on -// its bank's in-flight walk (mshr_match) instead of joining a queue. -// Trades a full multi-port CAM + shared MSHR for per-bank ports and slots: -// cheaper lookup hardware at scale, one outstanding walk per bank. +// L1 TLB storage: the pure lookup + miss-handling core, with no address +// translation of its own. The entry array is banked: NUM_BANKS CAMs selected +// by the low VPN bits, each with a single lookup port, so a lookup costs +// TLB_SIZE comparators in total instead of NUM_REQS x TLB_SIZE. Lanes that +// contend for one bank are serialized (bank_conflict tells the parent to +// hold the losers; lowest lane wins); lanes on different banks proceed in +// the same cycle. Misses go to the shared non-blocking miss station +// (`VX_tlb_mshr`: park / dedup / replay / kill), so a miss never blocks its +// bank and same-VPN requests join the in-flight walk. The fill installs into +// the bank of the walked VPN. The parent `VX_mmu` drives the VPN probes and +// consumes the raw lookup results (PPN, flags) plus the replay/kill streams, +// doing the VA→PA splice and permission checks itself. module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter NUM_REQS = DCACHE_NUM_REQS, - parameter TLB_SIZE = `VX_CFG_DTLB_SIZE, - parameter NUM_BANKS = 4, - parameter PAYLOAD_W = 1, - parameter ID_WIDTH = `CLOG2(NUM_BANKS) + parameter NUM_REQS = DCACHE_NUM_REQS, + parameter TLB_SIZE = `VX_CFG_DTLB_SIZE, + parameter NUM_BANKS = `VX_CFG_L1_TLB_NUM_BANKS, + parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, + parameter REPLAY_DEPTH = 2, + parameter PAYLOAD_W = 1, + parameter ID_WIDTH = `CLOG2(MSHR_SIZE) ) ( input wire clk, input wire reset, @@ -26,9 +41,9 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( output mmu_perf_t mmu_perf, `endif - // Per-lane lookup. A lane that loses its bank's port this cycle gets - // bank_conflict (parent must hold it); hit/miss is only meaningful for - // lanes with bank_conflict == 0. + // Per-lane combinational lookup (VPN in, raw translation out). A lane + // that loses its bank's port this cycle gets bank_conflict (parent must + // hold it); hit/miss is only meaningful for lanes with bank_conflict == 0. input wire [NUM_REQS-1:0][TLB_VPN_WIDTH-1:0] lookup_vpn, input wire [NUM_REQS-1:0] lookup_valid, output wire [NUM_REQS-1:0] lookup_hit, @@ -59,12 +74,12 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( output wire kill_valid, input wire kill_ready, - // Structural-fault sideband. + // Structural-fault sideband (from the miss station). output wire mshr_fault_valid, output wire [TLB_VPN_WIDTH-1:0] mshr_fault_vpn, output tlb_access_e mshr_fault_access, - // Miss/fill fabric to the shared walker complex (id = bank index). + // Miss/fill fabric to the shared walker complex. VX_tlb_bus_if.master tlb_bus_if, input wire flush, @@ -72,7 +87,6 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( ); `STATIC_ASSERT(`IS_POW2(NUM_BANKS), ("NUM_BANKS must be a power of 2")) `STATIC_ASSERT((TLB_SIZE % NUM_BANKS) == 0, ("NUM_BANKS must divide TLB_SIZE")) - `STATIC_ASSERT(ID_WIDTH >= `CLOG2(NUM_BANKS), ("bank index must fit the bus id")) localparam ENTRIES_PER_BANK = TLB_SIZE / NUM_BANKS; localparam BANK_W = `UP(`CLOG2(NUM_BANKS)); @@ -83,26 +97,6 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( else bank_of = vpn_lo; endfunction - // --------------------------------------------------------------------- - // Per-bank parked-miss slot - // --------------------------------------------------------------------- - typedef enum logic [1:0] { - B_IDLE, B_WALK_REQ, B_WALK_WAIT, B_DRAIN - } bank_state_e; - - bank_state_e bk_state [NUM_BANKS]; - logic [TLB_VPN_WIDTH-1:0] bk_vpn [NUM_BANKS]; - tlb_access_e bk_access [NUM_BANKS]; - logic bk_amo [NUM_BANKS]; - logic [PAYLOAD_W-1:0] bk_payload [NUM_BANKS]; - logic bk_fault [NUM_BANKS]; - logic [TLB_PPN_WIDTH-1:0] bk_ppn [NUM_BANKS]; - logic [TLB_LEVEL_WIDTH-1:0] bk_level [NUM_BANKS]; - logic [TLB_FLAGS_WIDTH-1:0] bk_flags [NUM_BANKS]; - // A walk in flight when the flush arrived resolved against the old page - // table: drop its fill and re-walk (see the shared walker's discipline). - logic bk_stale [NUM_BANKS]; - // --------------------------------------------------------------------- // Bank lookup port arbitration: lowest contending lane wins the bank. // --------------------------------------------------------------------- @@ -127,16 +121,19 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( end // --------------------------------------------------------------------- - // Entry storage: one CAM per bank, granted lane only. + // Entry array: one single-port CAM per bank, granted lane only. // --------------------------------------------------------------------- + wire install_valid; + tlb_entry_t install_entry; + wire [BANK_W-1:0] install_bank = bank_of(install_entry.vpn[BANK_W-1:0]); + wire [NUM_BANKS-1:0] bank_install_valid; - tlb_entry_t install_entry; + wire [NUM_BANKS-1:0] bank_install_evict; wire [NUM_BANKS-1:0] bank_lookup_hit; wire [NUM_BANKS-1:0][TLB_PPN_WIDTH-1:0] bank_lookup_ppn; wire [NUM_BANKS-1:0][TLB_FLAGS_WIDTH-1:0] bank_lookup_flags; wire [NUM_BANKS-1:0][TLB_VPN_WIDTH-1:0] bank_lookup_vpn; wire [NUM_BANKS-1:0] bank_access_hit; - wire [NUM_BANKS-1:0] bank_install_evict; `ifndef PERF_ENABLE `UNUSED_VAR (bank_install_evict) `endif @@ -151,8 +148,9 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( end end end - assign bank_lookup_vpn[b] = lookup_vpn[owner]; - assign bank_access_hit[b] = access_hit[owner] && (lane_bank[owner] == BANK_W'(b)); + assign bank_lookup_vpn[b] = lookup_vpn[owner]; + assign bank_access_hit[b] = access_hit[owner] && (lane_bank[owner] == BANK_W'(b)); + assign bank_install_valid[b] = install_valid && (install_bank == BANK_W'(b)); VX_tlb_cam #( .NUM_REQS (1), @@ -178,193 +176,121 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( assign lookup_hit[l] = lane_grant[l] && bank_lookup_hit[lane_bank[l]]; assign lookup_ppn[l] = bank_lookup_ppn[lane_bank[l]]; assign lookup_flags[l] = bank_lookup_flags[lane_bank[l]]; - // A lane whose VPN matches its bank's in-flight walk waits for that - // fill rather than re-walking (categorized cat_park; park_ready holds - // it off until the bank drains and the entry installs). - assign mshr_match[l] = lane_grant[l] - && (bk_state[lane_bank[l]] != B_IDLE) - && (bk_vpn[lane_bank[l]] == lookup_vpn[l]); end // --------------------------------------------------------------------- - // Park: one slot per bank; the bank must be idle. + // Miss station (shared across banks) // --------------------------------------------------------------------- - wire [BANK_W-1:0] park_bank = bank_of(park_vpn[BANK_W-1:0]); - assign park_ready = !flush && (bk_state[park_bank] == B_IDLE); - wire park_fire = park_valid && park_ready; - `UNUSED_VAR (park_lane) - - // --------------------------------------------------------------------- - // Walk issue: round-robin over banks in B_WALK_REQ. - // --------------------------------------------------------------------- - reg [BANK_W-1:0] issue_rr; - logic [BANK_W-1:0] issue_sel; - logic issue_any; - always @(*) begin - issue_sel = '0; - issue_any = 1'b0; - for (int i = NUM_BANKS-1; i >= 0; --i) begin - automatic logic [BANK_W-1:0] b = BANK_W'(int'(issue_rr) + i + 1); - if (bk_state[b] == B_WALK_REQ) begin - issue_sel = b; - issue_any = 1'b1; - end - end - end + wire [`UP(ID_WIDTH)-1:0] tlb_req_id; + tlb_access_e tlb_req_access; + wire tlb_req_amo; + wire [TLB_VPN_WIDTH-1:0] tlb_req_vpn; + + // L1 drain glue: faulted entries drain as kills, live ones as replays. + wire drain_valid; + wire [PAYLOAD_W-1:0] drain_qdata; + wire drain_fault; + wire [TLB_PPN_WIDTH-1:0] drain_ppn; + wire [TLB_LEVEL_WIDTH-1:0] drain_level; + wire [TLB_FLAGS_WIDTH-1:0] drain_flags; + + assign replay_valid = drain_valid && !drain_fault; + assign replay_payload = drain_qdata; + assign replay_ppn = drain_ppn; + assign replay_level = drain_level; + assign replay_flags = drain_flags; + assign kill_valid = drain_valid && drain_fault; + wire drain_ready = drain_fault ? kill_ready : replay_ready; + + VX_tlb_mshr #( + .NUM_REQS (NUM_REQS), + .MSHR_SIZE (MSHR_SIZE), + .QDATA_W (PAYLOAD_W), + .QDEPTH (REPLAY_DEPTH), + .DEDUP_LIVE_EXCLUDES_FAULT (1), + .ID_WIDTH (ID_WIDTH) + ) mshr ( + .clk (clk), + .reset (reset), + .probe_vpn (lookup_vpn), + .probe_match (mshr_match), + .alloc_valid (park_valid), + .alloc_vpn (park_vpn), + .alloc_access (park_access), + .alloc_amo (park_amo), + .alloc_lane (park_lane), + .alloc_qdata (park_payload), + .alloc_ready (park_ready), + .issue_valid (tlb_bus_if.req_valid), + .issue_slot (tlb_req_id), + .issue_access (tlb_req_access), + .issue_amo (tlb_req_amo), + .issue_vpn (tlb_req_vpn), + .issue_ready (tlb_bus_if.req_ready), + .fill_valid (tlb_bus_if.rsp_valid), + .fill_slot (tlb_bus_if.rsp_data.id), + .fill_fault (tlb_bus_if.rsp_data.fault), + .fill_level (tlb_bus_if.rsp_data.level), + .fill_ppn (tlb_bus_if.rsp_data.ppn), + .fill_flags (tlb_bus_if.rsp_data.flags), + .fill_ready (tlb_bus_if.rsp_ready), + .install_valid (install_valid), + .install_entry (install_entry), + .fault_valid (mshr_fault_valid), + .fault_vpn (mshr_fault_vpn), + .fault_access (mshr_fault_access), + .drain_valid (drain_valid), + .drain_qdata (drain_qdata), + .drain_fault (drain_fault), + .drain_ppn (drain_ppn), + .drain_level (drain_level), + .drain_flags (drain_flags), + .drain_ready (drain_ready), + .flush (flush), + .empty (empty) + ); - assign tlb_bus_if.req_valid = issue_any; assign tlb_bus_if.req_data = '{ - id: `UP(ID_WIDTH)'(issue_sel), - access: bk_access[issue_sel], - amo: bk_amo[issue_sel], - vpn: bk_vpn[issue_sel] + id: tlb_req_id, + access: tlb_req_access, + amo: tlb_req_amo, + vpn: tlb_req_vpn }; - wire issue_fire = tlb_bus_if.req_valid && tlb_bus_if.req_ready; - - // --------------------------------------------------------------------- - // Fill: install (unless stale/faulted) and stage the bank for drain. - // --------------------------------------------------------------------- - assign tlb_bus_if.rsp_ready = 1'b1; - wire fill_fire = tlb_bus_if.rsp_valid && tlb_bus_if.rsp_ready; - wire [BANK_W-1:0] fill_bank = BANK_W'(tlb_bus_if.rsp_data.id); - wire fill_ok = fill_fire && !tlb_bus_if.rsp_data.fault - && !bk_stale[fill_bank] && !flush; - - assign install_entry = '{ - level: tlb_bus_if.rsp_data.level, - vpn: bk_vpn[fill_bank], - ppn: tlb_bus_if.rsp_data.ppn, - flags: tlb_bus_if.rsp_data.flags - }; - for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_install - assign bank_install_valid[b] = fill_ok && (fill_bank == BANK_W'(b)); - end - - // --------------------------------------------------------------------- - // Drain: one bank per cycle, round-robin; faulted walks kill. - // --------------------------------------------------------------------- - reg [BANK_W-1:0] drain_rr; - logic [BANK_W-1:0] drain_sel; - logic drain_any; - always @(*) begin - drain_sel = '0; - drain_any = 1'b0; - for (int i = NUM_BANKS-1; i >= 0; --i) begin - automatic logic [BANK_W-1:0] b = BANK_W'(int'(drain_rr) + i + 1); - if (bk_state[b] == B_DRAIN) begin - drain_sel = b; - drain_any = 1'b1; - end - end - end - - assign replay_valid = drain_any && !bk_fault[drain_sel]; - assign replay_payload = bk_payload[drain_sel]; - assign replay_ppn = bk_ppn[drain_sel]; - assign replay_level = bk_level[drain_sel]; - assign replay_flags = bk_flags[drain_sel]; - assign kill_valid = drain_any && bk_fault[drain_sel]; - wire drain_fire = drain_any && (bk_fault[drain_sel] ? kill_ready : replay_ready); - - // Structural faults surface as the kill drains. - assign mshr_fault_valid = kill_valid && kill_ready; - assign mshr_fault_vpn = bk_vpn[drain_sel]; - assign mshr_fault_access = bk_access[drain_sel]; - - // --------------------------------------------------------------------- - // Bank state - // --------------------------------------------------------------------- - always @(posedge clk) begin - if (reset) begin - for (int b = 0; b < NUM_BANKS; ++b) begin - bk_state[b] <= B_IDLE; - bk_stale[b] <= 1'b0; - end - issue_rr <= '0; - drain_rr <= '0; - end else begin - if (park_fire) begin - bk_state [park_bank] <= B_WALK_REQ; - bk_vpn [park_bank] <= park_vpn; - bk_access [park_bank] <= park_access; - bk_amo [park_bank] <= park_amo; - bk_payload[park_bank] <= park_payload; - bk_stale [park_bank] <= 1'b0; - end - if (issue_fire) begin - bk_state[issue_sel] <= B_WALK_WAIT; - issue_rr <= issue_sel; - end - if (fill_fire) begin - if (bk_stale[fill_bank]) begin - // stale walk: discard the result and walk again - bk_state[fill_bank] <= B_WALK_REQ; - bk_stale[fill_bank] <= 1'b0; - end else begin - bk_state[fill_bank] <= B_DRAIN; - bk_fault[fill_bank] <= tlb_bus_if.rsp_data.fault; - bk_ppn [fill_bank] <= tlb_bus_if.rsp_data.ppn; - bk_level[fill_bank] <= tlb_bus_if.rsp_data.level; - bk_flags[fill_bank] <= tlb_bus_if.rsp_data.flags; - end - end - if (drain_fire) begin - bk_state[drain_sel] <= B_IDLE; - drain_rr <= drain_sel; - end - if (flush) begin - for (int b = 0; b < NUM_BANKS; ++b) begin - if (bk_state[b] == B_WALK_WAIT) begin - bk_stale[b] <= 1'b1; - end - end - end - end - end - - logic any_busy; - always @(*) begin - any_busy = 1'b0; - for (int b = 0; b < NUM_BANKS; ++b) begin - if (bk_state[b] != B_IDLE) any_busy = 1'b1; - end - end - assign empty = !any_busy; // --------------------------------------------------------------------- // Performance counters // --------------------------------------------------------------------- `ifdef PERF_ENABLE - reg [PERF_CTR_BITS-1:0] perf_reads, perf_hits, perf_misses, perf_evicts; - logic [`CLOG2(NUM_REQS+1)-1:0] reads_now, hits_now; - always @(*) begin - reads_now = '0; - hits_now = '0; - for (int l = 0; l < NUM_REQS; ++l) begin - if (lane_grant[l]) reads_now = reads_now + 1; - if (lookup_hit[l] && access_hit[l]) hits_now = hits_now + 1; - end - end + wire [`CLOG2(NUM_REQS+1)-1:0] n_hits; + `POP_COUNT(n_hits, access_hit); + wire miss_ev = park_valid && park_ready; + + reg [PERF_CTR_BITS-1:0] perf_reads, perf_hits, perf_misses, perf_evicts, perf_walks; always @(posedge clk) begin if (reset) begin perf_reads <= '0; perf_hits <= '0; perf_misses <= '0; perf_evicts <= '0; + perf_walks <= '0; end else begin - perf_reads <= perf_reads + PERF_CTR_BITS'(reads_now); - perf_hits <= perf_hits + PERF_CTR_BITS'(hits_now); - perf_misses <= perf_misses + PERF_CTR_BITS'(park_fire); - perf_evicts <= perf_evicts + PERF_CTR_BITS'((| (bank_install_valid & bank_install_evict))); + perf_reads <= perf_reads + PERF_CTR_BITS'(n_hits) + PERF_CTR_BITS'(miss_ev); + perf_hits <= perf_hits + PERF_CTR_BITS'(n_hits); + perf_misses <= perf_misses + PERF_CTR_BITS'(miss_ev); + if (| (bank_install_valid & bank_install_evict)) begin + perf_evicts <= perf_evicts + PERF_CTR_BITS'(1); + end + if (tlb_bus_if.req_valid && tlb_bus_if.req_ready) begin + perf_walks <= perf_walks + PERF_CTR_BITS'(1); + end end end + assign mmu_perf.tlb_reads = perf_reads; assign mmu_perf.tlb_hits = perf_hits; assign mmu_perf.tlb_misses = perf_misses; assign mmu_perf.tlb_evictions = perf_evicts; - // Every parked miss issues exactly one walk (re-walks after a flush are - // counted again on issue). - assign mmu_perf.ptw_walks = perf_misses; + assign mmu_perf.ptw_walks = perf_walks; assign mmu_perf.ptw_latency = '0; `endif From 3a46067aa5bf6a087bcd8e630154e3e17adb0ab9 Mon Sep 17 00:00:00 2001 From: Thomas Weatherly Date: Tue, 1 Sep 2026 20:26:22 -0400 Subject: [PATCH 5/5] simx: device-level walker and banked L1 timing twin Mirror the RTL topology in the cycle model: the Ptw moves from Cluster to ProcessorImpl behind a PtwMux that folds every cluster's L2-TLB miss link into one walker (cluster index in the high slot bits, like VX_tlb_bus_arb) and fetches PTEs on the new last LLC input (VX_CFG_L3_PTW_IDX). The cluster L2 port 0 is bound directly again; SATP fan-out, the fault latch, drain and the MPM PTW counters follow the walker to processor scope. The per-core Tlb is partitioned into VX_CFG_L1_TLB_NUM_BANKS banks by low VPN bits and the Mmu forward path grants each bank's lookup port to the lowest requesting port per tick; the MSHR path is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DJk9ipw4g1ZP3zZShiYaMq --- sim/simx/cluster.cpp | 69 ++++++++++++--------------------------- sim/simx/cluster.h | 11 +++---- sim/simx/constants.h | 7 ++++ sim/simx/csr_unit.cpp | 5 ++- sim/simx/mem/mmu.cpp | 9 +++++ sim/simx/mem/mmu_tlb.cpp | 34 +++++++++++++------ sim/simx/mem/mmu_tlb.h | 21 +++++++++--- sim/simx/mem/ptw.cpp | 49 +++++++++++++++++++++++++++ sim/simx/mem/ptw.h | 34 +++++++++++++++++++ sim/simx/processor.cpp | 60 ++++++++++++++++++++++------------ sim/simx/processor_impl.h | 9 ++++- 11 files changed, 213 insertions(+), 95 deletions(-) diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 862db3efbe..7e36e6e7b0 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -156,8 +156,6 @@ class Cluster::Impl { uint32_t num_tlb_clients = sockets_per_cluster * VX_CFG_SOCKET_SIZE * 2; snprintf(sname, 100, "%s-l2tlb", name.c_str()); l2tlb_ = L2Tlb::Create(sname, num_tlb_clients); - snprintf(sname, 100, "%s-ptw", name.c_str()); - ptw_ = Ptw::Create(sname); for (uint32_t s = 0; s < sockets_per_cluster; ++s) { for (uint32_t c = 0; c < VX_CFG_SOCKET_SIZE; ++c) { @@ -170,22 +168,11 @@ class Cluster::Impl { } } - l2tlb_->PtwReqOut.bind(&ptw_->ReqIn); - ptw_->RspOut.bind(&l2tlb_->PtwRspIn); - - // PTE fetches take the high-priority row: every translated access - // behind a walk is blocked until it completes, and the walkers issue - // far too little traffic to starve the demand stream. The reverse - // order starves the walker outright once untranslated clients (OM, - // raster) share this port, since they never stall waiting on it. - snprintf(sname, 100, "%s-ptwarb", name.c_str()); - auto ptwarb = MemArbiter::Create(sname, ArbiterType::Priority, 2, 1); - ptw_->MemReqOut.bind(&ptwarb->ReqIn.at(0)); - ptwarb->RspOut.at(0).bind(&ptw_->MemRspIn); - l2_port0_req->bind(&ptwarb->ReqIn.at(1)); - ptwarb->RspOut.at(1).bind(l2_port0_rsp); - ptwarb->ReqOut.at(0).bind(&l2cache_->core_req_in.at(0)); - l2cache_->core_rsp_out.at(0).bind(&ptwarb->RspIn.at(0)); + // The walker lives at the device (bound by the processor through the + // PtwMux); PTE fetches no longer touch this cluster's L2 cache, so + // port 0 goes back to its demand client undivided. + l2_port0_req->bind(&l2cache_->core_req_in.at(0)); + l2cache_->core_rsp_out.at(0).bind(l2_port0_rsp); #endif #ifdef VX_CFG_EXT_OM_ENABLE @@ -347,9 +334,9 @@ class Cluster::Impl { return true; } #ifdef VX_CFG_VM_ENABLE - // A pending walk or parked fill holds no channel packet while it - // waits, so completion must ask the TLB complex directly. - if (l2tlb_->busy() || ptw_->busy()) { + // A parked fill holds no channel packet while it waits, so completion + // must ask the TLB directly. (The device walker is the processor's.) + if (l2tlb_->busy()) { return true; } #endif @@ -424,17 +411,16 @@ class Cluster::Impl { #endif #ifdef VX_CFG_VM_ENABLE perf_stats.l2tlb = l2tlb_->perf_stats(); - perf_stats.ptw = ptw_->perf_stats(); #endif return perf_stats; } #ifdef VX_CFG_VM_ENABLE void set_mmu_satp(uint64_t value) { - // Single DCR source of truth: fan the device-programmed satp to the shared - // walker and to every core's L1 MMUs. Mirrors the RTL, where the L1 TLBs - // and the PTW all source satp from the DCR broadcast (not the per-core CSR). - ptw_->set_satp(value); + // Single DCR source of truth: fan the device-programmed satp to every + // core's L1 MMUs. Mirrors the RTL, where the L1 TLBs source satp from + // the DCR broadcast (not the per-core CSR); the device walker receives + // its copy from the processor directly. for (auto& socket : sockets_) { for (uint32_t c = 0; c < cores_per_socket_; ++c) { socket->core(c)->set_satp(value); @@ -442,22 +428,12 @@ class Cluster::Impl { } } - void mmu_clear_fault() { - ptw_->clear_fault(); + SimChannel& ptw_req_out() { + return l2tlb_->PtwReqOut; } - uint64_t mmu_fault_va() const { - return ptw_->fault_info().va; - } - - uint32_t mmu_fault_info() const { - auto& f = ptw_->fault_info(); - if (!f.valid) { - return 0; - } - return VX_MMU_FAULT_VALID - | (((uint32_t)f.access << VX_MMU_FAULT_ACCESS_SH) & VX_MMU_FAULT_ACCESS) - | (f.amo ? VX_MMU_FAULT_AMO : 0u); + SimChannel& ptw_rsp_in() { + return l2tlb_->PtwRspIn; } #endif @@ -609,7 +585,6 @@ class Cluster::Impl { uint32_t cores_per_socket_; #ifdef VX_CFG_VM_ENABLE L2Tlb::Ptr l2tlb_; - Ptw::Ptr ptw_; #endif #ifdef VX_CFG_EXT_OM_ENABLE OmCore::Ptr om_core_; @@ -672,16 +647,12 @@ void Cluster::set_mmu_satp(uint64_t value) { impl_->set_mmu_satp(value); } -void Cluster::mmu_clear_fault() { - impl_->mmu_clear_fault(); -} - -uint64_t Cluster::mmu_fault_va() const { - return impl_->mmu_fault_va(); +SimChannel& Cluster::ptw_req_out() { + return impl_->ptw_req_out(); } -uint32_t Cluster::mmu_fault_info() const { - return impl_->mmu_fault_info(); +SimChannel& Cluster::ptw_rsp_in() { + return impl_->ptw_rsp_in(); } #endif diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index 4f5f075390..bfc7119ee5 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -66,7 +66,6 @@ class Cluster : public SimObject { #endif #ifdef VX_CFG_VM_ENABLE L2Tlb::PerfStats l2tlb; - Ptw::PerfStats ptw; #endif }; @@ -98,12 +97,12 @@ class Cluster : public SimObject { int dcr_write(uint32_t addr, uint32_t value); #ifdef VX_CFG_VM_ENABLE - // Host-side VM control: device SATP for the walker complex, the - // device-idle TLB flush broadcast, and first-fault readback. + // Host-side VM control: device SATP fan-out to the cores' L1 MMUs. void set_mmu_satp(uint64_t value); - void mmu_clear_fault(); - uint64_t mmu_fault_va() const; - uint32_t mmu_fault_info() const; + // The cluster L2 TLB's walker link, exported for the processor to bind + // to the device-level walker (through the PtwMux). + SimChannel& ptw_req_out(); + SimChannel& ptw_rsp_in(); #endif int dcr_read(uint32_t addr, uint32_t tag, uint32_t* value); diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 66e9472307..0f3d393a9e 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -66,7 +66,14 @@ inline constexpr uint32_t VX_CFG_DCACHE_NUM_REQS = (VX_CFG_NUM_LSU_BLOCKS * DCAC inline constexpr uint32_t NUM_SOCKETS = __UP(VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE); inline constexpr uint32_t VX_CFG_L2_NUM_REQS = NUM_SOCKETS * VX_CFG_L1_MEM_PORTS; +// +1 under VM: the device-level walker's PTE fetches attach as one more LLC +// client on the last requestor slot (mirrors L3_NUM_REQS in VX_gpu_pkg.sv). +#ifdef VX_CFG_VM_ENABLE +inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_L2_MEM_PORTS + 1; +inline constexpr uint32_t VX_CFG_L3_PTW_IDX = VX_CFG_NUM_CLUSTERS * VX_CFG_L2_MEM_PORTS; +#else inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_L2_MEM_PORTS; +#endif inline constexpr uint32_t PER_ISSUE_WARPS = VX_CFG_NUM_WARPS / VX_CFG_ISSUE_WIDTH; inline constexpr uint32_t ISSUE_WIS_BITS = log2ceil(PER_ISSUE_WARPS); diff --git a/sim/simx/csr_unit.cpp b/sim/simx/csr_unit.cpp index 5cf46469ba..c3f6096b78 100644 --- a/sim/simx/csr_unit.cpp +++ b/sim/simx/csr_unit.cpp @@ -218,7 +218,6 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { } #ifdef VX_CFG_VM_ENABLE auto mmu_perf = core_->mmu_perf_stats(); - auto cluster_mmu_perf = core_->socket()->cluster()->perf_stats(); #endif switch (addr) { CSR_READ_64(VX_CSR_MPM_MEM_READS, proc_perf.mem_reads); @@ -234,8 +233,8 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { CSR_READ_64(VX_CSR_MPM_TLB_HITS, mmu_perf.tlb_hits); CSR_READ_64(VX_CSR_MPM_TLB_MISSES, mmu_perf.tlb_misses); CSR_READ_64(VX_CSR_MPM_TLB_EVICTS, mmu_perf.tlb_evictions); - CSR_READ_64(VX_CSR_MPM_PTW_WALKS, cluster_mmu_perf.ptw.walks); - CSR_READ_64(VX_CSR_MPM_PTW_LATENCY, cluster_mmu_perf.ptw.walk_latency); + CSR_READ_64(VX_CSR_MPM_PTW_WALKS, proc_perf.ptw.walks); + CSR_READ_64(VX_CSR_MPM_PTW_LATENCY, proc_perf.ptw.walk_latency); #endif } } break; diff --git a/sim/simx/mem/mmu.cpp b/sim/simx/mem/mmu.cpp index f2c440a3fe..f3481bf4c3 100644 --- a/sim/simx/mem/mmu.cpp +++ b/sim/simx/mem/mmu.cpp @@ -252,6 +252,10 @@ void Mmu::on_tick() { // while older requests are still parked, so only same-address order is // guaranteed — those share a VPN, hence one entry and its arrival-order // parked list. Replays drain ahead of new input on the same port. + // The entry array is banked with one lookup port per bank: the lowest + // port wins a bank each cycle and later ports on the same bank hold + // (the RTL's bank_conflict). + std::vector bank_taken(tlb_.num_banks(), false); for (uint32_t p = 0; p < num_ports_; ++p) { if (!replay_.at(p).empty()) { if (ReqOut.at(p).try_send(replay_.at(p).front())) { @@ -273,6 +277,11 @@ void Mmu::on_tick() { } uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; + uint32_t bank = tlb_.bank_of(vpn); + if (bank_taken.at(bank)) { + continue; + } + bank_taken.at(bank) = true; auto res = tlb_.lookup(vpn); if (res.hit) { // A cached translation still has to satisfy the access: the entry diff --git a/sim/simx/mem/mmu_tlb.cpp b/sim/simx/mem/mmu_tlb.cpp index 38ae29abcf..c3f179469e 100644 --- a/sim/simx/mem/mmu_tlb.cpp +++ b/sim/simx/mem/mmu_tlb.cpp @@ -12,13 +12,19 @@ #include "mmu_tlb.h" #include "tlb_types.h" #include +#include #include namespace vortex { -Tlb::Tlb(uint32_t size) +Tlb::Tlb(uint32_t size, uint32_t num_banks) : entries_(size) -{} + , num_banks_(num_banks) + , bank_size_(size / num_banks) +{ + assert(num_banks != 0 && (num_banks & (num_banks - 1)) == 0); + assert(size % num_banks == 0); +} static constexpr uint32_t VPN_LEVEL_BITS = TLB_VPN_LEVEL_BITS; static_assert((VX_VM_PT_SIZE / VX_VM_PTE_SIZE) == (1u << VPN_LEVEL_BITS), @@ -26,7 +32,12 @@ static_assert((VX_VM_PT_SIZE / VX_VM_PTE_SIZE) == (1u << VPN_LEVEL_BITS), Tlb::Result Tlb::lookup(uint64_t vpn) { ++reads_; - for (auto& e : entries_) { + // Only the VPN's own bank is searched: a superpage translation serves + // lookups from other banks only after each re-walks and installs its own + // copy, exactly as the banked CAMs behave. + uint32_t base = bank_of(vpn) * bank_size_; + for (uint32_t i = base; i < base + bank_size_; ++i) { + auto& e = entries_[i]; if (!e.valid) { continue; } @@ -43,21 +54,24 @@ Tlb::Result Tlb::lookup(uint64_t vpn) { } void Tlb::fill(uint64_t vpn, uint64_t ppn, uint8_t flags, uint8_t level) { - // Prefer an invalid slot; fall back to a non-MRU victim. If all slots - // are valid AND every slot has mru=true, clear all MRU bits and evict slot 0. + // Victim selection stays within the VPN's bank: prefer an invalid slot, + // fall back to a non-MRU victim, and if the whole bank is valid + MRU, + // clear the bank's MRU bits and evict its slot 0. + uint32_t base = bank_of(vpn) * bank_size_; int victim = -1; - for (size_t i = 0; i < entries_.size(); ++i) { + for (uint32_t i = base; i < base + bank_size_; ++i) { if (!entries_[i].valid) { victim = (int)i; break; } } if (victim < 0) { - for (size_t i = 0; i < entries_.size(); ++i) { + for (uint32_t i = base; i < base + bank_size_; ++i) { if (!entries_[i].mru) { victim = (int)i; break; } } } if (victim < 0) { - // All entries are valid + MRU. Clear MRU bits and pick slot 0. - for (auto& e : entries_) e.mru = false; - victim = 0; + for (uint32_t i = base; i < base + bank_size_; ++i) { + entries_[i].mru = false; + } + victim = (int)base; } if (entries_[victim].valid) { diff --git a/sim/simx/mem/mmu_tlb.h b/sim/simx/mem/mmu_tlb.h index 6e3756e106..7376efb64d 100644 --- a/sim/simx/mem/mmu_tlb.h +++ b/sim/simx/mem/mmu_tlb.h @@ -16,11 +16,19 @@ namespace vortex { -// Per-core TLB. Small fully-associative CAM of {vpn → ppn} translations -// with MRU-style eviction. Tracks MMU perf counters (VX_DCR_MPM_CLASS_MEM). +// Per-core TLB, banked: the entry array splits into num_banks single-ported +// partitions selected by the low VPN bits (mirrors hw/rtl/vm/VX_tlb_l1.sv). +// Each partition is a small fully-associative CAM with MRU-style eviction; +// the per-cycle one-lookup-per-bank discipline is enforced by the Mmu stage. +// Tracks MMU perf counters (VX_DCR_MPM_CLASS_MEM). class Tlb { public: - explicit Tlb(uint32_t size = VX_CFG_TLB_SIZE); + explicit Tlb(uint32_t size = VX_CFG_TLB_SIZE, + uint32_t num_banks = VX_CFG_L1_TLB_NUM_BANKS); + + // Which bank a VPN's lookup (and fill) must use. + uint32_t bank_of(uint64_t vpn) const { return vpn & (num_banks_ - 1); } + uint32_t num_banks() const { return num_banks_; } struct Result { bool hit = false; @@ -57,9 +65,12 @@ class Tlb { uint8_t level = 0; }; - // Linear flat array; small enough (typ. 32 entries) for a per-cycle - // linear scan to model CAM lookup behavior. + // Flat array, partitioned by bank: bank b owns the contiguous slice + // [b*bank_size_, (b+1)*bank_size_). Small enough (typ. 32 entries) for a + // per-cycle linear scan to model CAM lookup behavior. std::vector entries_; + uint32_t num_banks_; + uint32_t bank_size_; uint64_t reads_ = 0; uint64_t hits_ = 0; diff --git a/sim/simx/mem/ptw.cpp b/sim/simx/mem/ptw.cpp index fe7c79b2b0..0a320f55b7 100644 --- a/sim/simx/mem/ptw.cpp +++ b/sim/simx/mem/ptw.cpp @@ -242,4 +242,53 @@ void Ptw::on_tick() { } } +/////////////////////////////////////////////////////////////////////////////// + +PtwMux::PtwMux(const SimContext& ctx, const char* name, uint32_t num_inputs) + : SimObject(ctx, name) + , ReqIn(num_inputs, this) + , RspOut(num_inputs, this) + , ReqOut(this) + , RspIn(this) + , num_inputs_(num_inputs) +{} + +PtwMux::~PtwMux() {} + +void PtwMux::on_reset() { + grant_rr_ = 0; +} + +void PtwMux::on_tick() { + // Fills route back by the input index carried in the slot's high bits. + if (!RspIn.empty()) { + TlbRsp rsp = RspIn.peek(); + uint32_t input = rsp.slot >> SLOT_BITS; + __assert(input < num_inputs_, "PtwMux fill routes to a missing input"); + if (!RspOut.at(input).full()) { + rsp.slot &= (1u << SLOT_BITS) - 1; + RspOut.at(input).send(rsp, 1); + RspIn.pop(); + } + } + + // One request per tick, round-robin over the clusters. + if (ReqOut.full()) { + return; + } + for (uint32_t i = 0; i < num_inputs_; ++i) { + uint32_t g = (grant_rr_ + i) % num_inputs_; + if (ReqIn.at(g).empty()) { + continue; + } + TlbReq req = ReqIn.at(g).peek(); + __assert(req.slot < (1u << SLOT_BITS), "L2-TLB slot overflows the mux id space"); + req.slot |= g << SLOT_BITS; // report_only fills nothing; encoding is harmless + ReqOut.send(req, 1); + ReqIn.at(g).pop(); + grant_rr_ = g + 1; + break; + } +} + #endif // VX_CFG_VM_ENABLE diff --git a/sim/simx/mem/ptw.h b/sim/simx/mem/ptw.h index a3f62f6719..565d7aa75d 100644 --- a/sim/simx/mem/ptw.h +++ b/sim/simx/mem/ptw.h @@ -110,6 +110,40 @@ class Ptw : public SimObject { friend class SimObject; }; +// Folds the clusters' L2-TLB walker links into the one device-level Ptw, +// mirroring the RTL's VX_tlb_bus_arb: the input (cluster) index rides the +// high bits of `slot` on the way in and is stripped off the fill on the way +// out, exactly the bus-ID growth of the hardware arb. Pure routing — the +// walker itself neither knows nor cares how many clusters feed it. +class PtwMux : public SimObject { +public: + using Ptr = std::shared_ptr; + + // Cluster side. + std::vector> ReqIn; + std::vector> RspOut; + + // Walker side. + SimChannel ReqOut; + SimChannel RspIn; + + PtwMux(const SimContext& ctx, const char* name, uint32_t num_inputs); + ~PtwMux(); + +protected: + void on_reset(); + void on_tick(); + +private: + // High-bit position for the input index: above the L2 TLB's slot space. + static constexpr uint32_t SLOT_BITS = log2ceil(VX_CFG_L2_TLB_MSHR_SIZE); + + uint32_t num_inputs_; + uint32_t grant_rr_ = 0; + + friend class SimObject; +}; + } // namespace vortex #endif // VX_CFG_VM_ENABLE diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 973aa7b40d..2992844580 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -134,6 +134,21 @@ ProcessorImpl::ProcessorImpl() } } +#ifdef VX_CFG_VM_ENABLE + // Device-level walker: every cluster's L2-TLB miss link folds through the + // mux into one shared Ptw whose PTE fetches ride the LLC's last input slot. + dev_ptw_ = Ptw::Create("dev-ptw"); + dev_ptw_mux_ = PtwMux::Create("dev-ptwmux", VX_CFG_NUM_CLUSTERS); + for (uint32_t i = 0; i < VX_CFG_NUM_CLUSTERS; ++i) { + clusters_.at(i)->ptw_req_out().bind(&dev_ptw_mux_->ReqIn.at(i)); + dev_ptw_mux_->RspOut.at(i).bind(&clusters_.at(i)->ptw_rsp_in()); + } + dev_ptw_mux_->ReqOut.bind(&dev_ptw_->ReqIn); + dev_ptw_->RspOut.bind(&dev_ptw_mux_->RspIn); + dev_ptw_->MemReqOut.bind(&l3cache_->core_req_in.at(VX_CFG_L3_PTW_IDX)); + l3cache_->core_rsp_out.at(VX_CFG_L3_PTW_IDX).bind(&dev_ptw_->MemRspIn); +#endif + // connect L3 memory interfaces for (uint32_t i = 0; i < VX_CFG_L3_MEM_PORTS; ++i) { l3cache_->mem_req_out.at(i).bind(&memsim_->mem_req_in.at(i)); @@ -263,6 +278,13 @@ int ProcessorImpl::run() { exitcode |= cluster->get_exitcode(); } } +#ifdef VX_CFG_VM_ENABLE + // A walk in flight holds no channel packet while it waits on memory, + // so quiescence must ask the device walker directly. + if (dev_ptw_->busy()) { + any_running = true; + } +#endif // A page fault kills its accesses. Most warps drain on the kill // responses, but one on a path that owes no response (an instruction // fetch) would stall forever: end the launch as soon as a fault is @@ -281,13 +303,10 @@ int ProcessorImpl::run() { bool ProcessorImpl::mmu_fault_pending() const { #ifdef VX_CFG_VM_ENABLE - for (auto& cluster : clusters_) { - if (cluster->mmu_fault_info() & VX_MMU_FAULT_VALID) { - return true; - } - } -#endif + return dev_ptw_->fault_info().valid; +#else return false; +#endif } void ProcessorImpl::forward_delegated_launch() { @@ -342,13 +361,12 @@ int ProcessorImpl::dcr_write(uint32_t addr, uint32_t value) { if (addr == VX_DCR_MMU_FAULT_INFO) { // Write-to-clear: the host drops the report once it has read it, so a // fault raised by one launch stays readable across the next one's reset. - for (auto& cluster : clusters_) { - cluster->mmu_clear_fault(); - } + dev_ptw_->clear_fault(); return 0; } if (addr == VX_DCR_MMU_SATP_HI) { mmu_satp_ = (mmu_satp_ & 0xFFFFFFFF) | ((uint64_t)value << 32); + dev_ptw_->set_satp(mmu_satp_); for (auto& cluster : clusters_) { cluster->set_mmu_satp(mmu_satp_); } @@ -376,23 +394,20 @@ int ProcessorImpl::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { if (addr == VX_DCR_MMU_FAULT_VA || addr == VX_DCR_MMU_FAULT_VA_HI || addr == VX_DCR_MMU_FAULT_INFO) { - // Report the first cluster holding a latched fault; the latches clear - // on reset, so each launch starts with a clean report. + // The device walker owns the (single) fault latch; it clears on + // reset, so each launch starts with a clean report. *value = 0; - for (auto& cluster : clusters_) { - uint32_t info = cluster->mmu_fault_info(); - if (0 == (info & VX_MMU_FAULT_VALID)) { - continue; - } - uint64_t va = cluster->mmu_fault_va(); + const auto& f = dev_ptw_->fault_info(); + if (f.valid) { if (addr == VX_DCR_MMU_FAULT_INFO) { - *value = info; + *value = VX_MMU_FAULT_VALID + | (((uint32_t)f.access << VX_MMU_FAULT_ACCESS_SH) & VX_MMU_FAULT_ACCESS) + | (f.amo ? VX_MMU_FAULT_AMO : 0u); } else if (addr == VX_DCR_MMU_FAULT_VA) { - *value = (uint32_t)va; + *value = (uint32_t)f.va; } else { - *value = (uint32_t)(va >> 32); + *value = (uint32_t)(f.va >> 32); } - break; } return 0; } @@ -424,6 +439,9 @@ ProcessorImpl::PerfStats ProcessorImpl::perf_stats() const { perf.mem_latency = perf_mem_latency_; perf.l3cache = l3cache_->perf_stats(); perf.memsim = memsim_->perf_stats(); +#ifdef VX_CFG_VM_ENABLE + perf.ptw = dev_ptw_->perf_stats(); +#endif return perf; } diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index a8e24d1bcd..87bf2ecfe8 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -26,6 +26,9 @@ class ProcessorImpl { struct PerfStats { Cache::PerfStats l3cache; Memory::PerfStats memsim; +#ifdef VX_CFG_VM_ENABLE + Ptw::PerfStats ptw; +#endif uint64_t mem_reads = 0; uint64_t mem_writes = 0; uint64_t mem_latency = 0; @@ -77,7 +80,7 @@ class ProcessorImpl { // the frame kick is forwarded to every cluster's raster engine instead. void forward_delegated_launch(); - // True once any cluster's walker complex has latched a page fault. + // True once the device walker has latched a page fault. bool mmu_fault_pending() const; Kmu::Ptr kmu_; @@ -85,6 +88,10 @@ class ProcessorImpl { Memory::Ptr memsim_; #ifdef VX_CFG_VM_ENABLE uint64_t mmu_satp_ = 0; // assembled from the two DCR halves + // Device-level walker: one Ptw serves every cluster's L2 TLB, its PTE + // fetches on the LLC's last input slot (mirrors hw/rtl/Vortex.sv). + Ptw::Ptr dev_ptw_; + PtwMux::Ptr dev_ptw_mux_; #endif RAM* ram_ = nullptr; // functional backing store (set by attach_ram) Cache::Ptr l3cache_;