diff --git a/VX_config.toml b/VX_config.toml index c7206c6ed..7834c35ac 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -141,6 +141,11 @@ VX_CFG_NUM_VPU_BLOCKS = 1 # VM page-table format is a HW<->SW contract, moved to VX_types.toml [vm]; TLB depth stays here. VX_CFG_TLB_SIZE = 32 +# L1 TLB banks (power of two, dividing the TLB sizes): the entry array is +# split by low VPN bits into single-lookup-port banks; lanes contending for +# a bank serialize, misses go to the shared miss station below. +VX_CFG_L1_TLB_NUM_BANKS = 4 + # L1 TLB stage (per core): D-side / I-side entries and the per-instance # miss station depth (distinct outstanding VPN misses). VX_CFG_DTLB_SIZE = 16 diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 0f6e001ec..4a323439c 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -45,6 +45,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_VM_ENABLE // Device MMU sideband from the top DCR surface. input wire [`VX_CFG_XLEN-1:0] mmu_satp, + // Device-level walker: the L2 TLB's miss bus, exported to the device. + VX_tlb_bus_if.master dev_ptw_if, input wire mmu_flush_req, output wire mmu_flush_done, output wire mmu_fault_valid, @@ -328,7 +330,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_tlb_flush_if l2_flush_if (); VX_tlb_flush_if ptw_flush_if (); - wire l2_empty, ptw_empty; + wire l2_empty; VX_tlb_l2 #( .INSTANCE_ID (`SFORMATF(("%s-l2tlb", INSTANCE_ID))), @@ -342,44 +344,30 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .empty (l2_empty) ); - VX_mmu_fault_if ptw_fault_if (); - VX_mem_bus_if #( - .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), - .TAG_WIDTH (L2_TAG_WIDTH) - ) ptw_mem_if (); - - VX_ptw #( - .ID_WIDTH (L2_TLB_SLOT_WIDTH), - .MEM_TAG_WIDTH (L2_TAG_WIDTH) - ) ptw ( - .clk (clk), - .reset (reset), - .satp (mmu_satp), - .miss_if (l2_ptw_if), - .mem_bus_if (ptw_mem_if), - .flush_if (ptw_flush_if), - .fault_if (ptw_fault_if), - .empty (ptw_empty) - ); + // The walker lives at the device: export the L2 TLB's miss bus. The + // L2 cache carries no PTE-fetch client; its bandwidth is all sockets'. + assign dev_ptw_if.req_valid = l2_ptw_if.req_valid; + assign dev_ptw_if.req_data = l2_ptw_if.req_data; + assign l2_ptw_if.req_ready = dev_ptw_if.req_ready; + assign l2_ptw_if.rsp_valid = dev_ptw_if.rsp_valid; + assign l2_ptw_if.rsp_data = dev_ptw_if.rsp_data; + assign dev_ptw_if.rsp_ready = l2_ptw_if.rsp_ready; + + // Flush root fans to the cluster L2; each socket self-times its own L1 + // TLB flush off the SATP DCR write, and the device walker reports its + // own done leg at the top. + assign l2_flush_if.req = mmu_flush_req; + assign mmu_flush_done = l2_flush_if.done; - // PTE fetches attach as one more L2-cache client (like ocache/rcache). - `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_PTW_IDX], ptw_mem_if); + // Structural faults surface at the device walker; L1 permission faults + // are not reported here — cluster_tlb_bus_if stays a pure translation + // fabric across the socket boundary. + assign mmu_fault_valid = 1'b0; + assign mmu_fault_va = '0; + assign mmu_fault_access = 2'b0; + assign mmu_fault_amo = 1'b0; - // Flush root fans to the cluster L2 + walker; each socket self-times its own - // L1 TLB flush off the SATP DCR write, so only these two legs report done. - assign l2_flush_if.req = mmu_flush_req; - assign ptw_flush_if.req = mmu_flush_req; - assign mmu_flush_done = l2_flush_if.done && ptw_flush_if.done; - - // Only the shared walker's structural faults are surfaced. L1 permission - // faults are not reported: cluster_tlb_bus_if is the sole MMU signal - // crossing the socket boundary, and it stays a pure translation fabric. - assign mmu_fault_valid = ptw_fault_if.valid; - assign mmu_fault_va = ptw_fault_if.va; - assign mmu_fault_access = ptw_fault_if.access; - assign mmu_fault_amo = ptw_fault_if.amo; - - wire mmu_busy = ~l2_empty || ~ptw_empty; + wire mmu_busy = ~l2_empty; `else wire mmu_busy = 1'b0; `endif diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 978e90250..3a0327eed 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -1704,8 +1704,7 @@ package VX_gpu_pkg; // The shared page-table walker attaches one PTE-fetch port under VM, right // after the socket and graphics ports (like ocache/rcache). - localparam L2_PTW_REQS = `VX_CFG_VM_ENABLED; - localparam L2_PTW_IDX = L2_SOCKET_REQS + L2_GFX_REQS; + localparam L2_PTW_REQS = 0; // walker is device-level (LLC client) localparam L2_NUM_REQS = L2_SOCKET_REQS + L2_GFX_REQS + L2_PTW_REQS; @@ -1720,6 +1719,8 @@ package VX_gpu_pkg; localparam TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + `ARB_SEL_BITS(2, 1); localparam TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + `ARB_SEL_BITS(NUM_SOCKETS, 1); localparam L2_TLB_SLOT_WIDTH = `CLOG2(`VX_CFG_L2_TLB_MSHR_SIZE); + // Device-level walker: cluster L2-TLB miss buses arb into one walker. + localparam TLB_DEV_ID_WIDTH = L2_TLB_SLOT_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_CLUSTERS, 1); // Memory request data bits (mem transacts in sectors) localparam L2_MEM_DATA_WIDTH = (L2_SECTOR_SIZE * 8); @@ -1743,7 +1744,11 @@ package VX_gpu_pkg; localparam L3_SECTOR_SIZE = `VX_CFG_L3_SECTOR_SIZE; // Input request size - localparam L3_NUM_REQS = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS; + // The device-level walker attaches its PTE fetches as one more LLC + // client on the last requestor slot. + localparam L3_PTW_IDX = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS; + localparam L3_NUM_REQS = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS + + `VX_CFG_VM_ENABLED; // Core request tag bits localparam L3_TAG_WIDTH = L2_MEM_TAG_WIDTH; diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index abf7c8dad..7602d0bde 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -117,7 +117,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( VX_mem_bus_if #( .DATA_SIZE (L2_SECTOR_SIZE), .TAG_WIDTH (L3_TAG_WIDTH) - ) per_cluster_mem_bus_if[`VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS](); + ) per_cluster_mem_bus_if[L3_NUM_REQS](); VX_mem_bus_if #( .DATA_SIZE (L3_SECTOR_SIZE), @@ -219,6 +219,78 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( wire [`VX_CFG_NUM_CLUSTERS-1:0][1:0] cl_mmu_fault_access; wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_fault_amo; + // Per-cluster L2-TLB miss export buses (used by the device-level walker). + VX_tlb_bus_if #(.ID_WIDTH (L2_TLB_SLOT_WIDTH)) per_cluster_dev_ptw_if [`VX_CFG_NUM_CLUSTERS] (); + wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_flush_done_in; + + // One shared walker at the device: the clusters' L2-TLB miss buses arb + // into it, PTE fetches ride a dedicated LLC client port, and the flush + // done-tree gains the walker's leg. Structural faults surface here. + VX_tlb_bus_if #(.ID_WIDTH (TLB_DEV_ID_WIDTH)) dev_ptw_bus_if (); + + VX_tlb_bus_arb #( + .NUM_INPUTS (`VX_CFG_NUM_CLUSTERS), + .ID_WIDTH_IN (L2_TLB_SLOT_WIDTH), + .OUT_BUF (3) + ) tlb_dev_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (per_cluster_dev_ptw_if), + .bus_out_if (dev_ptw_bus_if) + ); + + VX_tlb_flush_if dev_ptw_flush_if (); + VX_mmu_fault_if dev_ptw_fault_if (); + wire dev_ptw_empty; + `UNUSED_VAR (dev_ptw_empty) + + VX_mem_bus_if #( + .DATA_SIZE (L2_SECTOR_SIZE), + .TAG_WIDTH (L3_TAG_WIDTH) + ) dev_ptw_mem_if (); + + VX_ptw #( + .ID_WIDTH (TLB_DEV_ID_WIDTH), + .DATA_SIZE (L2_SECTOR_SIZE), + .MEM_TAG_WIDTH (L3_TAG_WIDTH) + ) dev_ptw ( + .clk (clk), + .reset (reset), + .satp (mmu_satp), + .miss_if (dev_ptw_bus_if), + .mem_bus_if (dev_ptw_mem_if), + .flush_if (dev_ptw_flush_if), + .fault_if (dev_ptw_fault_if), + .empty (dev_ptw_empty) + ); + + `ASSIGN_VX_MEM_BUS_IF (per_cluster_mem_bus_if[L3_PTW_IDX], dev_ptw_mem_if); + + assign dev_ptw_flush_if.req = mmu_flush_req; + // The walker's done leg joins cluster 0's slot of the done-tree; the + // fault report likewise lands on cluster 0's lines (the DCR fault latch + // is device-global, so attribution is not lost). + for (genvar c = 0; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_dev_flush_done + assign cl_mmu_flush_done_in[c] = (c == 0) + ? (cl_mmu_flush_done[c] && dev_ptw_flush_if.done) + : cl_mmu_flush_done[c]; + end + wire mmu_fault_valid_in = dev_ptw_fault_if.valid; + wire [`VX_CFG_XLEN-1:0] mmu_fault_va_in = dev_ptw_fault_if.va; + wire [1:0] mmu_fault_access_in = dev_ptw_fault_if.access; + wire mmu_fault_amo_in = dev_ptw_fault_if.amo; + + wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_fault_valid_in; + wire [`VX_CFG_NUM_CLUSTERS-1:0][`VX_CFG_XLEN-1:0] cl_mmu_fault_va_in; + wire [`VX_CFG_NUM_CLUSTERS-1:0][1:0] cl_mmu_fault_access_in; + wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_fault_amo_in; + for (genvar c = 0; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_dev_fault_mux + assign cl_mmu_fault_valid_in[c] = cl_mmu_fault_valid[c] || ((c == 0) && mmu_fault_valid_in); + assign cl_mmu_fault_va_in[c] = ((c == 0) && mmu_fault_valid_in) ? mmu_fault_va_in : cl_mmu_fault_va[c]; + assign cl_mmu_fault_access_in[c] = ((c == 0) && mmu_fault_valid_in) ? mmu_fault_access_in : cl_mmu_fault_access[c]; + assign cl_mmu_fault_amo_in[c] = ((c == 0) && mmu_fault_valid_in) ? mmu_fault_amo_in : cl_mmu_fault_amo[c]; + end + VX_mmu_dcr mmu_dcr ( .clk (clk), .reset (reset), @@ -226,11 +298,11 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .dcr_bus_out_if (dcr_cluster_src_if), .satp (mmu_satp), .flush_req (mmu_flush_req), - .cluster_flush_done (cl_mmu_flush_done), - .cluster_fault_valid (cl_mmu_fault_valid), - .cluster_fault_va (cl_mmu_fault_va), - .cluster_fault_access (cl_mmu_fault_access), - .cluster_fault_amo (cl_mmu_fault_amo) + .cluster_flush_done (cl_mmu_flush_done_in), + .cluster_fault_valid (cl_mmu_fault_valid_in), + .cluster_fault_va (cl_mmu_fault_va_in), + .cluster_fault_access (cl_mmu_fault_access_in), + .cluster_fault_amo (cl_mmu_fault_amo_in) ); `else assign dcr_cluster_src_if.req_valid = dcr_bus_if.req_valid; @@ -278,6 +350,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `ifdef VX_CFG_VM_ENABLE .mmu_satp (mmu_satp), + .dev_ptw_if (per_cluster_dev_ptw_if[cluster_id]), .mmu_flush_req (mmu_flush_req), .mmu_flush_done (cl_mmu_flush_done[cluster_id]), .mmu_fault_valid (cl_mmu_fault_valid[cluster_id]), diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index d66338124..1967b2a12 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -145,9 +145,14 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire flush_clear; + wire [NUM_REQS-1:0] bank_conflict; + + // L1 storage: banked single-port CAMs (losing lanes hold via + // bank_conflict) in front of the shared non-blocking miss station. VX_tlb_l1 #( - .NUM_REQS (NUM_REQS), + .NUM_REQS (NUM_REQS), .TLB_SIZE (TLB_SIZE), + .NUM_BANKS (`VX_CFG_L1_TLB_NUM_BANKS), .MSHR_SIZE (MSHR_SIZE), .REPLAY_DEPTH (REPLAY_DEPTH), .PAYLOAD_W (PAYLOAD_W), @@ -159,9 +164,11 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .mmu_perf (mmu_perf), `endif .lookup_vpn (cam_vpn), + .lookup_valid (req_valid & ~req_bypass), .lookup_hit (cam_hit), .lookup_ppn (cam_ppn), .lookup_flags (cam_flags), + .bank_conflict (bank_conflict), .access_hit (cam_access_hit), .mshr_match (mshr_match), .park_valid (park_valid), @@ -199,9 +206,9 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( for (genvar l = 0; l < NUM_REQS; ++l) begin : g_cat assign perm_hit[l] = tlb_perm_ok(cam_flags[l], req_acc[l], req_amo[l]); assign cat_bypass[l] = req_valid[l] && req_bypass[l]; - assign cat_park[l] = req_valid[l] && !req_bypass[l] && (mshr_match[l] || !cam_hit[l]); - assign cat_hit[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && perm_hit[l]; - assign cat_pfault[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && !perm_hit[l]; + assign cat_park[l] = req_valid[l] && !req_bypass[l] && !bank_conflict[l] && (mshr_match[l] || !cam_hit[l]); + assign cat_hit[l] = req_valid[l] && !req_bypass[l] && !bank_conflict[l] && !mshr_match[l] && cam_hit[l] && perm_hit[l]; + assign cat_pfault[l] = req_valid[l] && !req_bypass[l] && !bank_conflict[l] && !mshr_match[l] && cam_hit[l] && !perm_hit[l]; end // Park arbitration: at most one lane parks a miss per cycle (lowest lane). diff --git a/hw/rtl/vm/VX_tlb_l1.sv b/hw/rtl/vm/VX_tlb_l1.sv index 705a8812c..a640b86e1 100644 --- a/hw/rtl/vm/VX_tlb_l1.sv +++ b/hw/rtl/vm/VX_tlb_l1.sv @@ -14,16 +14,21 @@ `include "VX_define.vh" // L1 TLB storage: the pure lookup + miss-handling core, with no address -// translation of its own. It holds the fully-associative entry array -// (parallel per-lane read), the non-blocking miss station (park / replay / -// kill), and the fill path from the shared walker over `tlb_bus`. The parent -// `VX_mmu` drives the VPN probes and consumes the raw lookup results (PPN, -// flags) plus the replay/kill streams, doing the VA→PA splice and permission -// checks itself. Splitting the storage out keeps "TLB = lookup" explicit and -// lets the banked variant live entirely here. +// translation of its own. The entry array is banked: NUM_BANKS CAMs selected +// by the low VPN bits, each with a single lookup port, so a lookup costs +// TLB_SIZE comparators in total instead of NUM_REQS x TLB_SIZE. Lanes that +// contend for one bank are serialized (bank_conflict tells the parent to +// hold the losers; lowest lane wins); lanes on different banks proceed in +// the same cycle. Misses go to the shared non-blocking miss station +// (`VX_tlb_mshr`: park / dedup / replay / kill), so a miss never blocks its +// bank and same-VPN requests join the in-flight walk. The fill installs into +// the bank of the walked VPN. The parent `VX_mmu` drives the VPN probes and +// consumes the raw lookup results (PPN, flags) plus the replay/kill streams, +// doing the VA→PA splice and permission checks itself. module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter NUM_REQS = DCACHE_NUM_REQS, + parameter NUM_REQS = DCACHE_NUM_REQS, parameter TLB_SIZE = `VX_CFG_DTLB_SIZE, + parameter NUM_BANKS = `VX_CFG_L1_TLB_NUM_BANKS, parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, parameter REPLAY_DEPTH = 2, parameter PAYLOAD_W = 1, @@ -36,11 +41,15 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( output mmu_perf_t mmu_perf, `endif - // Per-lane combinational lookup (VPN in, raw translation out). + // Per-lane combinational lookup (VPN in, raw translation out). A lane + // that loses its bank's port this cycle gets bank_conflict (parent must + // hold it); hit/miss is only meaningful for lanes with bank_conflict == 0. input wire [NUM_REQS-1:0][TLB_VPN_WIDTH-1:0] lookup_vpn, + input wire [NUM_REQS-1:0] lookup_valid, output wire [NUM_REQS-1:0] lookup_hit, output wire [NUM_REQS-1:0][TLB_PPN_WIDTH-1:0] lookup_ppn, output wire [NUM_REQS-1:0][TLB_FLAGS_WIDTH-1:0] lookup_flags, + output wire [NUM_REQS-1:0] bank_conflict, input wire [NUM_REQS-1:0] access_hit, output wire [NUM_REQS-1:0] mshr_match, @@ -76,37 +85,101 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( input wire flush, output wire empty ); + `STATIC_ASSERT(`IS_POW2(NUM_BANKS), ("NUM_BANKS must be a power of 2")) + `STATIC_ASSERT((TLB_SIZE % NUM_BANKS) == 0, ("NUM_BANKS must divide TLB_SIZE")) + + localparam ENTRIES_PER_BANK = TLB_SIZE / NUM_BANKS; + localparam BANK_W = `UP(`CLOG2(NUM_BANKS)); + localparam LANE_W = `UP(`CLOG2(NUM_REQS)); + + function automatic logic [BANK_W-1:0] bank_of(input logic [BANK_W-1:0] vpn_lo); + if (NUM_BANKS == 1) bank_of = '0; + else bank_of = vpn_lo; + endfunction + + // --------------------------------------------------------------------- + // Bank lookup port arbitration: lowest contending lane wins the bank. + // --------------------------------------------------------------------- + wire [NUM_REQS-1:0][BANK_W-1:0] lane_bank; + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lane_bank + assign lane_bank[l] = bank_of(lookup_vpn[l][BANK_W-1:0]); + end + + wire [NUM_REQS-1:0] lane_grant; + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_grant + logic older_same_bank; + always @(*) begin + older_same_bank = 1'b0; + for (int k = 0; k < l; ++k) begin + if (lookup_valid[k] && (lane_bank[k] == lane_bank[l])) begin + older_same_bank = 1'b1; + end + end + end + assign lane_grant[l] = lookup_valid[l] && !older_same_bank; + assign bank_conflict[l] = lookup_valid[l] && older_same_bank; + end + // --------------------------------------------------------------------- - // Entry array (fully-associative, parallel per-lane read) + // Entry array: one single-port CAM per bank, granted lane only. // --------------------------------------------------------------------- wire install_valid; tlb_entry_t install_entry; - wire install_evict; + wire [BANK_W-1:0] install_bank = bank_of(install_entry.vpn[BANK_W-1:0]); - VX_tlb_cam #( - .NUM_REQS (NUM_REQS), - .TLB_SIZE (TLB_SIZE) - ) cam ( - .clk (clk), - .reset (reset), - .lookup_vpn (lookup_vpn), - .lookup_hit (lookup_hit), - .lookup_ppn (lookup_ppn), - .lookup_flags (lookup_flags), - `UNUSED_PIN (lookup_ppn_raw), - `UNUSED_PIN (lookup_level), - .access_hit (access_hit), - .install_valid (install_valid), - .install_entry (install_entry), - .install_evict (install_evict), - .flush (flush) - ); + wire [NUM_BANKS-1:0] bank_install_valid; + wire [NUM_BANKS-1:0] bank_install_evict; + wire [NUM_BANKS-1:0] bank_lookup_hit; + wire [NUM_BANKS-1:0][TLB_PPN_WIDTH-1:0] bank_lookup_ppn; + wire [NUM_BANKS-1:0][TLB_FLAGS_WIDTH-1:0] bank_lookup_flags; + wire [NUM_BANKS-1:0][TLB_VPN_WIDTH-1:0] bank_lookup_vpn; + wire [NUM_BANKS-1:0] bank_access_hit; `ifndef PERF_ENABLE - `UNUSED_VAR (install_evict) + `UNUSED_VAR (bank_install_evict) `endif + for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_banks + logic [LANE_W-1:0] owner; + always @(*) begin + owner = '0; + for (int l = NUM_REQS-1; l >= 0; --l) begin + if (lane_grant[l] && (lane_bank[l] == BANK_W'(b))) begin + owner = LANE_W'(l); + end + end + end + assign bank_lookup_vpn[b] = lookup_vpn[owner]; + assign bank_access_hit[b] = access_hit[owner] && (lane_bank[owner] == BANK_W'(b)); + assign bank_install_valid[b] = install_valid && (install_bank == BANK_W'(b)); + + VX_tlb_cam #( + .NUM_REQS (1), + .TLB_SIZE (ENTRIES_PER_BANK) + ) cam ( + .clk (clk), + .reset (reset), + .lookup_vpn (bank_lookup_vpn[b]), + .lookup_hit (bank_lookup_hit[b]), + .lookup_ppn (bank_lookup_ppn[b]), + .lookup_flags (bank_lookup_flags[b]), + `UNUSED_PIN (lookup_ppn_raw), + `UNUSED_PIN (lookup_level), + .access_hit (bank_access_hit[b]), + .install_valid (bank_install_valid[b]), + .install_entry (install_entry), + .install_evict (bank_install_evict[b]), + .flush (flush) + ); + end + + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lane_out + assign lookup_hit[l] = lane_grant[l] && bank_lookup_hit[lane_bank[l]]; + assign lookup_ppn[l] = bank_lookup_ppn[lane_bank[l]]; + assign lookup_flags[l] = bank_lookup_flags[lane_bank[l]]; + end + // --------------------------------------------------------------------- - // Miss station + // Miss station (shared across banks) // --------------------------------------------------------------------- wire [`UP(ID_WIDTH)-1:0] tlb_req_id; tlb_access_e tlb_req_access; @@ -204,7 +277,7 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( perf_reads <= perf_reads + PERF_CTR_BITS'(n_hits) + PERF_CTR_BITS'(miss_ev); perf_hits <= perf_hits + PERF_CTR_BITS'(n_hits); perf_misses <= perf_misses + PERF_CTR_BITS'(miss_ev); - if (install_valid && install_evict) begin + if (| (bank_install_valid & bank_install_evict)) begin perf_evicts <= perf_evicts + PERF_CTR_BITS'(1); end if (tlb_bus_if.req_valid && tlb_bus_if.req_ready) begin diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 89991d3e8..7ac7aec7f 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -27,13 +27,16 @@ // per cluster: // VX_tlb_bus_arb : folds all sockets' miss ports -> the L2 TLB client bus // VX_tlb_l2 : shared L2 TLB -// VX_ptw : page-table walker (fed satp; PTE reads through the L2) -// VX_cache_wrap : the shared L2, fed by the sockets' L1 mem ports + the PTW +// at the device (topologically identical at one cluster): +// VX_tlb_bus_arb : folds the clusters' L2-TLB miss buses -> the walker +// VX_ptw : device-level page-table walker (fed satp; PTE reads on +// a dedicated LLC-client port, NOT through the L2) +// VX_cache_wrap : the shared L2, fed by the sockets' L1 mem ports // -// Memory hierarchy: the L1 dcaches and the PTW are clients of the shared L2 -// (the real L2_NUM_REQS client set, minus graphics), so the L2 input-arb and -// its PTE-fetch boundary carry realistic timing. The icaches' memory side and -// the L2's memory side exit to top-level ports. +// Memory hierarchy: the L1 dcaches are clients of the shared L2 (the real +// L2_NUM_REQS client set, minus graphics), so the L2 input-arb carries +// realistic timing. The icaches' memory side, the L2's memory side, and the +// walker's LLC-client port exit to top-level ports. // // Fidelity: every MMU/PTW timing boundary is present and loaded by its real // neighbour (MMU->real L1 cache, PTW->real L2). What is removed (cores/FPU/ @@ -118,7 +121,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( output wire [IC_MEM_PORTS-1:0] ic_mem_rsp_ready, // ----------------------------------------------------------------------- - // L2 memory side (master): L1 dcache + PTE traffic reach memory here + // L2 memory side (master): L1 dcache traffic reaches memory here // ----------------------------------------------------------------------- output wire [L2_MEM_PORTS-1:0] l2_mem_req_valid, output wire [L2_MEM_PORTS-1:0] l2_mem_req_rw, @@ -133,6 +136,23 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( input wire [L2_MEM_PORTS-1:0][L2_MEM_TAG_WIDTH-1:0] l2_mem_rsp_tag, output wire [L2_MEM_PORTS-1:0] l2_mem_rsp_ready, + // ----------------------------------------------------------------------- + // Device walker LLC-client port (master): PTE fetches exit here, at the + // boundary where production attaches per_cluster_mem_bus_if[L3_PTW_IDX] + // ----------------------------------------------------------------------- + output wire ptw_mem_req_valid, + output wire ptw_mem_req_rw, + output wire [L2_SECTOR_SIZE-1:0] ptw_mem_req_byteen, + output wire [L2_MEM_ADDR_W-1:0] ptw_mem_req_addr, + output wire [L3_TAG_WIDTH-1:0] ptw_mem_req_tag, + output wire [L2_SECTOR_SIZE*8-1:0] ptw_mem_req_data, + input wire ptw_mem_req_ready, + + input wire ptw_mem_rsp_valid, + input wire [L2_SECTOR_SIZE*8-1:0] ptw_mem_rsp_data, + input wire [L3_TAG_WIDTH-1:0] ptw_mem_rsp_tag, + output wire ptw_mem_rsp_ready, + // ----------------------------------------------------------------------- // Page-fault sideband (from the walker) + drain // ----------------------------------------------------------------------- @@ -448,7 +468,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // Cluster TLB + walker tail // ======================================================================== VX_tlb_bus_if #(.ID_WIDTH (TLB_CLUSTER_ID_WIDTH)) l2_client_if (); - VX_tlb_bus_if #(.ID_WIDTH (L2_TLB_SLOT_WIDTH)) l2_ptw_if (); + // one cluster's L2-TLB miss bus, as the device arb's input array + VX_tlb_bus_if #(.ID_WIDTH (L2_TLB_SLOT_WIDTH)) dev_ptw_in_if [1] (); VX_tlb_bus_arb #( .NUM_INPUTS (NS), @@ -470,39 +491,68 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .client_if (l2_client_if), - .ptw_if (l2_ptw_if), + .ptw_if (dev_ptw_in_if[0]), .flush_if (l2_flush_if), .empty (l2tlb_empty) ); + // Device-level walker tail: the arb is a passthrough at one cluster but + // keeps the production boundary (and its ID growth) in the netlist. + VX_tlb_bus_if #(.ID_WIDTH (TLB_DEV_ID_WIDTH)) dev_ptw_bus_if (); + + VX_tlb_bus_arb #( + .NUM_INPUTS (1), + .ID_WIDTH_IN (L2_TLB_SLOT_WIDTH), + .OUT_BUF (3) + ) tlb_dev_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (dev_ptw_in_if), + .bus_out_if (dev_ptw_bus_if) + ); + VX_mmu_fault_if ptw_fault_if (); VX_mem_bus_if #( - .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), - .TAG_WIDTH (L2_TAG_WIDTH) + .DATA_SIZE (L2_SECTOR_SIZE), + .TAG_WIDTH (L3_TAG_WIDTH) ) ptw_mem_if (); VX_ptw #( - .ID_WIDTH (L2_TLB_SLOT_WIDTH), - .MEM_TAG_WIDTH (L2_TAG_WIDTH) + .ID_WIDTH (TLB_DEV_ID_WIDTH), + .DATA_SIZE (L2_SECTOR_SIZE), + .MEM_TAG_WIDTH (L3_TAG_WIDTH) ) ptw ( .clk (clk), .reset (reset), .satp (satp), - .miss_if (l2_ptw_if), + .miss_if (dev_ptw_bus_if), .mem_bus_if (ptw_mem_if), .flush_if (ptw_flush_if), .fault_if (ptw_fault_if), .empty (ptw_empty) ); + assign ptw_mem_req_valid = ptw_mem_if.req_valid; + assign ptw_mem_req_rw = ptw_mem_if.req_data.rw; + assign ptw_mem_req_byteen = ptw_mem_if.req_data.byteen; + assign ptw_mem_req_addr = ptw_mem_if.req_data.addr; + assign ptw_mem_req_tag = ptw_mem_if.req_data.tag; + assign ptw_mem_req_data = ptw_mem_if.req_data.data; + assign ptw_mem_if.req_ready = ptw_mem_req_ready; + assign ptw_mem_if.rsp_valid = ptw_mem_rsp_valid; + assign ptw_mem_if.rsp_data.data = ptw_mem_rsp_data; + assign ptw_mem_if.rsp_data.tag = ptw_mem_rsp_tag; + assign ptw_mem_rsp_ready = ptw_mem_if.rsp_ready; + `UNUSED_VAR (ptw_mem_if.req_data.attr) + assign fault_valid = ptw_fault_if.valid; assign fault_va = ptw_fault_if.va; assign fault_access = ptw_fault_if.access; assign fault_amo = ptw_fault_if.amo; // ======================================================================== - // Shared L2: sockets' L1 dcache mem ports + the PTW (the real client set, - // minus graphics) -> memory ports + // Shared L2: sockets' L1 dcache mem ports (the real client set, minus + // graphics; the walker is an LLC client, not an L2 client) -> memory ports // ======================================================================== VX_mem_bus_if #( .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), @@ -513,8 +563,6 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( if (i < L2_SOCKET_REQS) begin : g_socket // socket L1 mem port (dcache), widened to the L2 client tag width `ASSIGN_VX_MEM_BUS_IF_EX (l2_core_if[i], all_dc_mem_if[i], L2_TAG_WIDTH, DCACHE_MEM_TAG_WIDTH, UUID_WIDTH); - end else if (i == L2_PTW_IDX) begin : g_ptw - `ASSIGN_VX_MEM_BUS_IF (l2_core_if[i], ptw_mem_if); end else begin : g_tie // graphics client slots (unused in this DUT) assign l2_core_if[i].req_valid = 1'b0; diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 862db3efb..7e36e6e7b 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -156,8 +156,6 @@ class Cluster::Impl { uint32_t num_tlb_clients = sockets_per_cluster * VX_CFG_SOCKET_SIZE * 2; snprintf(sname, 100, "%s-l2tlb", name.c_str()); l2tlb_ = L2Tlb::Create(sname, num_tlb_clients); - snprintf(sname, 100, "%s-ptw", name.c_str()); - ptw_ = Ptw::Create(sname); for (uint32_t s = 0; s < sockets_per_cluster; ++s) { for (uint32_t c = 0; c < VX_CFG_SOCKET_SIZE; ++c) { @@ -170,22 +168,11 @@ class Cluster::Impl { } } - l2tlb_->PtwReqOut.bind(&ptw_->ReqIn); - ptw_->RspOut.bind(&l2tlb_->PtwRspIn); - - // PTE fetches take the high-priority row: every translated access - // behind a walk is blocked until it completes, and the walkers issue - // far too little traffic to starve the demand stream. The reverse - // order starves the walker outright once untranslated clients (OM, - // raster) share this port, since they never stall waiting on it. - snprintf(sname, 100, "%s-ptwarb", name.c_str()); - auto ptwarb = MemArbiter::Create(sname, ArbiterType::Priority, 2, 1); - ptw_->MemReqOut.bind(&ptwarb->ReqIn.at(0)); - ptwarb->RspOut.at(0).bind(&ptw_->MemRspIn); - l2_port0_req->bind(&ptwarb->ReqIn.at(1)); - ptwarb->RspOut.at(1).bind(l2_port0_rsp); - ptwarb->ReqOut.at(0).bind(&l2cache_->core_req_in.at(0)); - l2cache_->core_rsp_out.at(0).bind(&ptwarb->RspIn.at(0)); + // The walker lives at the device (bound by the processor through the + // PtwMux); PTE fetches no longer touch this cluster's L2 cache, so + // port 0 goes back to its demand client undivided. + l2_port0_req->bind(&l2cache_->core_req_in.at(0)); + l2cache_->core_rsp_out.at(0).bind(l2_port0_rsp); #endif #ifdef VX_CFG_EXT_OM_ENABLE @@ -347,9 +334,9 @@ class Cluster::Impl { return true; } #ifdef VX_CFG_VM_ENABLE - // A pending walk or parked fill holds no channel packet while it - // waits, so completion must ask the TLB complex directly. - if (l2tlb_->busy() || ptw_->busy()) { + // A parked fill holds no channel packet while it waits, so completion + // must ask the TLB directly. (The device walker is the processor's.) + if (l2tlb_->busy()) { return true; } #endif @@ -424,17 +411,16 @@ class Cluster::Impl { #endif #ifdef VX_CFG_VM_ENABLE perf_stats.l2tlb = l2tlb_->perf_stats(); - perf_stats.ptw = ptw_->perf_stats(); #endif return perf_stats; } #ifdef VX_CFG_VM_ENABLE void set_mmu_satp(uint64_t value) { - // Single DCR source of truth: fan the device-programmed satp to the shared - // walker and to every core's L1 MMUs. Mirrors the RTL, where the L1 TLBs - // and the PTW all source satp from the DCR broadcast (not the per-core CSR). - ptw_->set_satp(value); + // Single DCR source of truth: fan the device-programmed satp to every + // core's L1 MMUs. Mirrors the RTL, where the L1 TLBs source satp from + // the DCR broadcast (not the per-core CSR); the device walker receives + // its copy from the processor directly. for (auto& socket : sockets_) { for (uint32_t c = 0; c < cores_per_socket_; ++c) { socket->core(c)->set_satp(value); @@ -442,22 +428,12 @@ class Cluster::Impl { } } - void mmu_clear_fault() { - ptw_->clear_fault(); + SimChannel& ptw_req_out() { + return l2tlb_->PtwReqOut; } - uint64_t mmu_fault_va() const { - return ptw_->fault_info().va; - } - - uint32_t mmu_fault_info() const { - auto& f = ptw_->fault_info(); - if (!f.valid) { - return 0; - } - return VX_MMU_FAULT_VALID - | (((uint32_t)f.access << VX_MMU_FAULT_ACCESS_SH) & VX_MMU_FAULT_ACCESS) - | (f.amo ? VX_MMU_FAULT_AMO : 0u); + SimChannel& ptw_rsp_in() { + return l2tlb_->PtwRspIn; } #endif @@ -609,7 +585,6 @@ class Cluster::Impl { uint32_t cores_per_socket_; #ifdef VX_CFG_VM_ENABLE L2Tlb::Ptr l2tlb_; - Ptw::Ptr ptw_; #endif #ifdef VX_CFG_EXT_OM_ENABLE OmCore::Ptr om_core_; @@ -672,16 +647,12 @@ void Cluster::set_mmu_satp(uint64_t value) { impl_->set_mmu_satp(value); } -void Cluster::mmu_clear_fault() { - impl_->mmu_clear_fault(); -} - -uint64_t Cluster::mmu_fault_va() const { - return impl_->mmu_fault_va(); +SimChannel& Cluster::ptw_req_out() { + return impl_->ptw_req_out(); } -uint32_t Cluster::mmu_fault_info() const { - return impl_->mmu_fault_info(); +SimChannel& Cluster::ptw_rsp_in() { + return impl_->ptw_rsp_in(); } #endif diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index 4f5f07539..bfc7119ee 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -66,7 +66,6 @@ class Cluster : public SimObject { #endif #ifdef VX_CFG_VM_ENABLE L2Tlb::PerfStats l2tlb; - Ptw::PerfStats ptw; #endif }; @@ -98,12 +97,12 @@ class Cluster : public SimObject { int dcr_write(uint32_t addr, uint32_t value); #ifdef VX_CFG_VM_ENABLE - // Host-side VM control: device SATP for the walker complex, the - // device-idle TLB flush broadcast, and first-fault readback. + // Host-side VM control: device SATP fan-out to the cores' L1 MMUs. void set_mmu_satp(uint64_t value); - void mmu_clear_fault(); - uint64_t mmu_fault_va() const; - uint32_t mmu_fault_info() const; + // The cluster L2 TLB's walker link, exported for the processor to bind + // to the device-level walker (through the PtwMux). + SimChannel& ptw_req_out(); + SimChannel& ptw_rsp_in(); #endif int dcr_read(uint32_t addr, uint32_t tag, uint32_t* value); diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 66e947230..0f3d393a9 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -66,7 +66,14 @@ inline constexpr uint32_t VX_CFG_DCACHE_NUM_REQS = (VX_CFG_NUM_LSU_BLOCKS * DCAC inline constexpr uint32_t NUM_SOCKETS = __UP(VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE); inline constexpr uint32_t VX_CFG_L2_NUM_REQS = NUM_SOCKETS * VX_CFG_L1_MEM_PORTS; +// +1 under VM: the device-level walker's PTE fetches attach as one more LLC +// client on the last requestor slot (mirrors L3_NUM_REQS in VX_gpu_pkg.sv). +#ifdef VX_CFG_VM_ENABLE +inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_L2_MEM_PORTS + 1; +inline constexpr uint32_t VX_CFG_L3_PTW_IDX = VX_CFG_NUM_CLUSTERS * VX_CFG_L2_MEM_PORTS; +#else inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_L2_MEM_PORTS; +#endif inline constexpr uint32_t PER_ISSUE_WARPS = VX_CFG_NUM_WARPS / VX_CFG_ISSUE_WIDTH; inline constexpr uint32_t ISSUE_WIS_BITS = log2ceil(PER_ISSUE_WARPS); diff --git a/sim/simx/csr_unit.cpp b/sim/simx/csr_unit.cpp index 5cf46469b..c3f6096b7 100644 --- a/sim/simx/csr_unit.cpp +++ b/sim/simx/csr_unit.cpp @@ -218,7 +218,6 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { } #ifdef VX_CFG_VM_ENABLE auto mmu_perf = core_->mmu_perf_stats(); - auto cluster_mmu_perf = core_->socket()->cluster()->perf_stats(); #endif switch (addr) { CSR_READ_64(VX_CSR_MPM_MEM_READS, proc_perf.mem_reads); @@ -234,8 +233,8 @@ Word CsrUnit::get_csr(uint32_t addr, uint32_t wid, uint32_t tid) { CSR_READ_64(VX_CSR_MPM_TLB_HITS, mmu_perf.tlb_hits); CSR_READ_64(VX_CSR_MPM_TLB_MISSES, mmu_perf.tlb_misses); CSR_READ_64(VX_CSR_MPM_TLB_EVICTS, mmu_perf.tlb_evictions); - CSR_READ_64(VX_CSR_MPM_PTW_WALKS, cluster_mmu_perf.ptw.walks); - CSR_READ_64(VX_CSR_MPM_PTW_LATENCY, cluster_mmu_perf.ptw.walk_latency); + CSR_READ_64(VX_CSR_MPM_PTW_WALKS, proc_perf.ptw.walks); + CSR_READ_64(VX_CSR_MPM_PTW_LATENCY, proc_perf.ptw.walk_latency); #endif } } break; diff --git a/sim/simx/mem/mmu.cpp b/sim/simx/mem/mmu.cpp index f2c440a3f..f3481bf4c 100644 --- a/sim/simx/mem/mmu.cpp +++ b/sim/simx/mem/mmu.cpp @@ -252,6 +252,10 @@ void Mmu::on_tick() { // while older requests are still parked, so only same-address order is // guaranteed — those share a VPN, hence one entry and its arrival-order // parked list. Replays drain ahead of new input on the same port. + // The entry array is banked with one lookup port per bank: the lowest + // port wins a bank each cycle and later ports on the same bank hold + // (the RTL's bank_conflict). + std::vector bank_taken(tlb_.num_banks(), false); for (uint32_t p = 0; p < num_ports_; ++p) { if (!replay_.at(p).empty()) { if (ReqOut.at(p).try_send(replay_.at(p).front())) { @@ -273,6 +277,11 @@ void Mmu::on_tick() { } uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; + uint32_t bank = tlb_.bank_of(vpn); + if (bank_taken.at(bank)) { + continue; + } + bank_taken.at(bank) = true; auto res = tlb_.lookup(vpn); if (res.hit) { // A cached translation still has to satisfy the access: the entry diff --git a/sim/simx/mem/mmu_tlb.cpp b/sim/simx/mem/mmu_tlb.cpp index 38ae29abc..c3f179469 100644 --- a/sim/simx/mem/mmu_tlb.cpp +++ b/sim/simx/mem/mmu_tlb.cpp @@ -12,13 +12,19 @@ #include "mmu_tlb.h" #include "tlb_types.h" #include +#include #include namespace vortex { -Tlb::Tlb(uint32_t size) +Tlb::Tlb(uint32_t size, uint32_t num_banks) : entries_(size) -{} + , num_banks_(num_banks) + , bank_size_(size / num_banks) +{ + assert(num_banks != 0 && (num_banks & (num_banks - 1)) == 0); + assert(size % num_banks == 0); +} static constexpr uint32_t VPN_LEVEL_BITS = TLB_VPN_LEVEL_BITS; static_assert((VX_VM_PT_SIZE / VX_VM_PTE_SIZE) == (1u << VPN_LEVEL_BITS), @@ -26,7 +32,12 @@ static_assert((VX_VM_PT_SIZE / VX_VM_PTE_SIZE) == (1u << VPN_LEVEL_BITS), Tlb::Result Tlb::lookup(uint64_t vpn) { ++reads_; - for (auto& e : entries_) { + // Only the VPN's own bank is searched: a superpage translation serves + // lookups from other banks only after each re-walks and installs its own + // copy, exactly as the banked CAMs behave. + uint32_t base = bank_of(vpn) * bank_size_; + for (uint32_t i = base; i < base + bank_size_; ++i) { + auto& e = entries_[i]; if (!e.valid) { continue; } @@ -43,21 +54,24 @@ Tlb::Result Tlb::lookup(uint64_t vpn) { } void Tlb::fill(uint64_t vpn, uint64_t ppn, uint8_t flags, uint8_t level) { - // Prefer an invalid slot; fall back to a non-MRU victim. If all slots - // are valid AND every slot has mru=true, clear all MRU bits and evict slot 0. + // Victim selection stays within the VPN's bank: prefer an invalid slot, + // fall back to a non-MRU victim, and if the whole bank is valid + MRU, + // clear the bank's MRU bits and evict its slot 0. + uint32_t base = bank_of(vpn) * bank_size_; int victim = -1; - for (size_t i = 0; i < entries_.size(); ++i) { + for (uint32_t i = base; i < base + bank_size_; ++i) { if (!entries_[i].valid) { victim = (int)i; break; } } if (victim < 0) { - for (size_t i = 0; i < entries_.size(); ++i) { + for (uint32_t i = base; i < base + bank_size_; ++i) { if (!entries_[i].mru) { victim = (int)i; break; } } } if (victim < 0) { - // All entries are valid + MRU. Clear MRU bits and pick slot 0. - for (auto& e : entries_) e.mru = false; - victim = 0; + for (uint32_t i = base; i < base + bank_size_; ++i) { + entries_[i].mru = false; + } + victim = (int)base; } if (entries_[victim].valid) { diff --git a/sim/simx/mem/mmu_tlb.h b/sim/simx/mem/mmu_tlb.h index 6e3756e10..7376efb64 100644 --- a/sim/simx/mem/mmu_tlb.h +++ b/sim/simx/mem/mmu_tlb.h @@ -16,11 +16,19 @@ namespace vortex { -// Per-core TLB. Small fully-associative CAM of {vpn → ppn} translations -// with MRU-style eviction. Tracks MMU perf counters (VX_DCR_MPM_CLASS_MEM). +// Per-core TLB, banked: the entry array splits into num_banks single-ported +// partitions selected by the low VPN bits (mirrors hw/rtl/vm/VX_tlb_l1.sv). +// Each partition is a small fully-associative CAM with MRU-style eviction; +// the per-cycle one-lookup-per-bank discipline is enforced by the Mmu stage. +// Tracks MMU perf counters (VX_DCR_MPM_CLASS_MEM). class Tlb { public: - explicit Tlb(uint32_t size = VX_CFG_TLB_SIZE); + explicit Tlb(uint32_t size = VX_CFG_TLB_SIZE, + uint32_t num_banks = VX_CFG_L1_TLB_NUM_BANKS); + + // Which bank a VPN's lookup (and fill) must use. + uint32_t bank_of(uint64_t vpn) const { return vpn & (num_banks_ - 1); } + uint32_t num_banks() const { return num_banks_; } struct Result { bool hit = false; @@ -57,9 +65,12 @@ class Tlb { uint8_t level = 0; }; - // Linear flat array; small enough (typ. 32 entries) for a per-cycle - // linear scan to model CAM lookup behavior. + // Flat array, partitioned by bank: bank b owns the contiguous slice + // [b*bank_size_, (b+1)*bank_size_). Small enough (typ. 32 entries) for a + // per-cycle linear scan to model CAM lookup behavior. std::vector entries_; + uint32_t num_banks_; + uint32_t bank_size_; uint64_t reads_ = 0; uint64_t hits_ = 0; diff --git a/sim/simx/mem/ptw.cpp b/sim/simx/mem/ptw.cpp index fe7c79b2b..0a320f55b 100644 --- a/sim/simx/mem/ptw.cpp +++ b/sim/simx/mem/ptw.cpp @@ -242,4 +242,53 @@ void Ptw::on_tick() { } } +/////////////////////////////////////////////////////////////////////////////// + +PtwMux::PtwMux(const SimContext& ctx, const char* name, uint32_t num_inputs) + : SimObject(ctx, name) + , ReqIn(num_inputs, this) + , RspOut(num_inputs, this) + , ReqOut(this) + , RspIn(this) + , num_inputs_(num_inputs) +{} + +PtwMux::~PtwMux() {} + +void PtwMux::on_reset() { + grant_rr_ = 0; +} + +void PtwMux::on_tick() { + // Fills route back by the input index carried in the slot's high bits. + if (!RspIn.empty()) { + TlbRsp rsp = RspIn.peek(); + uint32_t input = rsp.slot >> SLOT_BITS; + __assert(input < num_inputs_, "PtwMux fill routes to a missing input"); + if (!RspOut.at(input).full()) { + rsp.slot &= (1u << SLOT_BITS) - 1; + RspOut.at(input).send(rsp, 1); + RspIn.pop(); + } + } + + // One request per tick, round-robin over the clusters. + if (ReqOut.full()) { + return; + } + for (uint32_t i = 0; i < num_inputs_; ++i) { + uint32_t g = (grant_rr_ + i) % num_inputs_; + if (ReqIn.at(g).empty()) { + continue; + } + TlbReq req = ReqIn.at(g).peek(); + __assert(req.slot < (1u << SLOT_BITS), "L2-TLB slot overflows the mux id space"); + req.slot |= g << SLOT_BITS; // report_only fills nothing; encoding is harmless + ReqOut.send(req, 1); + ReqIn.at(g).pop(); + grant_rr_ = g + 1; + break; + } +} + #endif // VX_CFG_VM_ENABLE diff --git a/sim/simx/mem/ptw.h b/sim/simx/mem/ptw.h index a3f62f671..565d7aa75 100644 --- a/sim/simx/mem/ptw.h +++ b/sim/simx/mem/ptw.h @@ -110,6 +110,40 @@ class Ptw : public SimObject { friend class SimObject; }; +// Folds the clusters' L2-TLB walker links into the one device-level Ptw, +// mirroring the RTL's VX_tlb_bus_arb: the input (cluster) index rides the +// high bits of `slot` on the way in and is stripped off the fill on the way +// out, exactly the bus-ID growth of the hardware arb. Pure routing — the +// walker itself neither knows nor cares how many clusters feed it. +class PtwMux : public SimObject { +public: + using Ptr = std::shared_ptr; + + // Cluster side. + std::vector> ReqIn; + std::vector> RspOut; + + // Walker side. + SimChannel ReqOut; + SimChannel RspIn; + + PtwMux(const SimContext& ctx, const char* name, uint32_t num_inputs); + ~PtwMux(); + +protected: + void on_reset(); + void on_tick(); + +private: + // High-bit position for the input index: above the L2 TLB's slot space. + static constexpr uint32_t SLOT_BITS = log2ceil(VX_CFG_L2_TLB_MSHR_SIZE); + + uint32_t num_inputs_; + uint32_t grant_rr_ = 0; + + friend class SimObject; +}; + } // namespace vortex #endif // VX_CFG_VM_ENABLE diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index 973aa7b40..299284458 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -134,6 +134,21 @@ ProcessorImpl::ProcessorImpl() } } +#ifdef VX_CFG_VM_ENABLE + // Device-level walker: every cluster's L2-TLB miss link folds through the + // mux into one shared Ptw whose PTE fetches ride the LLC's last input slot. + dev_ptw_ = Ptw::Create("dev-ptw"); + dev_ptw_mux_ = PtwMux::Create("dev-ptwmux", VX_CFG_NUM_CLUSTERS); + for (uint32_t i = 0; i < VX_CFG_NUM_CLUSTERS; ++i) { + clusters_.at(i)->ptw_req_out().bind(&dev_ptw_mux_->ReqIn.at(i)); + dev_ptw_mux_->RspOut.at(i).bind(&clusters_.at(i)->ptw_rsp_in()); + } + dev_ptw_mux_->ReqOut.bind(&dev_ptw_->ReqIn); + dev_ptw_->RspOut.bind(&dev_ptw_mux_->RspIn); + dev_ptw_->MemReqOut.bind(&l3cache_->core_req_in.at(VX_CFG_L3_PTW_IDX)); + l3cache_->core_rsp_out.at(VX_CFG_L3_PTW_IDX).bind(&dev_ptw_->MemRspIn); +#endif + // connect L3 memory interfaces for (uint32_t i = 0; i < VX_CFG_L3_MEM_PORTS; ++i) { l3cache_->mem_req_out.at(i).bind(&memsim_->mem_req_in.at(i)); @@ -263,6 +278,13 @@ int ProcessorImpl::run() { exitcode |= cluster->get_exitcode(); } } +#ifdef VX_CFG_VM_ENABLE + // A walk in flight holds no channel packet while it waits on memory, + // so quiescence must ask the device walker directly. + if (dev_ptw_->busy()) { + any_running = true; + } +#endif // A page fault kills its accesses. Most warps drain on the kill // responses, but one on a path that owes no response (an instruction // fetch) would stall forever: end the launch as soon as a fault is @@ -281,13 +303,10 @@ int ProcessorImpl::run() { bool ProcessorImpl::mmu_fault_pending() const { #ifdef VX_CFG_VM_ENABLE - for (auto& cluster : clusters_) { - if (cluster->mmu_fault_info() & VX_MMU_FAULT_VALID) { - return true; - } - } -#endif + return dev_ptw_->fault_info().valid; +#else return false; +#endif } void ProcessorImpl::forward_delegated_launch() { @@ -342,13 +361,12 @@ int ProcessorImpl::dcr_write(uint32_t addr, uint32_t value) { if (addr == VX_DCR_MMU_FAULT_INFO) { // Write-to-clear: the host drops the report once it has read it, so a // fault raised by one launch stays readable across the next one's reset. - for (auto& cluster : clusters_) { - cluster->mmu_clear_fault(); - } + dev_ptw_->clear_fault(); return 0; } if (addr == VX_DCR_MMU_SATP_HI) { mmu_satp_ = (mmu_satp_ & 0xFFFFFFFF) | ((uint64_t)value << 32); + dev_ptw_->set_satp(mmu_satp_); for (auto& cluster : clusters_) { cluster->set_mmu_satp(mmu_satp_); } @@ -376,23 +394,20 @@ int ProcessorImpl::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { if (addr == VX_DCR_MMU_FAULT_VA || addr == VX_DCR_MMU_FAULT_VA_HI || addr == VX_DCR_MMU_FAULT_INFO) { - // Report the first cluster holding a latched fault; the latches clear - // on reset, so each launch starts with a clean report. + // The device walker owns the (single) fault latch; it clears on + // reset, so each launch starts with a clean report. *value = 0; - for (auto& cluster : clusters_) { - uint32_t info = cluster->mmu_fault_info(); - if (0 == (info & VX_MMU_FAULT_VALID)) { - continue; - } - uint64_t va = cluster->mmu_fault_va(); + const auto& f = dev_ptw_->fault_info(); + if (f.valid) { if (addr == VX_DCR_MMU_FAULT_INFO) { - *value = info; + *value = VX_MMU_FAULT_VALID + | (((uint32_t)f.access << VX_MMU_FAULT_ACCESS_SH) & VX_MMU_FAULT_ACCESS) + | (f.amo ? VX_MMU_FAULT_AMO : 0u); } else if (addr == VX_DCR_MMU_FAULT_VA) { - *value = (uint32_t)va; + *value = (uint32_t)f.va; } else { - *value = (uint32_t)(va >> 32); + *value = (uint32_t)(f.va >> 32); } - break; } return 0; } @@ -424,6 +439,9 @@ ProcessorImpl::PerfStats ProcessorImpl::perf_stats() const { perf.mem_latency = perf_mem_latency_; perf.l3cache = l3cache_->perf_stats(); perf.memsim = memsim_->perf_stats(); +#ifdef VX_CFG_VM_ENABLE + perf.ptw = dev_ptw_->perf_stats(); +#endif return perf; } diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index a8e24d1bc..87bf2ecfe 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -26,6 +26,9 @@ class ProcessorImpl { struct PerfStats { Cache::PerfStats l3cache; Memory::PerfStats memsim; +#ifdef VX_CFG_VM_ENABLE + Ptw::PerfStats ptw; +#endif uint64_t mem_reads = 0; uint64_t mem_writes = 0; uint64_t mem_latency = 0; @@ -77,7 +80,7 @@ class ProcessorImpl { // the frame kick is forwarded to every cluster's raster engine instead. void forward_delegated_launch(); - // True once any cluster's walker complex has latched a page fault. + // True once the device walker has latched a page fault. bool mmu_fault_pending() const; Kmu::Ptr kmu_; @@ -85,6 +88,10 @@ class ProcessorImpl { Memory::Ptr memsim_; #ifdef VX_CFG_VM_ENABLE uint64_t mmu_satp_ = 0; // assembled from the two DCR halves + // Device-level walker: one Ptw serves every cluster's L2 TLB, its PTE + // fetches on the LLC's last input slot (mirrors hw/rtl/Vortex.sv). + Ptw::Ptr dev_ptw_; + PtwMux::Ptr dev_ptw_mux_; #endif RAM* ram_ = nullptr; // functional backing store (set by attach_ram) Cache::Ptr l3cache_;