diff --git a/src/arch/armv8/aarch64/inc/arch/subarch/sysregs.h b/src/arch/armv8/aarch64/inc/arch/subarch/sysregs.h index 91a9e30ec..fb626001c 100644 --- a/src/arch/armv8/aarch64/inc/arch/subarch/sysregs.h +++ b/src/arch/armv8/aarch64/inc/arch/subarch/sysregs.h @@ -76,7 +76,18 @@ SYSREG_GEN_ACCESSORS(cntvoff_el2) SYSREG_GEN_ACCESSORS(sctlr_el1) SYSREG_GEN_ACCESSORS(cntkctl_el1) SYSREG_GEN_ACCESSORS(cntfrq_el0) +SYSREG_GEN_ACCESSORS(cnthp_ctl_el2) +SYSREG_GEN_ACCESSORS(cnthp_tval_el2) SYSREG_GEN_ACCESSORS(pmcr_el0) +SYSREG_GEN_ACCESSORS(mdcr_el2) +SYSREG_GEN_ACCESSORS(pmcntenset_el0) +SYSREG_GEN_ACCESSORS(pmcntenclr_el0) +SYSREG_GEN_ACCESSORS(pmselr_el0) +SYSREG_GEN_ACCESSORS(pmxevcntr_el0) +SYSREG_GEN_ACCESSORS(pmxevtyper_el0) +SYSREG_GEN_ACCESSORS(pmintenset_el1) +SYSREG_GEN_ACCESSORS(pmintenclr_el1) +SYSREG_GEN_ACCESSORS(pmovsclr_el0) SYSREG_GEN_ACCESSORS(par_el1) SYSREG_GEN_ACCESSORS(tcr_el2) SYSREG_GEN_ACCESSORS(ttbr0_el2) diff --git a/src/arch/armv8/generic_timer.c b/src/arch/armv8/generic_timer.c new file mode 100644 index 000000000..fd6d086af --- /dev/null +++ b/src/arch/armv8/generic_timer.c @@ -0,0 +1,198 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Bao Project and Contributors. All rights reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CNTHP_CTL_EL2_ENABLE 0 +#define CNTHP_CTL_EL2_IMASK 1 + +static irq_handler_t timer_handlers[CPU_MAX]; +static irqid_t timer_irq_id = INVALID_IRQID; +static spinlock_t timer_irq_lock = SPINLOCK_INITVAL; + +static void timer_arch_interrupt_handler(irqid_t int_id) +{ + irq_handler_t handler = timer_handlers[cpu()->id]; + + if (handler != NULL) { + handler(int_id); + } +} + +void timer_arch_define_irq_callback(irq_handler_t handler) +{ + irqid_t interrupt_id = platform.arch.generic_timer.interrupt_id; + + if ((interrupt_id == INVALID_IRQID) || (interrupt_id == 0U)) { + ERROR("Invalid hypervisor timer interrupt id\n"); + } + + timer_handlers[cpu()->id] = handler; + + spin_lock(&timer_irq_lock); + if (timer_irq_id == INVALID_IRQID) { + timer_irq_id = interrupts_reserve(interrupt_id, timer_arch_interrupt_handler); + if (timer_irq_id == INVALID_IRQID) { + ERROR("Failed to reserve hypervisor timer interrupt id = %u\n", interrupt_id); + } + } else if (timer_irq_id != interrupt_id) { + ERROR("Inconsistent hypervisor timer interrupt id = %u\n", interrupt_id); + } + spin_unlock(&timer_irq_lock); + + interrupts_arch_enable(timer_irq_id, true); +} + +/** + * @brief Enables the architecture-specific timer. + */ +void timer_arch_enable(void) +{ + uint64_t ctl_value; + + // Read the current value of the control register CNTHP_CTL_EL2. + ctl_value = sysreg_cnthp_ctl_el2_read(); + + // CNTHP_CTL_EL2 --> ENABLE, bit [0] + // --> 0b0 - Timer disabled + // --> 0b1 - Timer enabled + ctl_value = bit_set(ctl_value, CNTHP_CTL_EL2_ENABLE); + + // CNTHP_CTL_EL2 --> IMASK, bit [1] + // --> 0b0 - Timer interrupt is not masked by the IMASK bit + // --> 0b1 - Timer interrupt is masked by the IMASK bit + ctl_value = bit_clear(ctl_value, CNTHP_CTL_EL2_IMASK); + + // Write the updated control value back to the CNTHP_CTL_EL2 register. + sysreg_cnthp_ctl_el2_write(ctl_value); +} + +/** + * @brief Disables the architecture-specific timer. + */ +void timer_arch_disable(void) +{ + uint64_t ctl_value; + + // Read the current value of the control register CNTHP_CTL_EL2. + ctl_value = sysreg_cnthp_ctl_el2_read(); + + // CNTHP_CTL_EL2 --> ENABLE, bit [0] + // --> 0b0 - Timer disabled + // --> 0b1 - Timer enabled + ctl_value = bit_clear(ctl_value, CNTHP_CTL_EL2_ENABLE); + + // Write the updated control value back to the CNTHP_CTL_EL2 register. + sysreg_cnthp_ctl_el2_write(ctl_value); +} + +/** + * @brief Gets the system frequency from the architecture-specific timer. + * + * @return The system frequency in Hz. + */ +static inline uint64_t timer_arch_get_system_frequency(void) +{ + uint64_t frequency; + + // Read the system frequency from the architecture-specific register + // CNTFRQ_EL0. + frequency = sysreg_cntfrq_el0_read(); + + // Mask the lower 32 bits of the frequency value to get the system frequency + // in Hz. + return frequency & UINT32_MAX; +} + +/** + * @brief Sets the architecture-specific timer counter value. + * + * @param count The value to set for the timer counter (lower 32 bits). + */ +static inline void timer_arch_set_counter(uint64_t count) +{ + if (count > INT32_MAX) { + ERROR("Hypervisor timer count exceeds TVAL range\n"); + } + + // Write the new counter value to the architecture-specific register + // CNTHP_TVAL_EL2. + sysreg_cnthp_tval_el2_write(count); +} + +/** + * @brief Initializes the architecture-specific timer with the specified period. + * + * @param period The timer period in microseconds. + * @return The calculated counter value to set for the timer. + */ +uint64_t timer_arch_init(uint64_t period) +{ + uint64_t frequency; + uint64_t count_value; + + // Get the system frequency from the architecture-specific timer. + frequency = timer_arch_get_system_frequency(); + + // Calculate the count value to set for the timer based on the period and + // system frequency. + count_value = (period * frequency) / 1000000; + + // Set the timer counter to the calculated count value. + timer_arch_set_counter(count_value); + + // Enable the architecture-specific timer. + timer_arch_enable(); + + // Return the calculated counter value for the timer. + return count_value; +} + +/** + * @brief Reschedules the architecture-specific timer interrupt with the + * specified counter value. + * + * @param count The counter value to set for rescheduling the timer interrupt. + */ +void timer_arch_reschedule_interrupt(uint64_t count) +{ + // Set the timer counter to the specified count value for rescheduling the + // interrupt. + timer_arch_set_counter(count); +} + +/** + * @brief Reschedules the architecture-specific timer interrupt with the + * specified period in microseconds. + * + * @param period The timer period in microseconds to reschedule the interrupt. + * @return The calculated counter value to set for the timer. + */ +uint64_t timer_arch_reschedule_interrupt_us(uint64_t period) +{ + uint64_t frequency; + uint64_t count_value; + + // Get the system frequency from the architecture-specific timer. + frequency = timer_arch_get_system_frequency(); + + // Calculate the count value to set for the timer based on the period and + // system frequency. + count_value = (period * frequency) / 1000000; + + // Set the timer counter to the calculated count value for rescheduling + // the interrupt. + timer_arch_set_counter(count_value); + + // Return the calculated counter value for the timer. + return count_value; +} diff --git a/src/arch/armv8/gic.c b/src/arch/armv8/gic.c index ed2856eb6..2cefa94f8 100644 --- a/src/arch/armv8/gic.c +++ b/src/arch/armv8/gic.c @@ -106,6 +106,7 @@ void gic_handle() if (res == HANDLED_BY_HYP) { gicc_dir(ack); } + interrupts_post_handler(); } } diff --git a/src/arch/armv8/inc/arch/events.h b/src/arch/armv8/inc/arch/events.h new file mode 100644 index 000000000..2944e2b97 --- /dev/null +++ b/src/arch/armv8/inc/arch/events.h @@ -0,0 +1,105 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Bao Project and Contributors. All rights reserved. + */ + +#ifndef __ARCH_EVENTS_H__ +#define __ARCH_EVENTS_H__ + +#include + +#define EVENTS_ARCH_CNTR_MAX_NUM PMU_CNTR_MAX_NUM + +static inline ssize_t events_arch_cntr_alloc(void) +{ + return pmu_cntr_alloc(); +} + +static inline void events_arch_cntr_free(size_t counter) +{ + pmu_cntr_free(counter); +} + +static inline void events_arch_configure_counter_partition(size_t mem_throt_reserved) +{ + pmu_configure_counter_partition(mem_throt_reserved); +} + +/* enable events */ +static inline void events_arch_enable(void) +{ + pmu_enable(); +} + +/* disable events */ +static inline void events_arch_disable(void) +{ + pmu_disable(); +} + +/* enable events counter */ +static inline int events_arch_cntr_enable(size_t counter) +{ + return pmu_cntr_enable(counter); +} + +/* disable events counter */ +static inline void events_arch_cntr_disable(size_t counter) +{ + pmu_cntr_disable(counter); +} + +/* set event counter value */ +static inline void events_arch_cntr_set(size_t counter, unsigned long value) +{ + pmu_cntr_set(counter, value); +} + +/* get event counter value */ +static inline uint64_t events_arch_get_cntr_value(size_t counter) +{ + return pmu_cntr_get(counter); +} + +/* define event to count */ +static inline void events_arch_set_evtyper(size_t counter, size_t event) +{ + pmu_set_evtyper(counter, event); +} + +/*enable events interrupt*/ +static inline void events_arch_interrupt_enable(cpuid_t cpu_id) +{ + pmu_interrupt_enable(cpu_id); +} + +/*disable events interrupt*/ +static inline void events_arch_interrupt_disable(cpuid_t cpu_id) +{ + pmu_interrupt_disable(cpu_id); +} + +/* enable event counter interrupt */ +static inline void events_arch_cntr_irq_enable(size_t counter) +{ + pmu_set_cntr_irq_enable(counter); +} + +/* disable event counter interrupt */ +static inline void events_arch_cntr_irq_disable(size_t counter) +{ + pmu_set_cntr_irq_disable(counter); +} + +static inline void events_arch_clear_cntr_ovs(size_t counter) +{ + pmu_clear_cntr_ovs(counter); +} + +/* set event counter interrupt callback */ +static inline void events_arch_cntr_set_irq_callback(irq_handler_t handler, size_t counter) +{ + pmu_define_event_cntr_irq_callback(handler, counter); +} + +#endif /* __ARCH_EVENTS_H__ */ diff --git a/src/arch/armv8/inc/arch/generic_timer.h b/src/arch/armv8/inc/arch/generic_timer.h index 4c28e97ea..9104c44ea 100644 --- a/src/arch/armv8/inc/arch/generic_timer.h +++ b/src/arch/armv8/inc/arch/generic_timer.h @@ -3,10 +3,11 @@ * Copyright (c) Bao Project and Contributors. All rights reserved. */ -#ifndef GENERIC_TIMER_H -#define GENERIC_TIMER_H +#ifndef __ARCH_GENERIC_TIMER_H__ +#define __ARCH_GENERIC_TIMER_H__ #include +#include #define GENERIC_TIMER_CNTCTL_CNTCR_EN (0x1) @@ -20,4 +21,11 @@ struct generic_timer_cntctrl { uint32_t CounterID[12]; } __attribute__((packed, aligned(PAGE_SIZE))); -#endif /* GENERIC_TIMER_H */ +uint64_t timer_arch_init(uint64_t period); +void timer_arch_enable(void); +void timer_arch_disable(void); +void timer_arch_define_irq_callback(irq_handler_t handler); +void timer_arch_reschedule_interrupt(uint64_t count); +uint64_t timer_arch_reschedule_interrupt_us(uint64_t period); + +#endif /* __ARCH_GENERIC_TIMER_H__ */ diff --git a/src/arch/armv8/inc/arch/platform.h b/src/arch/armv8/inc/arch/platform.h index b82fc2915..9b332e6ca 100644 --- a/src/arch/armv8/inc/arch/platform.h +++ b/src/arch/armv8/inc/arch/platform.h @@ -33,8 +33,13 @@ struct arch_platform { struct { paddr_t base_addr; uint32_t fixed_freq; + irqid_t interrupt_id; } generic_timer; + struct { + irqid_t interrupt_id; + } events; + struct clusters { size_t num; size_t* core_num; diff --git a/src/arch/armv8/inc/arch/pmu.h b/src/arch/armv8/inc/arch/pmu.h new file mode 100644 index 000000000..c0d9993ff --- /dev/null +++ b/src/arch/armv8/inc/arch/pmu.h @@ -0,0 +1,134 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Bao Project and Contributors. All rights reserved. + */ + +#ifndef __ARCH_PMU_H__ +#define __ARCH_PMU_H__ + +#include +#include +#include + +#define PMU_CNTR_MAX_NUM 32UL + +#define PMCR_EL0_N_POS 11UL +#define PMCR_EL0_N_MASK (0x1FUL << PMCR_EL0_N_POS) + +#define MDCR_EL2_HPME (1UL << 7) +#define MDCR_EL2_HPMN_MASK 0x1FUL + +#define PMEVTYPER_P 31UL +#define PMEVTYPER_U 30UL +#define PMEVTYPER_NSK 29UL +#define PMEVTYPER_NSU 28UL +#define PMEVTYPER_NSH 27UL +#define PMEVTYPER_M 26UL +#define PMEVTYPER_MT 25UL +#define PMEVTYPER_SH 24UL + +#define ERROR_NO_MORE_EVENT_COUNTERS ((ssize_t)-1) + +ssize_t pmu_cntr_alloc(void); +void pmu_cntr_free(size_t counter); +void pmu_configure_counter_partition(size_t mem_throt_reserved); +void pmu_enable(void); +void pmu_interrupt_enable(cpuid_t cpu_id); +void pmu_interrupt_disable(cpuid_t cpu_id); +void pmu_define_event_cntr_irq_callback(irq_handler_t handler, size_t counter); + +static inline void pmu_disable(void) +{ + uint64_t mdcr = sysreg_mdcr_el2_read(); + mdcr = bit_clear(mdcr, MDCR_EL2_HPME); + sysreg_mdcr_el2_write(mdcr); +} + +static inline int pmu_cntr_enable(size_t counter) +{ + uint64_t pmcntenset = 0; + pmcntenset = bit_set(pmcntenset, counter); + sysreg_pmcntenset_el0_write(pmcntenset); + __asm__ volatile("isb"); + return 0; +} + +static inline void pmu_cntr_disable(size_t counter) +{ + uint64_t pmcntenclr = 0; + pmcntenclr = bit_set(pmcntenclr, counter); + sysreg_pmcntenclr_el0_write(pmcntenclr); +} + +static inline void pmu_cntr_set(size_t counter, unsigned long value) +{ + uint64_t pmselr = sysreg_pmselr_el0_read(); + pmselr = bit_insert(pmselr, (uint64_t)counter, 0, 5); + sysreg_pmselr_el0_write(pmselr); + __asm__ volatile("isb"); + + if ((value == 0UL) || (value > UINT32_MAX)) { + ERROR("Invalid PMU counter budget = %lu\n", value); + } + + unsigned long preload = (UINT32_MAX - value) + 1UL; + sysreg_pmxevcntr_el0_write(preload); + __asm__ volatile("isb"); +} + +static inline unsigned long pmu_cntr_get(size_t counter) +{ + uint64_t pmselr = sysreg_pmselr_el0_read(); + pmselr = bit_insert(pmselr, (uint64_t)counter, 0, 5); + sysreg_pmselr_el0_write(pmselr); + return sysreg_pmxevcntr_el0_read(); +} + +static inline void pmu_set_evtyper(size_t counter, size_t event) +{ + uint64_t pmselr = sysreg_pmselr_el0_read(); + pmselr = bit_insert(pmselr, (uint64_t)counter, 0, 5); + sysreg_pmselr_el0_write(pmselr); + __asm__ volatile("isb"); + + uint64_t pmxevtyper = sysreg_pmxevtyper_el0_read(); + + pmxevtyper = bit_clear(pmxevtyper, PMEVTYPER_P); + pmxevtyper = bit_clear(pmxevtyper, PMEVTYPER_U); + pmxevtyper = bit_clear(pmxevtyper, PMEVTYPER_NSK); + pmxevtyper = bit_clear(pmxevtyper, PMEVTYPER_NSU); + pmxevtyper = bit_clear(pmxevtyper, PMEVTYPER_NSH); + pmxevtyper = bit_clear(pmxevtyper, PMEVTYPER_M); + pmxevtyper = bit_clear(pmxevtyper, PMEVTYPER_MT); + pmxevtyper = bit_clear(pmxevtyper, PMEVTYPER_SH); + + pmxevtyper = bit_insert(pmxevtyper, (uint64_t)event, 0, 16); + sysreg_pmxevtyper_el0_write(pmxevtyper); + __asm__ volatile("isb"); +} + +static inline void pmu_set_cntr_irq_enable(size_t counter) +{ + uint64_t pmintenset = 0; + pmintenset = bit_set(pmintenset, counter); + sysreg_pmintenset_el1_write(pmintenset); + __asm__ volatile("isb"); +} + +static inline void pmu_set_cntr_irq_disable(size_t counter) +{ + uint64_t pmintenset = 0; + pmintenset = bit_set(pmintenset, counter); + sysreg_pmintenclr_el1_write(pmintenset); + __asm__ volatile("isb"); +} + +static inline void pmu_clear_cntr_ovs(size_t counter) +{ + uint64_t pmovsclr = 0; + pmovsclr = bit_set(pmovsclr, counter); + sysreg_pmovsclr_el0_write(pmovsclr); + __asm__ volatile("isb"); +} + +#endif /* __ARCH_PMU_H__ */ diff --git a/src/arch/armv8/objects.mk b/src/arch/armv8/objects.mk index 798f36214..d85248453 100644 --- a/src/arch/armv8/objects.mk +++ b/src/arch/armv8/objects.mk @@ -12,6 +12,11 @@ cpu-objs-y+=vgic.o cpu-objs-y+=vmm.o cpu-objs-y+=psci.o +ifeq ($(ARCH_SUB),aarch64) +cpu-objs-y+=generic_timer.o +cpu-objs-y+=pmu.o +endif + ifeq ($(GIC_VERSION), GICV2) cpu-objs-y+=vgicv2.o cpu-objs-y+=gicv2.o diff --git a/src/arch/armv8/pmu.c b/src/arch/armv8/pmu.c new file mode 100644 index 000000000..5bc95686d --- /dev/null +++ b/src/arch/armv8/pmu.c @@ -0,0 +1,194 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Bao Project and Contributors. All rights reserved. + */ + +#include +#include +#include +#include +#include + +static size_t pmu_event_counters_num[CPU_MAX]; +static size_t pmu_guest_counters_num[CPU_MAX]; +static size_t pmu_mem_throt_reserved_num[CPU_MAX]; +static BITMAP_ALLOC_ARRAY(pmu_events_bitmap, PMU_CNTR_MAX_NUM, CPU_MAX); +static irq_handler_t pmu_event_handlers[CPU_MAX][PMU_CNTR_MAX_NUM]; +static irqid_t pmu_irq_id_reserved = INVALID_IRQID; +static spinlock_t pmu_irq_lock = SPINLOCK_INITVAL; + +static inline size_t pmu_reserved_total(size_t cpu_index) +{ + return pmu_mem_throt_reserved_num[cpu_index]; +} + +static inline size_t pmu_hyp_alloc_start(size_t cpu_index) +{ + return pmu_guest_counters_num[cpu_index]; +} + +static inline void pmu_refresh_event_counter_count(size_t cpu_index) +{ + uint64_t pmcr = sysreg_pmcr_el0_read(); + size_t counters_num = (size_t)((pmcr & PMCR_EL0_N_MASK) >> PMCR_EL0_N_POS); + + if (counters_num > PMU_CNTR_MAX_NUM) { + counters_num = PMU_CNTR_MAX_NUM; + } + + pmu_event_counters_num[cpu_index] = counters_num; +} + +static inline void pmu_recompute_counter_partition(size_t cpu_index) +{ + size_t reserved_total = pmu_reserved_total(cpu_index); + size_t counters_num = pmu_event_counters_num[cpu_index]; + + if (reserved_total > counters_num) { + ERROR("PMU counters requested (%zu) exceed available counters (%zu)\n", reserved_total, + counters_num); + } + + pmu_guest_counters_num[cpu_index] = counters_num - reserved_total; +} + +static inline void pmu_restore_el2_control(size_t cpu_index) +{ + uint64_t mdcr = sysreg_mdcr_el2_read(); + mdcr &= ~((uint64_t)MDCR_EL2_HPMN_MASK); + mdcr |= (uint64_t)MDCR_EL2_HPME | (uint64_t)pmu_guest_counters_num[cpu_index]; + sysreg_mdcr_el2_write(mdcr); +} + +static inline size_t pmu_cpu_index(void) +{ + if (cpu()->id >= CPU_MAX) { + ERROR("Invalid CPU id = %lu\n", cpu()->id); + } + + return (size_t)cpu()->id; +} + +static inline irqid_t pmu_irq_id(void) +{ + irqid_t irq_id = platform.arch.events.interrupt_id; + if ((irq_id == INVALID_IRQID) || (irq_id == 0U)) { + ERROR("Invalid PMU interrupt id\n"); + } + + return irq_id; +} + +ssize_t pmu_cntr_alloc(void) +{ + size_t cpu_index = pmu_cpu_index(); + size_t alloc_start = pmu_hyp_alloc_start(cpu_index); + + for (size_t index = alloc_start; index < pmu_event_counters_num[cpu_index]; index++) { + if (bitmap_get(pmu_events_bitmap[cpu_index], index) == BITMAP_NOT_SET) { + bitmap_set(pmu_events_bitmap[cpu_index], index); + return (ssize_t)index; + } + } + + return ERROR_NO_MORE_EVENT_COUNTERS; +} + +void pmu_cntr_free(size_t counter) +{ + size_t cpu_index = pmu_cpu_index(); + + if (counter >= PMU_CNTR_MAX_NUM) { + ERROR("Invalid PMU counter index = %zu\n", counter); + } + + bitmap_clear(pmu_events_bitmap[cpu_index], counter); +} + +static void pmu_interrupt_handler(irqid_t int_id) +{ + size_t cpu_index = pmu_cpu_index(); + size_t alloc_start = pmu_hyp_alloc_start(cpu_index); + uint64_t overflow_status = sysreg_pmovsclr_el0_read(); + sysreg_pmovsclr_el0_write(overflow_status); + + for (size_t counter = alloc_start; counter < pmu_event_counters_num[cpu_index]; counter++) { + if (bit64_get(overflow_status, counter) != 0U) { + irq_handler_t handler = pmu_event_handlers[cpu_index][counter]; + if (handler != NULL) { + handler(int_id); + } + } + } +} + +void pmu_configure_counter_partition(size_t mem_throt_reserved) +{ + size_t cpu_index = pmu_cpu_index(); + + pmu_mem_throt_reserved_num[cpu_index] = mem_throt_reserved; + + pmu_refresh_event_counter_count(cpu_index); + pmu_recompute_counter_partition(cpu_index); + pmu_restore_el2_control(cpu_index); +} + +void pmu_enable(void) +{ + size_t cpu_index = pmu_cpu_index(); + + pmu_refresh_event_counter_count(cpu_index); + pmu_recompute_counter_partition(cpu_index); + bitmap_clear_consecutive(pmu_events_bitmap[cpu_index], 0, PMU_CNTR_MAX_NUM); + for (size_t i = 0; i < PMU_CNTR_MAX_NUM; i++) { + pmu_event_handlers[cpu_index][i] = NULL; + } + + pmu_restore_el2_control(cpu_index); + + uint64_t pmcr = sysreg_pmcr_el0_read(); + pmcr = bit_set(pmcr, 0); // PMCR_EL0.E + sysreg_pmcr_el0_write(pmcr); + __asm__ volatile("isb"); +} + +void pmu_interrupt_enable(cpuid_t cpu_id) +{ + UNUSED_ARG(cpu_id); + size_t cpu_index = pmu_cpu_index(); + pmu_restore_el2_control(cpu_index); + + irqid_t irq_id = pmu_irq_id(); + + spin_lock(&pmu_irq_lock); + if (pmu_irq_id_reserved == INVALID_IRQID) { + pmu_irq_id_reserved = interrupts_reserve(irq_id, pmu_interrupt_handler); + if (pmu_irq_id_reserved == INVALID_IRQID) { + ERROR("Failed to assign PMU interrupt id = %u\n", irq_id); + } + } else if (pmu_irq_id_reserved != irq_id) { + ERROR("Inconsistent PMU interrupt id = %u\n", irq_id); + } + spin_unlock(&pmu_irq_lock); + + interrupts_arch_enable(pmu_irq_id_reserved, true); +} + +void pmu_interrupt_disable(cpuid_t cpu_id) +{ + UNUSED_ARG(cpu_id); + if (pmu_irq_id_reserved != INVALID_IRQID) { + interrupts_arch_enable(pmu_irq_id_reserved, false); + } +} + +void pmu_define_event_cntr_irq_callback(irq_handler_t handler, size_t counter) +{ + size_t cpu_index = pmu_cpu_index(); + + if (counter >= PMU_CNTR_MAX_NUM) { + ERROR("Invalid PMU counter index = %zu\n", counter); + } + + pmu_event_handlers[cpu_index][counter] = handler; +} diff --git a/src/core/inc/config.h b/src/core/inc/config.h index 89f1589e0..7b36963bf 100644 --- a/src/core/inc/config.h +++ b/src/core/inc/config.h @@ -95,6 +95,11 @@ struct vm_config { */ colormap_t colors; + struct { + uint64_t budget; + uint64_t period_us; + } mem_throth; + /** * A description of the virtual platform available to the guest, i.e., the virtual machine * itself. diff --git a/src/core/inc/events.h b/src/core/inc/events.h new file mode 100644 index 000000000..472c27952 --- /dev/null +++ b/src/core/inc/events.h @@ -0,0 +1,97 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Bao Project and Contributors. All rights reserved. + */ + +#ifndef __EVENTS_H__ +#define __EVENTS_H__ + +#include + +#define EVENTS_CNTR_MAX_NUM EVENTS_ARCH_CNTR_MAX_NUM + +typedef size_t events_event_t; + +#define EVENTS_EVENT_INVALID ((events_event_t)~0UL) + +static inline ssize_t events_cntr_alloc(void) +{ + return events_arch_cntr_alloc(); +} + +static inline void events_cntr_free(size_t counter) +{ + events_arch_cntr_free(counter); +} + +static inline void events_configure_counter_partition(size_t mem_throt_reserved) +{ + events_arch_configure_counter_partition(mem_throt_reserved); +} + +static inline void events_enable(void) +{ + events_arch_enable(); +} + +static inline void events_disable(void) +{ + events_arch_disable(); +} + +static inline int events_cntr_enable(size_t counter) +{ + return events_arch_cntr_enable(counter); +} + +static inline void events_cntr_disable(size_t counter) +{ + events_arch_cntr_disable(counter); +} + +static inline void events_cntr_set(size_t counter, unsigned long value) +{ + events_arch_cntr_set(counter, value); +} + +static inline uint64_t events_get_cntr_value(size_t counter) +{ + return events_arch_get_cntr_value(counter); +} + +static inline void events_set_evtyper(size_t counter, events_event_t event) +{ + events_arch_set_evtyper(counter, (size_t)event); +} + +static inline void events_interrupt_enable(cpuid_t cpu_id) +{ + events_arch_interrupt_enable(cpu_id); +} + +static inline void events_interrupt_disable(cpuid_t cpu_id) +{ + events_arch_interrupt_disable(cpu_id); +} + +static inline void events_cntr_irq_enable(size_t counter) +{ + events_arch_cntr_irq_enable(counter); +} + +static inline void events_cntr_irq_disable(size_t counter) +{ + events_arch_cntr_irq_disable(counter); +} + +static inline void events_clear_cntr_ovs(size_t counter) +{ + events_arch_clear_cntr_ovs(counter); +} + +static inline void events_cntr_set_irq_callback(irq_handler_t handler, size_t counter) +{ + events_arch_cntr_set_irq_callback(handler, counter); +} + +#endif /* __EVENTS_H__ */ diff --git a/src/core/inc/generic_timer.h b/src/core/inc/generic_timer.h new file mode 100644 index 000000000..4539212a2 --- /dev/null +++ b/src/core/inc/generic_timer.h @@ -0,0 +1,41 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Bao Project and Contributors. All rights reserved + */ + +#ifndef __GENERIC_TIMER_H__ +#define __GENERIC_TIMER_H__ + +#include + +static inline uint64_t generic_timer_init(uint64_t period) +{ + return timer_init(period); +} + +static inline void generic_timer_enable(void) +{ + timer_enable(); +} + +static inline void generic_timer_disable(void) +{ + timer_disable(); +} + +static inline void generic_timer_define_irq_callback(irq_handler_t handler) +{ + timer_define_irq_callback(handler); +} + +static inline void generic_timer_reschedule_interrupt(uint64_t count) +{ + timer_reschedule_interrupt(count); +} + +static inline uint64_t generic_timer_reschedule_interrupt_us(uint64_t period) +{ + return timer_reschedule_interrupt_us(period); +} + +#endif /* __GENERIC_TIMER_H__ */ diff --git a/src/core/inc/interrupts.h b/src/core/inc/interrupts.h index d035443d2..dcb6755e0 100644 --- a/src/core/inc/interrupts.h +++ b/src/core/inc/interrupts.h @@ -40,6 +40,7 @@ void interrupts_ipi_clear(void); enum irq_res { HANDLED_BY_HYP, FORWARD_TO_VM }; enum irq_res interrupts_handle(irqid_t int_id); +void interrupts_post_handler(void); bool interrupts_vm_assign(struct vm* vm, irqid_t id); diff --git a/src/core/inc/mem_throt.h b/src/core/inc/mem_throt.h new file mode 100644 index 000000000..367975585 --- /dev/null +++ b/src/core/inc/mem_throt.h @@ -0,0 +1,25 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Bao Project and Contributors. All rights reserved + */ + +#ifndef __MEM_THROT_H__ +#define __MEM_THROT_H__ + +#include +#include + +#define MEM_THROT_PMU_COUNTERS 1UL + +typedef struct mem_throt_info { + unsigned long budget; + bool throttled; + bool initialized; + size_t counter_id; + uint64_t period_us; + uint64_t period_counts; +} mem_throt_t; + +void mem_throt_init(unsigned long budget, uint64_t period_us); + +#endif /* __MEM_THROT_H__ */ diff --git a/src/core/inc/timer.h b/src/core/inc/timer.h new file mode 100644 index 000000000..5fb1a09b1 --- /dev/null +++ b/src/core/inc/timer.h @@ -0,0 +1,42 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Bao Project and Contributors. All rights reserved. + */ + +#ifndef __TIMER_H__ +#define __TIMER_H__ + +#include +#include + +static inline uint64_t timer_init(uint64_t period) +{ + return timer_arch_init(period); +} + +static inline void timer_enable(void) +{ + timer_arch_enable(); +} + +static inline void timer_disable(void) +{ + timer_arch_disable(); +} + +static inline void timer_define_irq_callback(irq_handler_t handler) +{ + timer_arch_define_irq_callback(handler); +} + +static inline void timer_reschedule_interrupt(uint64_t count) +{ + timer_arch_reschedule_interrupt(count); +} + +static inline uint64_t timer_reschedule_interrupt_us(uint64_t period) +{ + return timer_arch_reschedule_interrupt_us(period); +} + +#endif /* __TIMER_H__ */ diff --git a/src/core/interrupts.c b/src/core/interrupts.c index 1426a24fd..ec4694077 100644 --- a/src/core/interrupts.c +++ b/src/core/interrupts.c @@ -17,6 +17,10 @@ irq_handler_t interrupt_handlers[MAX_INTERRUPT_HANDLERS]; irqid_t interrupts_ipi_id; +__attribute__((weak)) void interrupts_post_handler(void) +{ +} + void interrupts_cpu_sendipi(cpuid_t target_cpu) { interrupts_arch_ipi_send(target_cpu); diff --git a/src/core/mem_throt.c b/src/core/mem_throt.c new file mode 100644 index 000000000..d81239740 --- /dev/null +++ b/src/core/mem_throt.c @@ -0,0 +1,129 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Bao Project and Contributors. All rights reserved + */ + +#include +#include +#include +#include + +#ifndef PLAT_MEM_THROT_EVENT +#define PLAT_MEM_THROT_EVENT EVENTS_EVENT_INVALID +#endif + +static mem_throt_t mem_throt_state[CPU_MAX]; + +static inline mem_throt_t* mem_throt_get_state(void) +{ + if (cpu()->id >= CPU_MAX) { + ERROR("Invalid CPU id = %lu\n", cpu()->id); + } + + return &mem_throt_state[cpu()->id]; +} + +static inline void mem_throt_rearm_counter(mem_throt_t* state) +{ + events_cntr_set(state->counter_id, state->budget); + + /* Restore PMU IRQ delivery in case low-power transitions clear PMU/GIC enable + * bits. */ + events_interrupt_enable(cpu()->id); + events_clear_cntr_ovs(state->counter_id); + events_cntr_irq_enable(state->counter_id); + events_cntr_enable(state->counter_id); +} + +static inline events_event_t mem_throt_platform_event(void) +{ + events_event_t event = (events_event_t)PLAT_MEM_THROT_EVENT; + + if (event == EVENTS_EVENT_INVALID) { + ERROR("Missing platform memory throttling event\n"); + } + + return event; +} + +static void mem_throt_period_timer_callback(irqid_t int_id) +{ + UNUSED_ARG(int_id); + mem_throt_t* state = mem_throt_get_state(); + + timer_disable(); + events_cntr_disable(state->counter_id); + mem_throt_rearm_counter(state); + state->throttled = false; + timer_reschedule_interrupt(state->period_counts); + timer_enable(); +} + +static void mem_throt_event_overflow_callback(irqid_t int_id) +{ + UNUSED_ARG(int_id); + mem_throt_t* state = mem_throt_get_state(); + + events_cntr_disable(state->counter_id); + events_cntr_irq_disable(state->counter_id); + state->throttled = true; +} + +void interrupts_post_handler(void) +{ + mem_throt_t* state = mem_throt_get_state(); + + if (state->initialized && state->throttled) { + cpu_standby(); + } +} + +static void mem_throt_timer_init(irq_handler_t handler) +{ + mem_throt_t* state = mem_throt_get_state(); + + timer_define_irq_callback(handler); + state->period_counts = timer_reschedule_interrupt_us(state->period_us); + timer_enable(); +} + +static void mem_throt_events_init(events_event_t event, unsigned long budget, irq_handler_t handler) +{ + mem_throt_t* state = mem_throt_get_state(); + events_enable(); + + ssize_t counter = events_cntr_alloc(); + if (counter == ERROR_NO_MORE_EVENT_COUNTERS) { + ERROR("No more event counters!\n"); + } + + state->counter_id = (size_t)counter; + + events_set_evtyper(state->counter_id, event); + events_cntr_set(state->counter_id, budget); + events_cntr_set_irq_callback(handler, state->counter_id); + events_clear_cntr_ovs(state->counter_id); + events_interrupt_enable(cpu()->id); + events_cntr_irq_enable(state->counter_id); + events_cntr_enable(state->counter_id); +} + +void mem_throt_init(unsigned long budget, uint64_t period_us) +{ + mem_throt_t* state = mem_throt_get_state(); + + if (period_us == 0U) { + ERROR("Invalid memory throttling period\n"); + } + + state->throttled = false; + state->initialized = false; + state->budget = budget; + state->period_us = period_us; + + events_configure_counter_partition(1); + mem_throt_timer_init(mem_throt_period_timer_callback); + mem_throt_events_init(mem_throt_platform_event(), state->budget, + mem_throt_event_overflow_callback); + state->initialized = true; +} diff --git a/src/core/objects.mk b/src/core/objects.mk index 488857932..7ad850037 100644 --- a/src/core/objects.mk +++ b/src/core/objects.mk @@ -16,3 +16,9 @@ core-objs-y+=hypercall.o core-objs-y+=shmem.o core-objs-y+=remio.o core-objs-y+=platform.o + +ifeq ($(ARCH),armv8) +ifeq ($(ARCH_SUB),aarch64) +core-objs-y+=mem_throt.o +endif +endif diff --git a/src/core/vm.c b/src/core/vm.c index 11bbe8812..cfe80ff7b 100644 --- a/src/core/vm.c +++ b/src/core/vm.c @@ -11,6 +11,20 @@ #include #include #include +#include +#include + + +static inline bool vm_mem_throt_enabled(const struct vm_config* vm_config) +{ + return (vm_config->mem_throth.budget > 0U) && (vm_config->mem_throth.period_us > 0U); +} + +static inline void vm_configure_pmu_counter_partition(const struct vm_config* vm_config) +{ + size_t mem_throt_reserved = vm_mem_throt_enabled(vm_config) ? MEM_THROT_PMU_COUNTERS : 0U; + events_configure_counter_partition(mem_throt_reserved); +} static void vm_master_init(struct vm* vm, const struct vm_config* vm_config, vmid_t vm_id) { @@ -345,6 +359,11 @@ struct vm* vm_init(struct vm_allocation* vm_alloc, struct cpu_synctoken* vm_init vm_init_remio(vm, vm_config); } + vm_configure_pmu_counter_partition(vm_config); + if (vm_mem_throt_enabled(vm_config)) { + mem_throt_init(vm_config->mem_throth.budget, vm_config->mem_throth.period_us); + } + cpu_sync_and_clear_msgs(&vm->sync); return vm; diff --git a/src/platform/zcu104/inc/plat/platform.h b/src/platform/zcu104/inc/plat/platform.h index 7c5893789..21cf2f418 100644 --- a/src/platform/zcu104/inc/plat/platform.h +++ b/src/platform/zcu104/inc/plat/platform.h @@ -8,4 +8,19 @@ #include +#define PLAT_EVENT_L1D_CACHE_REFILL 0x03U +#define PLAT_EVENT_DATA_MEMORY_ACCESS 0x13U +#define PLAT_EVENT_L2D_CACHE_ACCESS 0x16U +#define PLAT_EVENT_L2D_CACHE_REFILL 0x17U +#define PLAT_EVENT_BUS_ACCESS 0x19U +#define PLAT_EVENT_EXTERNAL_MEMORY_REQUEST 0xC0U + +#define PLAT_MEM_THROT_EVENT PLAT_EVENT_L1D_CACHE_REFILL + +#define DATA_MEMORY_ACCESS PLAT_EVENT_DATA_MEMORY_ACCESS +#define L2D_CACHE_ACCESS PLAT_EVENT_L2D_CACHE_ACCESS +#define L2D_CACHE_REFILL PLAT_EVENT_L2D_CACHE_REFILL +#define BUS_ACCESS PLAT_EVENT_BUS_ACCESS +#define EXTERNAL_MEMORY_REQUEST PLAT_EVENT_EXTERNAL_MEMORY_REQUEST + #endif diff --git a/src/platform/zcu104/zcu104_desc.c b/src/platform/zcu104/zcu104_desc.c index 423249c89..e58c3f89d 100644 --- a/src/platform/zcu104/zcu104_desc.c +++ b/src/platform/zcu104/zcu104_desc.c @@ -47,6 +47,11 @@ struct platform platform = { .generic_timer = { .base_addr = 0xFF260000, + .interrupt_id = 26 + }, + + .events = { + .interrupt_id = 175, }, }, };