diff options
Diffstat (limited to 'bsps/aarch64/shared/cache/cache.c')
| -rw-r--r-- | bsps/aarch64/shared/cache/cache.c | 616 |
1 files changed, 616 insertions, 0 deletions
diff --git a/bsps/aarch64/shared/cache/cache.c b/bsps/aarch64/shared/cache/cache.c new file mode 100644 index 0000000000..d7ea33206f --- /dev/null +++ b/bsps/aarch64/shared/cache/cache.c @@ -0,0 +1,616 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ + +/** + * @file + * + * @ingroup RTEMSBSPsAArch64Shared + * + * @brief AArch64 cache defines and implementation. + */ + +/* + * Copyright (C) 2020 On-Line Applications Research Corporation (OAR) + * Written by Kinsey Moore <kinsey.moore@oarcorp.com> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rtems.h> +#include <bsp.h> +#include <bsp/utility.h> + +#define AARCH64_CACHE_L1_CPU_DATA_ALIGNMENT ((size_t)64) +#define AARCH64_CACHE_L1_DATA_LINE_MASK \ + ( AARCH64_CACHE_L1_CPU_DATA_ALIGNMENT - 1 ) +#define AARCH64_CACHE_PREPARE_MVA(mva) \ + ((const void *) (((size_t) (mva)) & AARCH64_CACHE_L1_DATA_LINE_MASK)) + +static inline +void AArch64_data_cache_clean_and_invalidate_line(const void *d_addr) +{ + d_addr = AARCH64_CACHE_PREPARE_MVA(d_addr); + + __asm__ volatile ( + "dc civac, %[d_addr]" + : + : [d_addr] "r" (d_addr) + : "memory" + ); +} + +static inline void _CPU_cache_flush_1_data_line(const void *d_addr) +{ + /* Flush the Data cache */ + AArch64_data_cache_clean_and_invalidate_line( d_addr ); + + /* Wait for L1 flush to complete */ + _AARCH64_Data_synchronization_barrier(); +} + +static inline void +_CPU_cache_flush_data_range( + const void *d_addr, + size_t n_bytes +) +{ + _AARCH64_Data_synchronization_barrier(); + if ( n_bytes != 0 ) { + size_t adx = (size_t) d_addr & ~AARCH64_CACHE_L1_DATA_LINE_MASK; + const size_t ADDR_LAST = (size_t) d_addr + n_bytes - 1; + + for (; adx <= ADDR_LAST; adx += AARCH64_CACHE_L1_CPU_DATA_ALIGNMENT ) { + /* Store and invalidate the Data cache line */ + AArch64_data_cache_clean_and_invalidate_line( (void*)adx ); + } + /* Wait for L1 store to complete */ + _AARCH64_Data_synchronization_barrier(); + } + _AARCH64_Data_synchronization_barrier(); +} + +static inline void AArch64_data_cache_invalidate_line(const void *d_addr) +{ + d_addr = AARCH64_CACHE_PREPARE_MVA(d_addr); + + __asm__ volatile ( + "dc ivac, %[d_addr]" + : + : [d_addr] "r" (d_addr) + : "memory" + ); +} + +static inline void _CPU_cache_invalidate_1_data_line(const void *d_addr) +{ + /* Invalidate the data cache line */ + AArch64_data_cache_invalidate_line( d_addr ); + + /* Wait for L1 invalidate to complete */ + _AARCH64_Data_synchronization_barrier(); +} + +static inline void +_CPU_cache_invalidate_data_range( + const void *d_addr, + size_t n_bytes +) +{ + if ( n_bytes != 0 ) { + size_t adx = (size_t) d_addr + & ~AARCH64_CACHE_L1_DATA_LINE_MASK; + const size_t end = (size_t)d_addr + n_bytes -1; + + /* Back starting address up to start of a line and invalidate until end */ + for (; + adx <= end; + adx += AARCH64_CACHE_L1_CPU_DATA_ALIGNMENT ) { + /* Invalidate the Instruction cache line */ + AArch64_data_cache_invalidate_line( (void*)adx ); + } + /* Wait for L1 invalidate to complete */ + _AARCH64_Data_synchronization_barrier(); + } +} + +static inline void _CPU_cache_freeze_data(void) +{ + /* TODO */ +} + +static inline void _CPU_cache_unfreeze_data(void) +{ + /* TODO */ +} + +static inline void AArch64_instruction_cache_invalidate_line(const void *i_addr) +{ + /* __builtin___clear_cache is explicitly only for instruction cacche */ + __builtin___clear_cache((void *)i_addr, ((char *)i_addr) + sizeof(void*) - 1); +} + +static inline void _CPU_cache_invalidate_1_instruction_line(const void *d_addr) +{ + /* Invalidate the Instruction cache line */ + AArch64_instruction_cache_invalidate_line( d_addr ); + + /* Wait for L1 invalidate to complete */ + _AARCH64_Data_synchronization_barrier(); +} + +static inline void +_CPU_cache_invalidate_instruction_range( const void *i_addr, size_t n_bytes) +{ + if ( n_bytes != 0 ) { + __builtin___clear_cache((void *)i_addr, ((char *)i_addr) + n_bytes - 1); + } + _AARCH64_Instruction_synchronization_barrier(); +} + +static inline void _CPU_cache_freeze_instruction(void) +{ + /* TODO */ +} + +static inline void _CPU_cache_unfreeze_instruction(void) +{ + /* TODO */ +} + +static inline uint64_t +AArch64_get_ccsidr(void) +{ + uint64_t val; + + __asm__ volatile ( + "mrs %[val], CCSIDR_EL1\n" + : [val] "=&r" (val) + ); + + return val; +} + +#define CCSIDR_NUMSETS(val) BSP_FLD64(val, 13, 27) +#define CCSIDR_NUMSETS_GET(reg) BSP_FLD64GET(reg, 13, 27) +#define CCSIDR_NUMSETS_SET(reg, val) BSP_FLD64SET(reg, val, 13, 27) +#define CCSIDR_ASSOCIATIVITY(val) BSP_FLD64(val, 3, 12) +#define CCSIDR_ASSOCIATIVITY_GET(reg) BSP_FLD64GET(reg, 3, 12) +#define CCSIDR_ASSOCIATIVITY_SET(reg, val) BSP_FLD64SET(reg, val, 3, 12) +/* line size == 1 << (GET(reg)+4): 0 -> (1 << 4) == 16 */ +#define CCSIDR_LINE_SIZE(val) BSP_FLD64(val, 0, 2) +#define CCSIDR_LINE_SIZE_GET(reg) BSP_FLD64GET(reg, 0, 2) +#define CCSIDR_LINE_SIZE_SET(reg, val) BSP_FLD64SET(reg, val, 0, 2) + +static inline uint64_t +AArch64_ccsidr_get_line_power(uint64_t ccsidr) +{ + return CCSIDR_LINE_SIZE_GET(ccsidr) + 4; +} + +static inline uint64_t +AArch64_ccsidr_get_associativity(uint64_t ccsidr) +{ + return CCSIDR_ASSOCIATIVITY_GET(ccsidr) + 1; +} + +static inline uint64_t +AArch64_ccsidr_get_num_sets(uint64_t ccsidr) +{ + return CCSIDR_NUMSETS_GET(ccsidr) + 1; +} + +static inline void +AArch64_set_csselr(uint64_t val) +{ + __asm__ volatile ( + "msr CSSELR_EL1, %[val]\n" + : + : [val] "r" (val) + ); +} +#define CSSELR_TND BSP_BIT64(4) +/* This field is level-1: L1 cache is 0, L2 cache is 1, etc */ +#define CSSELR_LEVEL(val) BSP_FLD64(val, 1, 3) +#define CSSELR_LEVEL_GET(reg) BSP_FLD64GET(reg, 1, 3) +#define CSSELR_LEVEL_SET(reg, val) BSP_FLD64SET(reg, val, 1, 3) +#define CSSELR_IND BSP_BIT64(0) + +static inline uint64_t AArch64_get_ccsidr_for_level(uint64_t val) +{ + AArch64_set_csselr(val); + return AArch64_get_ccsidr(); +} + +static inline void AArch64_data_cache_clean_level(uint64_t level) +{ + uint64_t ccsidr; + uint64_t line_power; + uint64_t associativity; + uint64_t way; + uint64_t way_shift; + + ccsidr = AArch64_get_ccsidr_for_level(CSSELR_LEVEL(level)); + + line_power = AArch64_ccsidr_get_line_power(ccsidr); + associativity = AArch64_ccsidr_get_associativity(ccsidr); + way_shift = __builtin_clz(associativity - 1); + + for (way = 0; way < associativity; ++way) { + uint64_t num_sets = AArch64_ccsidr_get_num_sets(ccsidr); + uint64_t set; + + for (set = 0; set < num_sets; ++set) { + uint64_t set_and_way = (way << way_shift) + | (set << line_power) + | (level << 1); + + __asm__ volatile ( + "dc csw, %[set_and_way]" + : + : [set_and_way] "r" (set_and_way) + : "memory" + ); + } + } +} + +static inline uint64_t +AArch64_get_clidr(void) +{ + uint64_t val; + + __asm__ volatile ( + "mrs %[val], CLIDR_EL1\n" + : [val] "=&r" (val) + ); + + return val; +} + +#define CLIDR_LOC(val) BSP_FLD64(val, 24, 26) +#define CLIDR_LOC_GET(reg) BSP_FLD64GET(reg, 24, 26) +#define CLIDR_LOC_SET(reg, val) BSP_FLD64SET(reg, val, 24, 26) +#define CLIDR_CTYPE7(val) BSP_FLD64(val, 18, 20) +#define CLIDR_CTYPE7_GET(reg) BSP_FLD64GET(reg, 18, 20) +#define CLIDR_CTYPE7_SET(reg, val) BSP_FLD64SET(reg, val, 18, 20) +#define CLIDR_CTYPE6(val) BSP_FLD64(val, 15, 17) +#define CLIDR_CTYPE6_GET(reg) BSP_FLD64GET(reg, 15, 17) +#define CLIDR_CTYPE6_SET(reg, val) BSP_FLD64SET(reg, val, 15, 17) +#define CLIDR_CTYPE5(val) BSP_FLD64(val, 12, 14) +#define CLIDR_CTYPE5_GET(reg) BSP_FLD64GET(reg, 12, 14) +#define CLIDR_CTYPE5_SET(reg, val) BSP_FLD64SET(reg, val, 12, 14) +#define CLIDR_CTYPE4(val) BSP_FLD64(val, 9, 11) +#define CLIDR_CTYPE4_GET(reg) BSP_FLD64GET(reg, 9, 11) +#define CLIDR_CTYPE4_SET(reg, val) BSP_FLD64SET(reg, val, 9, 11) +#define CLIDR_CTYPE3(val) BSP_FLD64(val, 6, 8) +#define CLIDR_CTYPE3_GET(reg) BSP_FLD64GET(reg, 6, 8) +#define CLIDR_CTYPE3_SET(reg, val) BSP_FLD64SET(reg, val, 6, 8) +#define CLIDR_CTYPE2(val) BSP_FLD64(val, 3, 5) +#define CLIDR_CTYPE2_GET(reg) BSP_FLD64GET(reg, 3, 5) +#define CLIDR_CTYPE2_SET(reg, val) BSP_FLD64SET(reg, val, 3, 5) +#define CLIDR_CTYPE1(val) BSP_FLD64(val, 0, 2) +#define CLIDR_CTYPE1_GET(reg) BSP_FLD64GET(reg, 0, 2) +#define CLIDR_CTYPE1_SET(reg, val) BSP_FLD64SET(reg, val, 0, 2) + +static inline +uint64_t AArch64_clidr_get_cache_type(uint64_t clidr, uint64_t level) +{ + switch (level) + { + case 1: + return CLIDR_CTYPE1_GET(clidr); + case 2: + return CLIDR_CTYPE2_GET(clidr); + case 3: + return CLIDR_CTYPE3_GET(clidr); + case 4: + return CLIDR_CTYPE4_GET(clidr); + case 5: + return CLIDR_CTYPE5_GET(clidr); + case 6: + return CLIDR_CTYPE6_GET(clidr); + case 7: + return CLIDR_CTYPE7_GET(clidr); + default: + return 0; + } +} + +static inline uint64_t AArch64_clidr_get_level_of_coherency(uint64_t clidr) +{ + return CLIDR_LOC_GET(clidr); +} + +static inline void AArch64_data_cache_clean_all_levels(void) +{ + uint64_t clidr = AArch64_get_clidr(); + uint64_t loc = AArch64_clidr_get_level_of_coherency(clidr); + uint64_t level = 0; + + for (level = 0; level < loc; ++level) { + uint64_t ctype = AArch64_clidr_get_cache_type(clidr, level); + + /* Check if this level has a data cache or unified cache */ + if (((ctype & (0x6)) == 2) || (ctype == 4)) { + AArch64_data_cache_clean_level(level); + } + } +} + +static inline void _CPU_cache_flush_entire_data(void) +{ + _AARCH64_Data_synchronization_barrier(); + AArch64_data_cache_clean_all_levels(); + _AARCH64_Data_synchronization_barrier(); +} + +static inline void AArch64_cache_invalidate_level(uint64_t level) +{ + uint64_t ccsidr; + uint64_t line_power; + uint64_t associativity; + uint64_t way; + uint64_t way_shift; + + ccsidr = AArch64_get_ccsidr_for_level(CSSELR_LEVEL(level)); + + line_power = AArch64_ccsidr_get_line_power(ccsidr); + associativity = AArch64_ccsidr_get_associativity(ccsidr); + way_shift = __builtin_clz(associativity - 1); + + for (way = 0; way < associativity; ++way) { + uint64_t num_sets = AArch64_ccsidr_get_num_sets(ccsidr); + uint64_t set; + + for (set = 0; set < num_sets; ++set) { + uint64_t set_and_way = (way << way_shift) + | (set << line_power) + | (level << 1); + + __asm__ volatile ( + "dc isw, %[set_and_way]" + : + : [set_and_way] "r" (set_and_way) + : "memory" + ); + } + } +} + +static inline void AArch64_data_cache_invalidate_all_levels(void) +{ + uint64_t clidr = AArch64_get_clidr(); + uint64_t loc = AArch64_clidr_get_level_of_coherency(clidr); + uint64_t level = 0; + + for (level = 0; level < loc; ++level) { + uint64_t ctype = AArch64_clidr_get_cache_type(clidr, level); + + /* Check if this level has a data cache or unified cache */ + if ((ctype & 0x2) || (ctype == 4)) { + AArch64_cache_invalidate_level(level); + } + } +} + +static inline void _CPU_cache_invalidate_entire_data(void) +{ + AArch64_data_cache_invalidate_all_levels(); +} + +static inline uint64_t +AArch64_get_sctlr(void) +{ + uint64_t val; + + __asm__ volatile ( + "mrs %[val], SCTLR_EL1\n" + : [val] "=&r" (val) + ); + + return val; +} + +static inline void +AArch64_set_sctlr(uint64_t val) +{ + __asm__ volatile ( + "msr SCTLR_EL1, %[val]\n" + : + : [val] "r" (val) + ); +} + +#define SCTLR_TWEDEL(val) BSP_FLD64(val, 46, 49) +#define SCTLR_TWEDEL_GET(reg) BSP_FLD64GET(reg, 46, 49) +#define SCTLR_TWEDEL_SET(reg, val) BSP_FLD64SET(reg, val, 46, 49) +#define SCTLR_TWEDEN BSP_BIT64(45) +#define SCTLR_DSSBS BSP_BIT64(44) +#define SCTLR_ATA BSP_BIT64(43) +#define SCTLR_ATA0 BSP_BIT64(42) +#define SCTLR_TCF(val) BSP_FLD64(val, 40, 41) +#define SCTLR_TCF_GET(reg) BSP_FLD64GET(reg, 40, 41) +#define SCTLR_TCF_SET(reg, val) BSP_FLD64SET(reg, val, 40, 41) +#define SCTLR_TCF0(val) BSP_FLD64(val, 38, 39) +#define SCTLR_TCF0_GET(reg) BSP_FLD64GET(reg, 38, 39) +#define SCTLR_TCF0_SET(reg, val) BSP_FLD64SET(reg, val, 38, 39) +#define SCTLR_ITFSB BSP_BIT64(37) +#define SCTLR_BT1 BSP_BIT64(36) +#define SCTLR_BT0 BSP_BIT64(35) +#define SCTLR_ENIA BSP_BIT64(31) +#define SCTLR_ENIB BSP_BIT64(30) +#define SCTLR_LSMAOE BSP_BIT64(29) +#define SCTLR_NTLSMD BSP_BIT64(28) +#define SCTLR_ENDA BSP_BIT64(27) +#define SCTLR_UCI BSP_BIT64(26) +#define SCTLR_EE BSP_BIT64(25) +#define SCTLR_E0E BSP_BIT64(24) +#define SCTLR_SPAN BSP_BIT64(23) +#define SCTLR_EIS BSP_BIT64(22) +#define SCTLR_IESB BSP_BIT64(21) +#define SCTLR_TSCXT BSP_BIT64(20) +#define SCTLR_WXN BSP_BIT64(19) +#define SCTLR_NTWE BSP_BIT64(18) +#define SCTLR_NTWI BSP_BIT64(16) +#define SCTLR_UCT BSP_BIT64(15) +#define SCTLR_DZE BSP_BIT64(14) +#define SCTLR_ENDB BSP_BIT64(13) +#define SCTLR_I BSP_BIT64(12) +#define SCTLR_EOS BSP_BIT64(11) +#define SCTLR_ENRCTX BSP_BIT64(10) +#define SCTLR_UMA BSP_BIT64(9) +#define SCTLR_SED BSP_BIT64(8) +#define SCTLR_ITD BSP_BIT64(7) +#define SCTLR_NAA BSP_BIT64(6) +#define SCTLR_CP15BEN BSP_BIT64(5) +#define SCTLR_SA0 BSP_BIT64(4) +#define SCTLR_SA BSP_BIT64(3) +#define SCTLR_C BSP_BIT64(2) +#define SCTLR_A BSP_BIT64(1) +#define SCTLR_M BSP_BIT64(0) + +static inline void _CPU_cache_enable_data(void) +{ + rtems_interrupt_level level; + uint64_t sctlr; + + rtems_interrupt_local_disable(level); + sctlr = AArch64_get_sctlr(); + sctlr |= SCTLR_C; + AArch64_set_sctlr(sctlr); + rtems_interrupt_local_enable(level); +} + +static inline void _CPU_cache_disable_data(void) +{ + rtems_interrupt_level level; + uint64_t sctlr; + + rtems_interrupt_local_disable(level); + AArch64_data_cache_clean_all_levels(); + AArch64_data_cache_invalidate_all_levels(); + sctlr = AArch64_get_sctlr(); + sctlr &= ~SCTLR_C; + AArch64_set_sctlr(sctlr); + rtems_interrupt_local_enable(level); +} + +static inline +void AArch64_instruction_cache_inner_shareable_invalidate_all(void) +{ + __asm__ volatile ( + "ic ialluis\n" + : + : + : "memory" + ); +} + +static inline void AArch64_instruction_cache_invalidate(void) +{ + __asm__ volatile ( + "ic iallu\n" + : + : + : "memory" + ); +} + +static inline void _CPU_cache_invalidate_entire_instruction(void) +{ + /* There is no way to manage branch prediction in AArch64. + * See D4.4.12 in the ARMv8 technical manual. */ + + #ifdef RTEMS_SMP + /* invalidate I-cache inner shareable */ + AArch64_instruction_cache_inner_shareable_invalidate_all(); + #endif /* RTEMS_SMP */ + + /* I+BTB cache invalidate */ + AArch64_instruction_cache_invalidate(); + + _AARCH64_Instruction_synchronization_barrier(); +} + +static inline void _CPU_cache_enable_instruction(void) +{ + rtems_interrupt_level level; + uint64_t sctlr; + + rtems_interrupt_local_disable(level); + sctlr = AArch64_get_sctlr(); + sctlr |= SCTLR_I; + AArch64_set_sctlr(sctlr); + rtems_interrupt_local_enable(level); +} + +static inline void _CPU_cache_disable_instruction(void) +{ + rtems_interrupt_level level; + uint64_t sctlr; + + rtems_interrupt_local_disable(level); + sctlr = AArch64_get_sctlr(); + sctlr &= ~SCTLR_I; + AArch64_set_sctlr(sctlr); + rtems_interrupt_local_enable(level); +} + +static inline size_t AArch64_get_cache_size( + uint64_t level, + bool instruction +) +{ + uint64_t clidr; + uint64_t loc; + uint64_t ccsidr; + + clidr = AArch64_get_clidr(); + loc = AArch64_clidr_get_level_of_coherency(clidr); + + if (level >= loc) { + return 0; + } + + if (level == 0) { + level = loc - 1; + } + + ccsidr = AArch64_get_ccsidr_for_level( + CSSELR_LEVEL(level) | (instruction ? CSSELR_IND : 0) + ); + + return (1U << (AArch64_ccsidr_get_line_power(ccsidr)+4)) + * AArch64_ccsidr_get_associativity(ccsidr) + * AArch64_ccsidr_get_num_sets(ccsidr); +} + +static inline size_t _CPU_cache_get_data_cache_size(uint64_t level) +{ + return AArch64_get_cache_size(level, false); +} + +static inline size_t _CPU_cache_get_instruction_cache_size(uint64_t level) +{ + return AArch64_get_cache_size(level, true); +} + +#include "../../shared/cache/cacheimpl.h" |