diff --git a/arch/Kconfig b/arch/Kconfig index 39800a1b7b7f..1e28103509f2 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -730,6 +730,13 @@ config CPU_HAS_DCACHE help This hidden configuration should be selected when the CPU has a d-cache. +config CPU_CACHE_INCOHERENT + bool + help + This hidden configuration should be selected when the CPU has + incoherent cache. This applies to intra-CPU multiprocessing + incoherence and makes only sense when MP_NUM_CPUS > 1. + config CPU_HAS_ICACHE bool help @@ -925,6 +932,17 @@ config ICACHE help This option enables the support for the instruction cache (i-cache). +config CACHE_DOUBLEMAP + bool "Cache double-mapping support" + depends on CPU_CACHE_INCOHERENT + default y + help + Double-mapping behavior where a pointer can be cheaply converted to + point to the same cached/uncached memory at different locations. + + This applies to intra-CPU multiprocessing incoherence and makes only + sense when MP_NUM_CPUS > 1. + config CACHE_MANAGEMENT bool "Cache management features" depends on DCACHE || ICACHE diff --git a/doc/zephyr.doxyfile.in b/doc/zephyr.doxyfile.in index 2332b2f5932d..29b4651765b5 100644 --- a/doc/zephyr.doxyfile.in +++ b/doc/zephyr.doxyfile.in @@ -2399,6 +2399,7 @@ PREDEFINED = __DOXYGEN__ \ XEN_GUEST_HANDLE_64(x)= \ _LINKER \ __deprecated= \ + __sparse_cache= \ __packed= \ __aligned(x)= \ __attribute_nonnull(...)= \ diff --git a/include/zephyr/arch/cache.h b/include/zephyr/arch/cache.h index f2943bdd7f9c..1516f03ee729 100644 --- a/include/zephyr/arch/cache.h +++ b/include/zephyr/arch/cache.h @@ -335,6 +335,20 @@ size_t arch_icache_line_size_get(void); #endif /* CONFIG_ICACHE || __DOXYGEN__ */ +#if CONFIG_CACHE_DOUBLEMAP || __DOXYGEN__ +bool arch_cache_is_ptr_cached(void *ptr); +#define cache_is_ptr_cached(ptr) arch_cache_is_ptr_cached(ptr) + +bool arch_cache_is_ptr_uncached(void *ptr); +#define cache_is_ptr_uncached(ptr) arch_cache_is_ptr_uncached(ptr) + +void __sparse_cache *arch_cache_cached_ptr_get(void *ptr); +#define cache_cached_ptr(ptr) arch_cache_cached_ptr_get(ptr) + +void *arch_cache_uncached_ptr_get(void __sparse_cache *ptr); +#define cache_uncached_ptr(ptr) arch_cache_uncached_ptr_get(ptr) +#endif /* CONFIG_CACHE_DOUBLEMAP */ + /** * @} */ diff --git a/include/zephyr/cache.h b/include/zephyr/cache.h index 994d96dccbe7..f5547f64afde 100644 --- a/include/zephyr/cache.h +++ b/include/zephyr/cache.h @@ -15,6 +15,7 @@ #include #include +#include #ifdef __cplusplus extern "C" { @@ -441,6 +442,107 @@ static ALWAYS_INLINE size_t sys_cache_instr_line_size_get(void) #endif } +/** + * @brief Test if a pointer is in cached region. + * + * Some hardware may map the same physical memory twice + * so that it can be seen in both (incoherent) cached mappings + * and a coherent "shared" area. This tests if a particular + * pointer is within the cached, coherent area. + * + * @param ptr Pointer + * + * @retval True if pointer is in cached region. + * @retval False if pointer is not in cached region. + */ +static ALWAYS_INLINE bool sys_cache_is_ptr_cached(void *ptr) +{ +#if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP) + return cache_is_ptr_cached(ptr); +#else + ARG_UNUSED(ptr); + + return false; +#endif +} + +/** + * @brief Test if a pointer is in un-cached region. + * + * Some hardware may map the same physical memory twice + * so that it can be seen in both (incoherent) cached mappings + * and a coherent "shared" area. This tests if a particular + * pointer is within the un-cached, incoherent area. + * + * @param ptr Pointer + * + * @retval True if pointer is not in cached region. + * @retval False if pointer is in cached region. + */ +static ALWAYS_INLINE bool sys_cache_is_ptr_uncached(void *ptr) +{ +#if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP) + return cache_is_ptr_uncached(ptr); +#else + ARG_UNUSED(ptr); + + return false; +#endif +} + +/** + * @brief Return cached pointer to a RAM address + * + * This function takes a pointer to any addressable object (either in + * cacheable memory or not) and returns a pointer that can be used to + * refer to the same memory through the L1 data cache. Data read + * through the resulting pointer will reflect locally cached values on + * the current CPU if they exist, and writes will go first into the + * cache and be written back later. + * + * @note This API returns the same pointer if CONFIG_CACHE_DOUBLEMAP is not + * enabled. + * + * @see arch_uncached_ptr() + * + * @param ptr A pointer to a valid C object + * @return A pointer to the same object via the L1 dcache + */ +static ALWAYS_INLINE void __sparse_cache *sys_cache_cached_ptr_get(void *ptr) +{ +#if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP) + return cache_cached_ptr(ptr); +#else + return (__sparse_force void __sparse_cache *)ptr; +#endif +} + +/** + * @brief Return uncached pointer to a RAM address + * + * This function takes a pointer to any addressable object (either in + * cacheable memory or not) and returns a pointer that can be used to + * refer to the same memory while bypassing the L1 data cache. Data + * in the L1 cache will not be inspected nor modified by the access. + * + * @note This API returns the same pointer if CONFIG_CACHE_DOUBLEMAP is not + * enabled. + * + * @see arch_cached_ptr() + * + * @param ptr A pointer to a valid C object + * @return A pointer to the same object bypassing the L1 dcache + */ +static ALWAYS_INLINE void *sys_cache_uncached_ptr_get(void __sparse_cache *ptr) +{ +#if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP) + return cache_uncached_ptr(ptr); +#else + return (__sparse_force void *)ptr; +#endif +} + + #ifdef CONFIG_LIBMETAL static ALWAYS_INLINE void sys_cache_flush(void *addr, size_t size) {