cache: introduce incoherent cache interface

Introduce a set of cache APIs used on architectures with cache incoherency.

Signed-off-by: Anas Nashif <[email protected]>

arch/Kconfig

@@ -730,6 +730,13 @@ config CPU_HAS_DCACHE
 	help
 	  This hidden configuration should be selected when the CPU has a d-cache.
 
+config CPU_CACHE_INCOHERENT
+	bool
+	help
+	  This hidden configuration should be selected when the CPU has
+	  incoherent cache. This applies to intra-CPU multiprocessing
+	  incoherence and makes only sense when MP_NUM_CPUS > 1.
+
 config CPU_HAS_ICACHE
 	bool
 	help
@@ -925,6 +932,17 @@ config ICACHE
 	help
 	  This option enables the support for the instruction cache (i-cache).
 
+config CACHE_DOUBLEMAP
+	bool "Cache double-mapping support"
+	depends on CPU_CACHE_INCOHERENT
+	default y
+	help
+	  Double-mapping behavior where a pointer can be cheaply converted to
+	  point to the same cached/uncached memory at different locations.
+
+	  This applies to intra-CPU multiprocessing incoherence and makes only
+	  sense when MP_NUM_CPUS > 1.
+
 config CACHE_MANAGEMENT
 	bool "Cache management features"
 	depends on DCACHE || ICACHE

doc/zephyr.doxyfile.in

@@ -2399,6 +2399,7 @@ PREDEFINED             = __DOXYGEN__ \
 			 XEN_GUEST_HANDLE_64(x)= \
                          _LINKER \
                          __deprecated= \
+			 __sparse_cache= \
                          __packed= \
                          __aligned(x)= \
                          __attribute_nonnull(...)= \

include/zephyr/arch/cache.h

@@ -335,6 +335,20 @@ size_t arch_icache_line_size_get(void);
 
 #endif /* CONFIG_ICACHE || __DOXYGEN__ */
 
+#if CONFIG_CACHE_DOUBLEMAP  || __DOXYGEN__
+bool arch_cache_is_ptr_cached(void *ptr);
+#define cache_is_ptr_cached(ptr) arch_cache_is_ptr_cached(ptr)
+
+bool arch_cache_is_ptr_uncached(void *ptr);
+#define cache_is_ptr_uncached(ptr) arch_cache_is_ptr_uncached(ptr)
+
+void __sparse_cache *arch_cache_cached_ptr_get(void *ptr);
+#define cache_cached_ptr(ptr) arch_cache_cached_ptr_get(ptr)
+
+void *arch_cache_uncached_ptr_get(void __sparse_cache *ptr);
+#define cache_uncached_ptr(ptr) arch_cache_uncached_ptr_get(ptr)
+#endif /* CONFIG_CACHE_DOUBLEMAP */
+
 /**
  * @}
  */

include/zephyr/cache.h

@@ -15,6 +15,7 @@
 
 #include <zephyr/kernel.h>
 #include <zephyr/arch/cpu.h>
+#include <zephyr/debug/sparse.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -441,6 +442,107 @@ static ALWAYS_INLINE size_t sys_cache_instr_line_size_get(void)
 #endif
 }
 
+/**
+ * @brief Test if a pointer is in cached region.
+ *
+ * Some hardware may map the same physical memory twice
+ * so that it can be seen in both (incoherent) cached mappings
+ * and a coherent "shared" area. This tests if a particular
+ * pointer is within the cached, coherent area.
+ *
+ * @param ptr Pointer
+ *
+ * @retval True if pointer is in cached region.
+ * @retval False if pointer is not in cached region.
+ */
+static ALWAYS_INLINE bool sys_cache_is_ptr_cached(void *ptr)
+{
+#if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP)
+	return cache_is_ptr_cached(ptr);
+#else
+	ARG_UNUSED(ptr);
+
+	return false;
+#endif
+}
+
+/**
+ * @brief Test if a pointer is in un-cached region.
+ *
+ * Some hardware may map the same physical memory twice
+ * so that it can be seen in both (incoherent) cached mappings
+ * and a coherent "shared" area. This tests if a particular
+ * pointer is within the un-cached, incoherent area.
+ *
+ * @param ptr Pointer
+ *
+ * @retval True if pointer is not in cached region.
+ * @retval False if pointer is in cached region.
+ */
+static ALWAYS_INLINE bool sys_cache_is_ptr_uncached(void *ptr)
+{
+#if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP)
+	return cache_is_ptr_uncached(ptr);
+#else
+	ARG_UNUSED(ptr);
+
+	return false;
+#endif
+}
+
+/**
+ * @brief Return cached pointer to a RAM address
+ *
+ * This function takes a pointer to any addressable object (either in
+ * cacheable memory or not) and returns a pointer that can be used to
+ * refer to the same memory through the L1 data cache.  Data read
+ * through the resulting pointer will reflect locally cached values on
+ * the current CPU if they exist, and writes will go first into the
+ * cache and be written back later.
+ *
+ * @note This API returns the same pointer if CONFIG_CACHE_DOUBLEMAP is not
+ * enabled.
+ *
+ * @see arch_uncached_ptr()
+ *
+ * @param ptr A pointer to a valid C object
+ * @return A pointer to the same object via the L1 dcache
+ */
+static ALWAYS_INLINE void __sparse_cache *sys_cache_cached_ptr_get(void *ptr)
+{
+#if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP)
+	return cache_cached_ptr(ptr);
+#else
+	return (__sparse_force void __sparse_cache *)ptr;
+#endif
+}
+
+/**
+ * @brief Return uncached pointer to a RAM address
+ *
+ * This function takes a pointer to any addressable object (either in
+ * cacheable memory or not) and returns a pointer that can be used to
+ * refer to the same memory while bypassing the L1 data cache.  Data
+ * in the L1 cache will not be inspected nor modified by the access.
+ *
+ * @note This API returns the same pointer if CONFIG_CACHE_DOUBLEMAP is not
+ * enabled.
+ *
+ * @see arch_cached_ptr()
+ *
+ * @param ptr A pointer to a valid C object
+ * @return A pointer to the same object bypassing the L1 dcache
+ */
+static ALWAYS_INLINE void *sys_cache_uncached_ptr_get(void __sparse_cache *ptr)
+{
+#if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP)
+	return cache_uncached_ptr(ptr);
+#else
+	return (__sparse_force void *)ptr;
+#endif
+}
+
+
 #ifdef CONFIG_LIBMETAL
 static ALWAYS_INLINE void sys_cache_flush(void *addr, size_t size)
 {