1st part of llext #8180

src/include/sof/lib_manager.h

@@ -78,9 +78,14 @@ struct ipc_lib_msg {
 	struct list_item list;
 };
 
+struct lib_manager_mod_ctx {
+	struct sof_man_fw_desc *desc;
+	size_t segment_size[3];
+};
+
 struct ext_library {
 	struct k_spinlock lock;	/* last locking CPU record */
-	struct sof_man_fw_desc *desc[LIB_MANAGER_MAX_LIBS];
+	struct lib_manager_mod_ctx *desc[LIB_MANAGER_MAX_LIBS];
 #ifdef CONFIG_LIBCODE_MODULE_SUPPORT
 	uint32_t mods_exec_load_cnt;
 #endif
@@ -97,6 +102,14 @@ static inline struct ext_library *ext_lib_get(void)
 {
 	return sof_get()->ext_library;
 }
+
+static inline struct lib_manager_mod_ctx *lib_manager_get_mod_ctx(int module_id)
+{
+	uint32_t lib_id = LIB_MANAGER_GET_LIB_ID(module_id);
+	struct ext_library *_ext_lib = ext_lib_get();
+
+	return _ext_lib->desc[lib_id];
+}
 #endif
 
 /*

src/include/sof/llext_manager.h

@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2024 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __SOF_LLEXT_MANAGER_H__
+#define __SOF_LLEXT_MANAGER_H__
+
+#include <stdint.h>
+
+struct comp_driver;
+struct comp_ipc_config;
+
+uint32_t llext_manager_allocate_module(const struct comp_driver *drv,
+				       struct comp_ipc_config *ipc_config,
+				       const void *ipc_specific_config);
+int llext_manager_free_module(const struct comp_driver *drv,
+			      struct comp_ipc_config *ipc_config);
+
+#endif

src/library_manager/lib_manager.c

@@ -19,6 +19,7 @@
 #include <rtos/spinlock.h>
 #include <sof/lib/cpu-clk-manager.h>
 #include <sof/lib_manager.h>
+#include <sof/llext_manager.h>
 #include <sof/audio/module_adapter/module/generic.h>
 #include <sof/audio/module_adapter/module/modules.h>
 
@@ -76,9 +77,8 @@ static int lib_manager_load_data_from_storage(void __sparse_cache *vma, void *s_
 static int lib_manager_load_module(const uint32_t module_id,
 				   const struct sof_man_module *const mod)
 {
-	const struct ext_library *const ext_lib = ext_lib_get();
-	const uint32_t lib_id = LIB_MANAGER_GET_LIB_ID(module_id);
-	const uintptr_t load_offset = POINTER_TO_UINT(ext_lib->desc[lib_id]);
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+	const uintptr_t load_offset = POINTER_TO_UINT(ctx->desc);
 	void *src;
 	void __sparse_cache *va_base;
 	size_t size;
@@ -265,6 +265,11 @@ static int lib_manager_free_module_instance(uint32_t module_id, uint32_t instanc
 	return sys_mm_drv_unmap_region((__sparse_force void *)va_base, bss_size);
 }
 
+static bool module_is_llext(struct sof_man_module *mod)
+{
+	return mod->type.load_type == SOF_MAN_MOD_TYPE_LLEXT;
+}
+
 uint32_t lib_manager_allocate_module(const struct comp_driver *drv,
 				     struct comp_ipc_config *ipc_config,
 				     const void *ipc_specific_config)
@@ -288,6 +293,9 @@ uint32_t lib_manager_allocate_module(const struct comp_driver *drv,
 
 	mod = (struct sof_man_module *)((char *)desc + SOF_MAN_MODULE_OFFSET(entry_index));
 
+	if (module_is_llext(mod))
+		return llext_manager_allocate_module(drv, ipc_config, ipc_specific_config);
+
 	ret = lib_manager_load_module(module_id, mod);
 	if (ret < 0)
 		return 0;
@@ -329,6 +337,9 @@ int lib_manager_free_module(const struct comp_driver *drv,
 	desc = lib_manager_get_library_module_desc(module_id);
 	mod = (struct sof_man_module *)((char *)desc + SOF_MAN_MODULE_OFFSET(entry_index));
 
+	if (module_is_llext(mod))
+		return llext_manager_free_module(drv, ipc_config);
+
 	ret = lib_manager_unload_module(mod);
 	if (ret < 0)
 		return ret;
@@ -381,9 +392,8 @@ void lib_manager_init(void)
 
 struct sof_man_fw_desc *lib_manager_get_library_module_desc(int module_id)
 {
-	uint32_t lib_id = LIB_MANAGER_GET_LIB_ID(module_id);
-	struct ext_library *_ext_lib = ext_lib_get();
-	uint8_t *buffptr = (uint8_t *)_ext_lib->desc[lib_id];
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+	uint8_t *buffptr = (uint8_t *)(ctx ? ctx->desc : NULL);
 
 	if (!buffptr)
 		return NULL;
@@ -393,9 +403,14 @@ struct sof_man_fw_desc *lib_manager_get_library_module_desc(int module_id)
 static void lib_manager_update_sof_ctx(struct sof_man_fw_desc *desc, uint32_t lib_id)
 {
 	struct ext_library *_ext_lib = ext_lib_get();
+	/* Never freed, will panic if fails */
+	struct lib_manager_mod_ctx *ctx = rmalloc(SOF_MEM_ZONE_SYS, 0, SOF_MEM_CAPS_RAM,
+						  sizeof(*ctx));
 
-	_ext_lib->desc[lib_id] = desc;
-	/* TODO: maybe need to call here dcache_writeback here? */
+	ctx->desc = desc;
+
+	_ext_lib->desc[lib_id] = ctx;
+	/* TODO: maybe need to call dcache_writeback here? */
 }
 
 #if CONFIG_INTEL_MODULES
@@ -599,9 +614,6 @@ static void __sparse_cache *lib_manager_allocate_store_mem(uint32_t size,
 		return NULL;
 	}
 
-	dcache_invalidate_region(local_add, size);
-	icache_invalidate_region(local_add, size);
-
 	return local_add;
 }
 

src/library_manager/llext_manager.c

@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// Copyright(c) 2022 Intel Corporation. All rights reserved.
+//
+// Author: Jaroslaw Stelter <[email protected]>
+//         Pawel Dobrowolski<[email protected]>
+
+/*
+ * Dynamic module loading functions.
+ */
+
+#include <sof/audio/buffer.h>
+#include <sof/audio/component_ext.h>
+#include <sof/common.h>
+#include <sof/compiler_attributes.h>
+#include <sof/ipc/topology.h>
+
+#include <rtos/sof.h>
+#include <rtos/spinlock.h>
+#include <sof/lib/cpu-clk-manager.h>
+#include <sof/lib_manager.h>
+#include <sof/llext_manager.h>
+#include <sof/audio/module_adapter/module/generic.h>
+#include <sof/audio/module_adapter/module/modules.h>
+
+#include <zephyr/cache.h>
+#include <zephyr/drivers/mm/system_mm.h>
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+LOG_MODULE_DECLARE(lib_manager, CONFIG_SOF_LOG_LEVEL);
+
+extern struct tr_ctx lib_manager_tr;
+
+#define PAGE_SZ		CONFIG_MM_DRV_PAGE_SIZE
+
+static int llext_manager_align_map(void __sparse_cache *vma, size_t size, uint32_t flags)
+{
+	size_t pre_pad_size = (uintptr_t)vma & (PAGE_SZ - 1);
+	void *aligned_vma = (__sparse_force uint8_t *)vma - pre_pad_size;
+
+	return sys_mm_drv_map_region(aligned_vma, POINTER_TO_UINT(NULL),
+				     ALIGN_UP(pre_pad_size + size, PAGE_SZ), flags);
+}
+
+static int llext_manager_align_unmap(void __sparse_cache *vma, size_t size)
+{
+	size_t pre_pad_size = (uintptr_t)vma & (PAGE_SZ - 1);
+	void *aligned_vma = (__sparse_force uint8_t *)vma - pre_pad_size;
+
+	return sys_mm_drv_unmap_region(aligned_vma, ALIGN_UP(pre_pad_size + size, PAGE_SZ));
+}
+
+static int llext_manager_load_data_from_storage(void __sparse_cache *vma, void *s_addr,
+						size_t size, uint32_t flags)
+{
+	int ret = llext_manager_align_map(vma, size, flags);
+
+	if (ret < 0) {
+		tr_err(&lib_manager_tr, "cannot map %u of %p", size, (__sparse_force void *)vma);
+		return ret;
+	}
+
+	/* TODO: Change attributes for memory to FLAGS  */
+	return memcpy_s((__sparse_force void *)vma, size, s_addr, size);
+}
+
+static int llext_manager_load_module(uint32_t module_id, struct sof_man_module *mod,
+				     struct sof_man_fw_desc *desc)
+{
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+	uint8_t *load_base = (uint8_t *)ctx->desc;
+	void __sparse_cache *va_base_text = (void __sparse_cache *)
+		mod->segment[SOF_MAN_SEGMENT_TEXT].v_base_addr;
+	void *src_txt = (void *)(load_base + mod->segment[SOF_MAN_SEGMENT_TEXT].file_offset);
+	size_t st_text_size = ctx->segment_size[SOF_MAN_SEGMENT_TEXT];
+	void __sparse_cache *va_base_rodata = (void __sparse_cache *)
+		mod->segment[SOF_MAN_SEGMENT_RODATA].v_base_addr;
+	void *src_rodata = (void *)(load_base +
+				    mod->segment[SOF_MAN_SEGMENT_RODATA].file_offset);
+	size_t st_rodata_size = ctx->segment_size[SOF_MAN_SEGMENT_RODATA];
+	int ret;
+
+	/* Copy Code */
+	ret = llext_manager_load_data_from_storage(va_base_text, src_txt, st_text_size,
+						   SYS_MM_MEM_PERM_RW | SYS_MM_MEM_PERM_EXEC);
+	if (ret < 0)
+		return ret;
+
+	/* .text contains instructions and it also often contains local data */
+	dcache_writeback_region(va_base_text, st_text_size);
+	icache_invalidate_region(va_base_text, st_text_size);
+
+	/* Copy RODATA */
+	ret = llext_manager_load_data_from_storage(va_base_rodata, src_rodata,
+						   st_rodata_size, SYS_MM_MEM_PERM_RW);
+	if (ret < 0)
+		goto e_text;
+
+	/* Some data can be accessed as uncached, in fact that's the default */
+	dcache_writeback_region(va_base_rodata, st_rodata_size);
+
+	return 0;
+
+e_text:
+	llext_manager_align_unmap(va_base_text, st_text_size);
+
+	return ret;
+}
+
+static int llext_manager_unload_module(uint32_t module_id, struct sof_man_module *mod,
+				       struct sof_man_fw_desc *desc)
+{
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+	void __sparse_cache *va_base_text = (void __sparse_cache *)
+		mod->segment[SOF_MAN_SEGMENT_TEXT].v_base_addr;
+	size_t st_text_size = ctx->segment_size[SOF_MAN_SEGMENT_TEXT];
+	void __sparse_cache *va_base_rodata = (void __sparse_cache *)
+		mod->segment[SOF_MAN_SEGMENT_RODATA].v_base_addr;
+	size_t st_rodata_size = ctx->segment_size[SOF_MAN_SEGMENT_RODATA];
+	int ret;
+
+	ret = llext_manager_align_unmap(va_base_text, st_text_size);
+	if (ret < 0)
+		return ret;
+
+	return llext_manager_align_unmap(va_base_rodata, st_rodata_size);
+}
+
+static void __sparse_cache *llext_manager_get_bss_address(uint32_t module_id,
+							  struct sof_man_module *mod)
+{
+	return (void __sparse_cache *)mod->segment[SOF_MAN_SEGMENT_BSS].v_base_addr;
+}
+
+static int llext_manager_allocate_module_bss(uint32_t module_id,
+					     uint32_t is_pages, struct sof_man_module *mod)
+{
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+	size_t bss_size = ctx->segment_size[SOF_MAN_SEGMENT_BSS];
+	void __sparse_cache *va_base = llext_manager_get_bss_address(module_id, mod);
+
+	if (is_pages * PAGE_SZ > bss_size) {
+		tr_err(&lib_manager_tr,
+		       "llext_manager_allocate_module_bss(): invalid is_pages: %u, required: %u",
+		       is_pages, bss_size / PAGE_SZ);
+		return -ENOMEM;
+	}
+
+	/* Map bss memory and clear it. */
+	if (llext_manager_align_map(va_base, bss_size, SYS_MM_MEM_PERM_RW) < 0)
+		return -ENOMEM;
+
+	memset((__sparse_force void *)va_base, 0, bss_size);
+
+	return 0;
+}
+
+static int llext_manager_free_module_bss(uint32_t module_id,
+					 struct sof_man_module *mod)
+{
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+	size_t bss_size = ctx->segment_size[SOF_MAN_SEGMENT_BSS];
+	void __sparse_cache *va_base = llext_manager_get_bss_address(module_id, mod);
+
+	/* Unmap bss memory. */
+	return llext_manager_align_unmap(va_base, bss_size);
+}
+
+uint32_t llext_manager_allocate_module(const struct comp_driver *drv,
+				       struct comp_ipc_config *ipc_config,
+				       const void *ipc_specific_config)
+{
+	struct sof_man_fw_desc *desc;
+	struct sof_man_module *mod;
+	const struct ipc4_base_module_cfg *base_cfg = ipc_specific_config;
+	int ret;
+	uint32_t module_id = IPC4_MOD_ID(ipc_config->id);
+	uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id);
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+
+	tr_dbg(&lib_manager_tr, "llext_manager_allocate_module(): mod_id: %#x",
+	       ipc_config->id);
+
+	desc = lib_manager_get_library_module_desc(module_id);
+	if (!ctx || !desc) {
+		tr_err(&lib_manager_tr,
+		       "llext_manager_allocate_module(): failed to get module descriptor");
+		return 0;
+	}
+
+	mod = (struct sof_man_module *)((char *)desc + SOF_MAN_MODULE_OFFSET(entry_index));
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(ctx->segment_size); i++)
+		ctx->segment_size[i] = mod->segment[i].flags.r.length * PAGE_SZ;
+
+	ret = llext_manager_load_module(module_id, mod, desc);
+	if (ret < 0)
+		return 0;
+
+	ret = llext_manager_allocate_module_bss(module_id, base_cfg->is_pages, mod);
+	if (ret < 0) {
+		tr_err(&lib_manager_tr,
+		       "llext_manager_allocate_module(): module allocation failed: %d", ret);
+		return 0;
+	}
+	return mod->entry_point;
+}
+
+int llext_manager_free_module(const struct comp_driver *drv,
+			      struct comp_ipc_config *ipc_config)
+{
+	struct sof_man_fw_desc *desc;
+	struct sof_man_module *mod;
+	uint32_t module_id = IPC4_MOD_ID(ipc_config->id);
+	uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id);
+	int ret;
+
+	tr_dbg(&lib_manager_tr, "llext_manager_free_module(): mod_id: %#x", ipc_config->id);
+
+	desc = lib_manager_get_library_module_desc(module_id);
+	mod = (struct sof_man_module *)((char *)desc + SOF_MAN_MODULE_OFFSET(entry_index));
+
+	ret = llext_manager_unload_module(module_id, mod, desc);
+	if (ret < 0)
+		return ret;
+
+	ret = llext_manager_free_module_bss(module_id, mod);
+	if (ret < 0) {
+		tr_err(&lib_manager_tr,
+		       "llext_manager_free_module(): free module bss failed: %d", ret);
+		return ret;
+	}
+	return 0;
+}