diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index af33bfd06ef0..7c08d1f32f25 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -192,9 +192,14 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) sof_heap_free(sof_sys_user_heap_get(), buffer->stream.addr); if (alloc && alloc->vreg) { - vregion_free(alloc->vreg, buffer); - if (!vregion_put(alloc->vreg)) - sof_heap_free(alloc->heap, alloc); + struct vregion *vreg = alloc->vreg; + + /* + * The alloc ctx of a DP module is in its vregion, which the + * last vregion_put() frees, together with the ctx. + */ + vregion_free(vreg, buffer); + vregion_put(vreg); } else { sof_heap_free(alloc ? alloc->heap : NULL, buffer); } diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 42a8356f0e68..2fa971fa063d 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -40,6 +40,62 @@ LOG_MODULE_DECLARE(module_adapter, CONFIG_SOF_LOG_LEVEL); */ static K_MUTEX_DEFINE(mod_res_lock); +/* + * Configuration buffers of DP modules are in the module heap: DP modules + * set their configuration in the DP thread, which may not use the LL user + * heap. Until the pipeline is complete, the vregion of a DP module only has + * its lifetime allocator, which does not reclaim freed memory, so + * module_set_configuration() keeps the staging buffer of a DP module for the + * next request. Other modules use the LL user heap, which reclaims freed + * memory at any time. + */ +static bool module_cfg_in_mod_heap(const struct processing_module *mod) +{ + return mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP; +} + +static void *module_cfg_alloc(struct processing_module *mod, size_t size) +{ + if (module_cfg_in_mod_heap(mod)) + return mod_alloc_ext(mod, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, size, 0); + + return sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, size, 0); +} + +static void module_cfg_free(struct processing_module *mod, void *ptr) +{ + if (!ptr) + return; + + if (module_cfg_in_mod_heap(mod)) + mod_free(mod, ptr); + else + sof_heap_free(sof_sys_user_heap_get(), ptr); +} + +/* Get a staging buffer for a configuration of @p size bytes */ +static void *module_cfg_staging_get(struct processing_module *mod, size_t size) +{ + if (!module_cfg_in_mod_heap(mod)) + return module_cfg_alloc(mod, size); + + if (mod->cfg_staging_size < size) { + module_cfg_free(mod, mod->cfg_staging); + mod->cfg_staging = module_cfg_alloc(mod, size); + mod->cfg_staging_size = mod->cfg_staging ? size : 0; + } + + return mod->cfg_staging; +} + +/* Release a staging buffer after a request, DP modules keep it */ +static void module_cfg_staging_put(struct processing_module *mod, void *buf) +{ + if (!module_cfg_in_mod_heap(mod)) + module_cfg_free(mod, buf); +} + __cold int module_load_config(struct comp_dev *dev, const void *cfg, size_t size) { int ret; @@ -62,15 +118,13 @@ __cold int module_load_config(struct comp_dev *dev, const void *cfg, size_t size if (!dst->data) { /* No space for config available yet, allocate now */ - dst->data = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, size, 0); + dst->data = module_cfg_alloc(mod, size); } else if (dst->size != size) { /* The size allocated for previous config doesn't match the new one. * Free old container and allocate new one. */ - sof_heap_free(sof_sys_user_heap_get(), dst->data); - dst->data = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, size, 0); + module_cfg_free(mod, dst->data); + dst->data = module_cfg_alloc(mod, size); } if (!dst->data) { comp_err(dev, "failed to allocate space for setup config."); @@ -671,7 +725,7 @@ int module_prepare(struct processing_module *mod, * as it has been applied during the procedure - it is safe to * free it. */ - sof_heap_free(sof_sys_user_heap_get(), md->cfg.data); + module_cfg_free(mod, md->cfg.data); md->cfg.avail = false; md->cfg.data = NULL; @@ -761,7 +815,7 @@ int module_reset(struct processing_module *mod) md->cfg.avail = false; md->cfg.size = 0; - sof_heap_free(sof_sys_user_heap_get(), md->cfg.data); + module_cfg_free(mod, md->cfg.data); md->cfg.data = NULL; #if CONFIG_IPC_MAJOR_3 @@ -829,12 +883,15 @@ __cold int module_free(struct processing_module *mod) /* Free all memory shared by module_adapter & module */ md->cfg.avail = false; md->cfg.size = 0; - sof_heap_free(sof_sys_user_heap_get(), md->cfg.data); + module_cfg_free(mod, md->cfg.data); md->cfg.data = NULL; if (md->runtime_params) { - sof_heap_free(sof_sys_user_heap_get(), md->runtime_params); + module_cfg_staging_put(mod, md->runtime_params); md->runtime_params = NULL; } + module_cfg_free(mod, mod->cfg_staging); + mod->cfg_staging = NULL; + mod->cfg_staging_size = 0; #if CONFIG_IPC_MAJOR_3 md->state = MODULE_DISABLED; #endif @@ -900,9 +957,7 @@ __cold int module_set_configuration(struct processing_module *mod, } /* Allocate buffer for new params */ - md->runtime_params = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, - data_offset_size, 0); + md->runtime_params = module_cfg_staging_get(mod, data_offset_size); if (!md->runtime_params) { comp_err(dev, "space allocation for new params failed"); return -ENOMEM; @@ -950,8 +1005,7 @@ __cold int module_set_configuration(struct processing_module *mod, md->new_cfg_size = 0; - if (md->runtime_params) - sof_heap_free(sof_sys_user_heap_get(), md->runtime_params); + module_cfg_staging_put(mod, md->runtime_params); md->runtime_params = NULL; return ret; diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index b0075453e434..1257d15978b9 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -142,7 +142,17 @@ struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv if (use_ppl_alloc) { alloc = ppl_alloc; } else { - alloc = sof_heap_alloc(mod_heap, flags, sizeof(*alloc), 0); + /* + * A DP module keeps its allocation context in its own vregion, + * so it is released with the vregion, and the kernel, which + * creates DP modules, does not allocate it from the LL user heap. + */ + if (!mod_vreg) + alloc = sof_heap_alloc(mod_heap, flags, sizeof(*alloc), 0); + else if (flags & SOF_MEM_FLAG_COHERENT) + alloc = vregion_alloc_coherent(mod_vreg, sizeof(*alloc)); + else + alloc = vregion_alloc(mod_vreg, sizeof(*alloc)); if (!alloc) goto ealloc; @@ -179,7 +189,7 @@ struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv return mod; edev: - if (!use_ppl_alloc) + if (!use_ppl_alloc && !mod_vreg) sof_heap_free(mod_heap, alloc); ealloc: if (mod_vreg) @@ -215,13 +225,12 @@ __cold static void module_adapter_mem_free(struct processing_module *mod) vregion_put(alloc->vreg); } else if (alloc->vreg) { /* - * This is DP userpsace case - * Only remove the alloc ctx, if vreg was freed. If it was not - * the DP userspace thread is still holding a reference to it, - * and will free alloc ctx eventually. + * This is the DP userspace case. The alloc ctx is in the + * vregion and is freed with it when the last reference, which + * can be held by a buffer of the module, is put. Do not access + * it after this. */ - if (!vregion_put(alloc->vreg)) - sof_heap_free(alloc->heap, alloc); + vregion_put(alloc->vreg); } else { sof_heap_free(alloc->heap, alloc); } diff --git a/src/include/module/module/base.h b/src/include/module/module/base.h index c311625428e4..e3b567fc04d3 100644 --- a/src/include/module/module/base.h +++ b/src/include/module/module/base.h @@ -176,6 +176,12 @@ struct processing_module { * the offsets of the fields above. */ uint32_t runtime_params_size; + + /* configuration staging buffer of a DP module, kept for the next + * request, see module_set_configuration() + */ + void *cfg_staging; + size_t cfg_staging_size; #endif /* SOF_MODULE_PRIVATE */ }; diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index dab67d29d1de..50500ed9d6fe 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -114,6 +114,11 @@ struct ipc_user { struct k_thread *audio_thread[IPC_DSP_CORE_COUNT]; /** @brief Original kernel driver pointer for restoring dev->drv after create */ const struct comp_driver *init_drv; + /** + * @brief Component created by the kernel, for the user thread to add + * to the component list (DP modules), see ipc4_init_module_instance() + */ + struct comp_dev *init_dev; /** * @brief User-accessible copy of comp_driver + tr_ctx for create(). * diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index 84330c55f378..cee658fc1da3 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -196,6 +196,11 @@ Objects in the kernel areas: (`struct zephyr_ll`), the LL scheduling domain and its Zephyr domain data, the DP scheduler data, and the LL tasks that kernel threads create at boot together with their task data. +DP modules are created by kernel code, but their allocation context, pin format arrays and +configuration buffers are in the module's own vregion, released with it, and the IPC user +thread adds the component to the component list. So creating a DP module does not allocate +from the LL user heap in kernel code. + The LL timer interrupt and the LL thread setup and teardown use kernel-only copies of the domain data pointer, the per-core semaphores and the LL threads, not the user-writable domain data. diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 232e81800201..b66bbe57613f 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1104,8 +1104,17 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) * init_drv = original kernel pointer */ const struct comp_driver *drv = ipc_user->init_drv; + struct comp_dev *kdev = ipc_user->init_dev; ipc_user->init_drv = NULL; + ipc_user->init_dev = NULL; + + /* created by the kernel (DP module), only add it here */ + if (kdev) { + result = ipc4_add_comp_dev(kdev); + break; + } + if (!drv) { result = IPC4_MOD_NOT_INITIALIZED; break; diff --git a/src/kernel/ipc/ipc4/handler-user-kernel.c b/src/kernel/ipc/ipc4/handler-user-kernel.c index ec4d49736fee..9701c2cb89f2 100644 --- a/src/kernel/ipc/ipc4/handler-user-kernel.c +++ b/src/kernel/ipc/ipc4/handler-user-kernel.c @@ -269,6 +269,34 @@ __cold int ipc4_init_module_instance(struct ipc4_message_request *ipc4) return IPC4_MOD_NOT_INITIALIZED; } +#ifdef CONFIG_SOF_USERSPACE_LL + /* + * The component container is allocated from the LL user heap, which + * the kernel does not allocate from: let the IPC user thread of this + * core add the component to the component list. + */ + int ret; + + ipc_user_get()->init_dev = dev; + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, cpu_get_id()); + if (ret) { + ipc_cmd_err(&ipc_tr, "error: failed to add module %x : %x: %d", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id, ret); + if (ret == -EAGAIN) { + /* + * The IPC user thread did not finish in time and may + * still add the component, so it cannot be freed here. + * Withdraw it in case the thread has not taken it yet. + */ + ipc_user_get()->init_dev = NULL; + return IPC4_MOD_NOT_INITIALIZED; + } + comp_free(dev); + return IPC4_MOD_NOT_INITIALIZED; + } +#endif + return 0; } diff --git a/src/kernel/ipc/ipc4/helper-kernel.c b/src/kernel/ipc/ipc4/helper-kernel.c index 2e8f5ee40474..6f0402bd573a 100644 --- a/src/kernel/ipc/ipc4/helper-kernel.c +++ b/src/kernel/ipc/ipc4/helper-kernel.c @@ -145,7 +145,14 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo } #endif +#ifndef CONFIG_SOF_USERSPACE_LL + /* + * With CONFIG_SOF_USERSPACE_LL, the component list and its containers + * are user memory: ipc4_init_module_instance() has the IPC user thread + * add the component. + */ ipc4_add_comp_dev(dev); +#endif comp_update_ibs_obs_cpc(dev);