From 9ec6da1f177e3e41b0302389a8a050a4934e7778 Mon Sep 17 00:00:00 2001 From: "Ganesh, Aditya" Date: Wed, 28 Oct 2026 10:21:23 -0700 Subject: [PATCH 1/7] Debug logs added Signed-off-by: Ganesh, Aditya --- src/audio/module_adapter/module/generic.c | 4 ++++ src/audio/module_adapter/module_adapter.c | 4 ++++ src/audio/volume/volume_ipc4.c | 24 +++++++++++++++++++++-- src/ipc/ipc-common.c | 5 +++-- src/ipc/ipc4/handler-user.c | 17 ++++++++++++++-- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 137767480e1d..7a1c6ce10fe5 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -133,7 +133,11 @@ int module_init(struct processing_module *mod) ret = scheduler_dp_thread_ipc(mod, SOF_IPC4_MOD_INIT_INSTANCE, ¶m); } else #endif + { + comp_info(dev, "module_init: calling module init callback"); ret = interface->init(mod); + comp_info(dev, "module_init: module init callback returned %d", ret); + } if (ret) { comp_err(dev, "error %d: module specific init failed", ret); diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 3ccae18cbe09..422fa0f70796 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -360,6 +360,10 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, goto err; } + comp_info(dev, "module adapter: proc_domain=%u period=%u", + (unsigned int)dev->ipc_config.proc_domain, + (unsigned int)dev->period); + module_adapter_reset_data(dst); dev->state = COMP_STATE_READY; diff --git a/src/audio/volume/volume_ipc4.c b/src/audio/volume/volume_ipc4.c index e111752d2b78..82bb71c62eb6 100644 --- a/src/audio/volume/volume_ipc4.c +++ b/src/audio/volume/volume_ipc4.c @@ -118,6 +118,9 @@ int volume_init(struct processing_module *mod) bool all_channels; uint32_t instance_id = IPC4_INST_ID(dev_comp_id(dev)); + comp_info(dev, "volume_init: entry, instance %u payload size %zu", + instance_id, cfg->size); + if (instance_id >= IPC4_MAX_PEAK_VOL_REG_SLOTS) { comp_err(dev, "instance_id %u out of array bounds.", instance_id); return -EINVAL; @@ -148,24 +151,34 @@ int volume_init(struct processing_module *mod) return -EINVAL; } + comp_info(dev, "volume_init: config validated, channels %u all_channels %u", + channels_count, (unsigned int)all_channels); + + comp_info(dev, "volume_init: allocating vol_data, size %zu", sizeof(struct vol_data)); cd = mod_zalloc(mod, sizeof(struct vol_data)); - if (!cd) + if (!cd) { + comp_err(dev, "volume_init: vol_data allocation failed"); return -ENOMEM; + } + comp_info(dev, "volume_init: vol_data allocated %p", cd); /* * malloc memory to store current volume 4 times to ensure the address * is 8-byte aligned for multi-way xtensa intrinsic operations. */ + comp_info(dev, "volume_init: allocating volume buffer, size %zu", vol_size); cd->vol = mod_alloc_align(mod, vol_size, SOF_FRAME_BYTE_ALIGN); if (!cd->vol) { mod_free(mod, cd); comp_err(dev, "Failed to allocate %d", vol_size); return -ENOMEM; } + comp_info(dev, "volume_init: volume buffer allocated %p", cd->vol); /* malloc memory to store temp peak volume 4 times to ensure the address * is 8-byte aligned for multi-way xtensa intrinsic operations. */ + comp_info(dev, "volume_init: allocating peak buffer, size %zu", vol_size); cd->peak_vol = mod_zalloc(mod, vol_size); if (!cd->peak_vol) { mod_free(mod, cd->vol); @@ -173,9 +186,11 @@ int volume_init(struct processing_module *mod) comp_err(dev, "Failed to allocate %d for peak_vol", vol_size); return -ENOMEM; } + comp_info(dev, "volume_init: peak buffer allocated %p", cd->peak_vol); md->private = cd; + comp_info(dev, "volume_init: initializing channels"); for (channel = 0; channel < channels_count; channel++) { channel_cfg = all_channels ? 0 : channel; @@ -189,11 +204,15 @@ int volume_init(struct processing_module *mod) /* set muted as false*/ cd->muted[channel] = false; + comp_info(dev, "volume_init: channel %u initialized, target volume %#x", + (unsigned int)channel, target_volume[channel]); } + comp_info(dev, "volume_init: initializing ramp"); init_ramp(cd, vol->config[0].curve_duration, target_volume[0]); volume_set_ramp_channel_counter(cd, channels_count); + comp_info(dev, "volume_init: ramp initialized"); cd->mailbox_offset = offsetof(struct ipc4_fw_registers, peak_vol_regs); cd->mailbox_offset += instance_id * sizeof(struct ipc4_peak_volume_regs); @@ -201,8 +220,10 @@ int volume_init(struct processing_module *mod) cd->attenuation = 0; cd->is_passthrough = false; + comp_info(dev, "volume_init: mailbox offset %#x, resetting state", cd->mailbox_offset); volume_reset_state(cd); + comp_info(dev, "volume_init: completed"); return 0; } @@ -478,4 +499,3 @@ int volume_peak_prepare(struct vol_data *cd, struct processing_module *mod) return 0; } - diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 897e3055922b..1418e145b03b 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -440,7 +440,7 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core if (core >= CONFIG_CORE_COUNT) return -EINVAL; - LOG_DBG("IPC: forward cmd %08x", primary); + LOG_INF("IPC: forward cmd %08x", primary); /* Copy message words — original buffer may be reused */ pdata->ipc_msg_pri = primary; @@ -455,6 +455,7 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core /* Wake the user thread */ k_event_set(pdata->event[core], IPC_USER_EVENT_CMD); + LOG_INF("IPC: waiting for user thread to complete"); /* Wait for user thread to complete */ ret = k_sem_take(pdata->sem, K_MSEC(100)); if (ret) { @@ -505,7 +506,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP, false, K_FOREVER); - LOG_DBG("IPC user wake, mask %u", mask); + LOG_INF("IPC user wake, mask %u", mask); if (mask & IPC_USER_EVENT_CMD) { ipc_user->result = ipc_user_thread_dispatch(ipc_user); diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 3f604ad74664..d30ea714007e 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1603,7 +1603,7 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { int err = ipc4_user_module_load(drv, mi); - + tr_info(&ipc_tr, "User module loaded"); if (err < 0) { ret = IPC4_MOD_NOT_INITIALIZED; break; @@ -1754,6 +1754,8 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) msg.primary.dat = ipc_user->ipc_msg_pri; msg.extension.dat = ipc_user->ipc_msg_ext; + tr_info(&ipc_tr, "user dispatch: message primary type %#x", msg.primary.r.type); + ipc_user->reply_ext = 0; if (msg.primary.r.msg_tgt == SOF_IPC4_MESSAGE_TARGET_MODULE_MSG) { @@ -1795,26 +1797,37 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) */ const struct comp_driver *drv = ipc_user->init_drv; + tr_info(&ipc_tr, "INIT_INSTANCE: driver received %p", drv); ipc_user->init_drv = NULL; if (!drv) { + tr_err(&ipc_tr, "INIT_INSTANCE: driver missing"); result = IPC4_MOD_NOT_INITIALIZED; break; } + tr_info(&ipc_tr, "INIT_INSTANCE: driver validated, creating component"); struct comp_dev *dev = comp_new_ipc4_user(&msg, drv); if (!dev) { + tr_err(&ipc_tr, "INIT_INSTANCE: component creation failed"); result = IPC4_MOD_NOT_INITIALIZED; break; } + tr_info(&ipc_tr, "INIT_INSTANCE: component created %p, registering", dev); result = ipc4_add_comp_dev(dev); - if (result != IPC4_SUCCESS) + if (result != IPC4_SUCCESS) { + tr_err(&ipc_tr, "INIT_INSTANCE: component registration failed %d", + result); break; + } + tr_info(&ipc_tr, "INIT_INSTANCE: component registered, updating IBS/OBS/CPC"); comp_update_ibs_obs_cpc(dev); + tr_info(&ipc_tr, "INIT_INSTANCE: IBS/OBS/CPC update completed"); result = 0; + tr_info(&ipc_tr, "INIT_INSTANCE: completed"); break; } case SOF_IPC4_MOD_DELETE_INSTANCE: { From 1e8d83b951344bcd5bb0388645cbdd3bc65a12c5 Mon Sep 17 00:00:00 2001 From: "Ganesh, Aditya" Date: Thu, 29 Oct 2026 00:29:35 -0700 Subject: [PATCH 2/7] zephyr: Modifying k_sys_fault_handler to abort usrspace thread This enables aborting a userspace thread if an exception occurs from userspace context on intel adsp ace platforms Signed-off-by: Ganesh, Aditya --- app/os_linux_overlay.conf | 2 ++ src/audio/volume/volume_ipc4.c | 13 +++++++++++-- src/ipc/ipc4/handler-user.c | 8 ++++---- zephyr/wrapper.c | 14 ++++++++++++++ 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/app/os_linux_overlay.conf b/app/os_linux_overlay.conf index 18024b4e6106..d107f2fc15c8 100644 --- a/app/os_linux_overlay.conf +++ b/app/os_linux_overlay.conf @@ -25,3 +25,5 @@ CONFIG_LOG_RUNTIME_DEFAULT_LEVEL=3 # Disable additional VMH buffers to use memory for virtual regions. CONFIG_VIRTUAL_HEAP_EXTENDED=n + +CONFIG_MTRACE_LOG_CACHE_BUF_SIZE=16384 \ No newline at end of file diff --git a/src/audio/volume/volume_ipc4.c b/src/audio/volume/volume_ipc4.c index 82bb71c62eb6..5f4dac8e84cf 100644 --- a/src/audio/volume/volume_ipc4.c +++ b/src/audio/volume/volume_ipc4.c @@ -32,6 +32,9 @@ LOG_MODULE_DECLARE(volume, CONFIG_SOF_LOG_LEVEL); #include "volume.h" +/* Debug only: remove to disable the deliberate fault in volume_init() */ +#define VOLUME_INJECT_FAULT 1 + void set_volume_process(struct vol_data *cd, struct comp_dev *dev, bool source_or_sink) { cd->scale_vol = vol_get_processing_function(dev, cd); @@ -116,10 +119,16 @@ int volume_init(struct processing_module *mod) uint8_t channel_cfg; uint8_t channel; bool all_channels; + + +#ifdef VOLUME_INJECT_FAULT + /* Debug only: deliberately write to NULL to raise a DSP exception */ + comp_err(dev, "volume_init: injecting NULL pointer write fault"); + *(volatile uint32_t *)NULL = 0xdeadbeef; +#endif + uint32_t instance_id = IPC4_INST_ID(dev_comp_id(dev)); - comp_info(dev, "volume_init: entry, instance %u payload size %zu", - instance_id, cfg->size); if (instance_id >= IPC4_MAX_PEAK_VOL_REG_SLOTS) { comp_err(dev, "instance_id %u out of array bounds.", instance_id); diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index d30ea714007e..35193623aa0b 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1805,7 +1805,7 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) break; } - tr_info(&ipc_tr, "INIT_INSTANCE: driver validated, creating component"); + //tr_info(&ipc_tr, "INIT_INSTANCE: driver validated, creating component"); struct comp_dev *dev = comp_new_ipc4_user(&msg, drv); if (!dev) { @@ -1814,7 +1814,7 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) break; } - tr_info(&ipc_tr, "INIT_INSTANCE: component created %p, registering", dev); + //tr_info(&ipc_tr, "INIT_INSTANCE: component created %p, registering", dev); result = ipc4_add_comp_dev(dev); if (result != IPC4_SUCCESS) { tr_err(&ipc_tr, "INIT_INSTANCE: component registration failed %d", @@ -1822,9 +1822,9 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) break; } - tr_info(&ipc_tr, "INIT_INSTANCE: component registered, updating IBS/OBS/CPC"); + //tr_info(&ipc_tr, "INIT_INSTANCE: component registered, updating IBS/OBS/CPC"); comp_update_ibs_obs_cpc(dev); - tr_info(&ipc_tr, "INIT_INSTANCE: IBS/OBS/CPC update completed"); + //tr_info(&ipc_tr, "INIT_INSTANCE: IBS/OBS/CPC update completed"); result = 0; tr_info(&ipc_tr, "INIT_INSTANCE: completed"); diff --git a/zephyr/wrapper.c b/zephyr/wrapper.c index afdc5b54a2e9..1516f052445b 100644 --- a/zephyr/wrapper.c +++ b/zephyr/wrapper.c @@ -28,6 +28,10 @@ /* Zephyr includes */ #include +#if defined(CONFIG_XTENSA) && defined(CONFIG_USERSPACE) +/* Zephyr arch-internal header, from zephyr/arch/xtensa/include */ +#include +#endif #include #include #include @@ -362,6 +366,14 @@ void k_sys_fatal_error_handler(unsigned int reason, } #endif +#if defined(CONFIG_XTENSA) && defined(CONFIG_USERSPACE) + if (reason != K_ERR_KERNEL_PANIC && xtensa_exception_is_user(esf)) { + LOG_ERR("ERROR: Unexpected fatal error %u in Xtensa user mode; " + "aborting thread.\n", reason); + return; + } +#endif + /* flush and switch to immediate mode */ LOG_PANIC(); @@ -370,6 +382,8 @@ void k_sys_fatal_error_handler(unsigned int reason, ipc_send_panic_notification(); #endif + + #if defined(CONFIG_ARCH_POSIX) || defined(CONFIG_ZEPHYR_POSIX) LOG_ERR("Halting emulation"); From 914cb1083f2c5636024a938060c5d4ff980ef26e Mon Sep 17 00:00:00 2001 From: "Ganesh, Aditya" Date: Wed, 18 Nov 2026 05:30:24 -0800 Subject: [PATCH 3/7] zephyr: cpu: Adding syscall for k_thread_cpu_pin This adds a syscall so that we can call k_thread_cpu_pin from userspace Signed-off-by: Ganesh, Aditya --- zephyr/include/sof/lib/cpu.h | 15 +++++++++++++++ zephyr/syscall/cpu.c | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/zephyr/include/sof/lib/cpu.h b/zephyr/include/sof/lib/cpu.h index a47b93898973..5537750ae73c 100644 --- a/zephyr/include/sof/lib/cpu.h +++ b/zephyr/include/sof/lib/cpu.h @@ -50,6 +50,21 @@ void cpu_notify_state_exit(enum pm_state state); */ #if defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) __syscall int cpu_get_id(void); + +struct k_thread; + +/** + * @brief Pin a (not yet started) thread to a single DSP core. + * + * k_thread_cpu_pin() is a plain kernel function that takes the scheduler + * lock and therefore cannot be called from user mode. This system call + * lets a user-mode thread pin a helper thread it has just created. + * + * @param thread Thread object the caller has access to + * @param cpu Target core id + * @return 0 on success, negative error code otherwise + */ +__syscall int sof_thread_cpu_pin(struct k_thread *thread, int cpu); #endif /* let the compiler optimise when in single core mode */ diff --git a/zephyr/syscall/cpu.c b/zephyr/syscall/cpu.c index ee1990db8fd4..6a389c761735 100644 --- a/zephyr/syscall/cpu.c +++ b/zephyr/syscall/cpu.c @@ -20,3 +20,27 @@ static inline int z_vrfy_cpu_get_id(void) return z_impl_cpu_get_id(); } #include + +/** + * @brief Supervisor implementation of sof_thread_cpu_pin(). + */ +int z_impl_sof_thread_cpu_pin(struct k_thread *thread, int cpu) +{ + return k_thread_cpu_pin(thread, cpu); +} + +/** + * @brief Userspace verification wrapper for sof_thread_cpu_pin(). + * + * The caller must hold access to an initialized thread object and the + * core id must be valid. + */ +static inline int z_vrfy_sof_thread_cpu_pin(struct k_thread *thread, int cpu) +{ + K_OOPS(K_SYSCALL_OBJ(thread, K_OBJ_THREAD)); + K_OOPS(K_SYSCALL_VERIFY_MSG(cpu >= 0 && cpu < CONFIG_MP_MAX_NUM_CPUS, + "invalid cpu %d", cpu)); + + return z_impl_sof_thread_cpu_pin(thread, cpu); +} +#include From 8ab6fc953f38545fd20f1cfd2400bf8f4c0d3b36 Mon Sep 17 00:00:00 2001 From: "Ganesh, Aditya" Date: Wed, 18 Nov 2026 05:35:02 -0800 Subject: [PATCH 4/7] ipc: Adding a new thread for module initialization per core This adds capability of invoking a thread for module initialization and is tied to the same core as the ipc_user thread invoking the module initialization Signed-off-by: Ganesh, Aditya --- src/include/sof/ipc/common.h | 8 ++++++++ src/ipc/ipc-common.c | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index a9e87bf40dc6..af8f5ea122ab 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -64,6 +64,14 @@ extern struct tr_ctx ipc_tr; struct ipc_user { struct k_thread *thread[IPC_DSP_CORE_COUNT]; + /** @brief Per-core helper thread objects used for INIT_INSTANCE processing */ + struct k_thread *init_thread[IPC_DSP_CORE_COUNT]; + /** @brief Per-core stacks for the INIT_INSTANCE helper threads */ + k_thread_stack_t *init_stack[IPC_DSP_CORE_COUNT]; + /** @brief Core the currently forwarded command is executed on */ + unsigned int cmd_core; + /** @brief Result of the INIT_INSTANCE helper thread */ + int init_result; struct k_sem *sem; struct k_event *event[IPC_DSP_CORE_COUNT]; /** @brief Copy of IPC4 message primary word forwarded to user thread */ diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 1418e145b03b..7e6a17f8e25d 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -418,6 +418,10 @@ SOF_DEFINE_REG_UUID(sec_core_init); static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); +/* Stacks for the per-core helper threads spawned by the IPC user thread */ +static K_THREAD_STACK_ARRAY_DEFINE(ipc_init_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); + /** * @brief Forward an IPC command to the user-space thread. * @@ -446,6 +450,7 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core pdata->ipc_msg_pri = primary; pdata->ipc_msg_ext = extension; pdata->ipc = ipc; + pdata->cmd_core = core; /* Prevent host completion until user thread finishes */ key = k_spin_lock(&ipc->lock); @@ -542,6 +547,14 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int c goto e_event; } + ipc_user->init_thread[core] = k_object_alloc(K_OBJ_THREAD); + if (!ipc_user->init_thread[core]) { + LOG_ERR("user IPC init thread alloc failed"); + ret = -ENOMEM; + goto e_event; + } + ipc_user->init_stack[core] = ipc_init_stack[core]; + k_thread_create(ipc_user->thread[core], ipc_user_stack[core], CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, @@ -556,6 +569,9 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int c * and signal completion on the primary core semaphore */ k_thread_access_grant(ipc_user->thread[core], ipc_user->sem, ipc_user->event[core]); + /* The IPC thread creates, pins and joins its INIT_INSTANCE helper thread */ + k_thread_access_grant(ipc_user->thread[core], ipc_user->init_thread[core], + ipc_user->init_stack[core]); user_grant_dai_access_all(ipc_user->thread[core]); user_grant_dma_access_all(ipc_user->thread[core]); k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread[core]); From 48f09a0fa3830efa767584b428da8f54a1eb6914 Mon Sep 17 00:00:00 2001 From: "Ganesh, Aditya" Date: Wed, 18 Nov 2026 05:36:36 -0800 Subject: [PATCH 5/7] ipc: ipc4: Invoking a new thread for SOF_IPC4_MOD_INIT_INSTANCE This invokes a thread to complete the module initialization. This is done so that if the module crashes during initialization it can be safely reported back to host without crashing the adsp Signed-off-by: Ganesh, Aditya --- app/os_linux_overlay.conf | 2 +- app/prj.conf | 2 +- src/include/sof/userspace/README.md | 2 +- src/ipc/ipc4/handler-user.c | 110 +++- src/kernel/ipc/ipc-common-kernel.c | 896 ++++++++++++++++++++++++++++ 5 files changed, 983 insertions(+), 29 deletions(-) create mode 100644 src/kernel/ipc/ipc-common-kernel.c diff --git a/app/os_linux_overlay.conf b/app/os_linux_overlay.conf index d107f2fc15c8..34a272b2a9da 100644 --- a/app/os_linux_overlay.conf +++ b/app/os_linux_overlay.conf @@ -26,4 +26,4 @@ CONFIG_LOG_RUNTIME_DEFAULT_LEVEL=3 # Disable additional VMH buffers to use memory for virtual regions. CONFIG_VIRTUAL_HEAP_EXTENDED=n -CONFIG_MTRACE_LOG_CACHE_BUF_SIZE=16384 \ No newline at end of file +CONFIG_MTRACE_LOG_CACHE_BUF_SIZE=32768 \ No newline at end of file diff --git a/app/prj.conf b/app/prj.conf index 5c22e19d6e0e..ef2028436d70 100644 --- a/app/prj.conf +++ b/app/prj.conf @@ -25,7 +25,7 @@ CONFIG_LOG_MODE_DEFERRED=y CONFIG_LOG_PROCESS_TRIGGER_THRESHOLD=1 # Frontend buffer must be large enough to cover all # typical bursts of log messages. -CONFIG_LOG_BUFFER_SIZE=4096 +CONFIG_LOG_BUFFER_SIZE=16384 # Use stack that is sufficient for SOF backends CONFIG_LOG_PROCESS_THREAD_STACK_SIZE=4096 # Use Linux kernel style timestamp format diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index 0ce4abf7d7cc..ad772ceb3305 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -117,7 +117,7 @@ syscall by subsystem: | Virtual regions | `vregion_create_map`, `vregion_set_interim`, `vregion_get`, `vregion_put`, `vregion_alloc_align`, `vregion_alloc_coherent_align`, `vregion_free` | `sof/lib/vregion.h` | `zephyr/syscall/vregion.c` / `zephyr/lib/vregion.c` | | DMA | `sof_dma_get`, `sof_dma_put`, `sof_dma_get_attribute`, `sof_dma_request_channel`, `sof_dma_release_channel`, `sof_dma_config`, `sof_dma_start`, `sof_dma_stop`, `sof_dma_get_status`, `sof_dma_reload`, `sof_dma_suspend`, `sof_dma_resume` | `sof/lib/sof_dma.h` | `zephyr/syscall/sof_dma.c` / `lib/dma.c` (get/put); remaining 10 are `static inline z_impl_*` in `sof/lib/sof_dma.h` | | DAI | `dai_get_device` | `sof/lib/dai-zephyr.h` | `zephyr/syscall/dai.c` / `lib/dai.c` | -| CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` | +| CPU | `cpu_get_id`, `sof_thread_cpu_pin` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` | | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` | | Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `library_manager/lib_manager.c` | | Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `schedule/zephyr_ll.c`, `schedule/zephyr_dp_schedule*.c` | diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 35193623aa0b..5e60a12fc796 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1740,6 +1740,64 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, } #ifdef CONFIG_SOF_USERSPACE_LL +/** + * @brief Helper thread entry performing SOF_IPC4_MOD_INIT_INSTANCE. + * + * Runs in user space on the same core as the IPC user thread that spawned + * it. The IPC message words and driver pointer are taken from the shared + * @c ipc_user context (@p p1); the result is stored in + * @c ipc_user->init_result. + * + * @param p1 struct ipc_user pointer + * @param p2 unused + * @param p3 unused + */ +static void ipc_user_init_instance_fn(void *p1, void *p2, void *p3) +{ + struct ipc_user *ipc_user = p1; + struct ipc4_message_request msg; + const struct comp_driver *drv = ipc_user->init_drv; + struct comp_dev *dev; + int result; + + ARG_UNUSED(p2); + ARG_UNUSED(p3); + + msg.primary.dat = ipc_user->ipc_msg_pri; + msg.extension.dat = ipc_user->ipc_msg_ext; + + /* User thread creates the component: drv->ops.create() runs in + * user-space so untrusted module code does not execute with kernel + * privileges. init_drv is the original kernel pointer. + */ + tr_info(&ipc_tr, "INIT_INSTANCE: driver received %p", drv); + ipc_user->init_drv = NULL; + if (!drv) { + tr_err(&ipc_tr, "INIT_INSTANCE: driver missing"); + ipc_user->init_result = IPC4_MOD_NOT_INITIALIZED; + return; + } + + dev = comp_new_ipc4_user(&msg, drv); + if (!dev) { + tr_err(&ipc_tr, "INIT_INSTANCE: component creation failed"); + ipc_user->init_result = IPC4_MOD_NOT_INITIALIZED; + return; + } + + result = ipc4_add_comp_dev(dev); + if (result != IPC4_SUCCESS) { + tr_err(&ipc_tr, "INIT_INSTANCE: component registration failed %d", result); + ipc_user->init_result = result; + return; + } + + comp_update_ibs_obs_cpc(dev); + + ipc_user->init_result = 0; + tr_info(&ipc_tr, "INIT_INSTANCE: completed"); +} + /* * IPC4 implementation of the user-thread dispatch hook. Runs in the * user-space IPC thread; reconstructs the IPC4 message from the forwarded @@ -1789,45 +1847,45 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) break; } case SOF_IPC4_MOD_INIT_INSTANCE: { - /* User thread creates the component — - * drv->ops.create() runs in user-space so untrusted - * module code does not execute with kernel privileges. - * - * init_drv = original kernel pointer + /* + * Run the instance creation in a helper thread pinned to + * the same core as this IPC user thread, then join it. + * Message words and init_drv are already in ipc_user. */ - const struct comp_driver *drv = ipc_user->init_drv; + unsigned int core = ipc_user->cmd_core; + struct k_thread *thread; - tr_info(&ipc_tr, "INIT_INSTANCE: driver received %p", drv); - ipc_user->init_drv = NULL; - if (!drv) { - tr_err(&ipc_tr, "INIT_INSTANCE: driver missing"); + if (core >= CONFIG_CORE_COUNT || !ipc_user->init_thread[core]) { result = IPC4_MOD_NOT_INITIALIZED; break; } - //tr_info(&ipc_tr, "INIT_INSTANCE: driver validated, creating component"); - struct comp_dev *dev = comp_new_ipc4_user(&msg, drv); - - if (!dev) { - tr_err(&ipc_tr, "INIT_INSTANCE: component creation failed"); + thread = ipc_user->init_thread[core]; + ipc_user->init_result = IPC4_MOD_NOT_INITIALIZED; + + k_thread_create(thread, ipc_user->init_stack[core], + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, + ipc_user_init_instance_fn, ipc_user, NULL, NULL, + -1, K_USER | K_INHERIT_PERMS, K_FOREVER); + result = sof_thread_cpu_pin(thread, core); + if (result) { + tr_err(&ipc_tr, "INIT_INSTANCE: cpu pin failed %d", result); + /* never started: abort so the objects can be reused */ + k_thread_abort(thread); result = IPC4_MOD_NOT_INITIALIZED; break; } + k_thread_name_set(thread, "ipc_init"); + k_thread_start(thread); - //tr_info(&ipc_tr, "INIT_INSTANCE: component created %p, registering", dev); - result = ipc4_add_comp_dev(dev); - if (result != IPC4_SUCCESS) { - tr_err(&ipc_tr, "INIT_INSTANCE: component registration failed %d", - result); + result = k_thread_join(thread, K_FOREVER); + if (result) { + tr_err(&ipc_tr, "INIT_INSTANCE: join failed %d", result); + result = IPC4_MOD_NOT_INITIALIZED; break; } - //tr_info(&ipc_tr, "INIT_INSTANCE: component registered, updating IBS/OBS/CPC"); - comp_update_ibs_obs_cpc(dev); - //tr_info(&ipc_tr, "INIT_INSTANCE: IBS/OBS/CPC update completed"); - - result = 0; - tr_info(&ipc_tr, "INIT_INSTANCE: completed"); + result = ipc_user->init_result; break; } case SOF_IPC4_MOD_DELETE_INSTANCE: { diff --git a/src/kernel/ipc/ipc-common-kernel.c b/src/kernel/ipc/ipc-common-kernel.c new file mode 100644 index 000000000000..2ccf95ba7a45 --- /dev/null +++ b/src/kernel/ipc/ipc-common-kernel.c @@ -0,0 +1,896 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2016 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Keyon Jie + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if CONFIG_IPC_MAJOR_4 +#include +#endif +#include +#include +#include +#include + +#ifdef __ZEPHYR__ +#include +#include +#endif + +#ifdef CONFIG_SOF_USERSPACE_LL +#include +#endif + +#include +#include + +/* Kernel part of the common IPC code; the part that also runs in user mode is in + * src/ipc/ipc-common.c + */ + +LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); + +SOF_DEFINE_REG_UUID(ipc); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); + +#ifdef CONFIG_SOF_USERSPACE_LL +K_APPMEM_PARTITION_DEFINE(ipc_context_part); + +K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; + +struct ipc *ipc_get(void) +{ + return &ipc_context; +} +#endif + +int ipc_process_on_core(uint32_t core, bool blocking) +{ + struct ipc *ipc = ipc_get(); + struct idc_msg msg = { .header = IDC_MSG_IPC, .core = core, }; + int ret; + + /* check if requested core is enabled */ + if (!cpu_is_core_enabled(core)) { + tr_err(&ipc_tr, "core #%d is disabled", core); + return -EACCES; + } + +#if CONFIG_IPC_MAJOR_3 + /* The other core will write there its response */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + ((struct sof_ipc_cmd_hdr *)ipc->comp_data)->size); +#endif + + /* + * If the primary core is waiting for secondary cores to complete, it + * will also reply to the host + */ + if (!blocking) { + k_spinlock_key_t key; + + ipc->core = core; + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_SECONDARY_CORE; + k_spin_unlock(&ipc->lock, key); + } + + /* send IDC message */ + ret = idc_send_msg(&msg, blocking ? IDC_BLOCKING : IDC_NON_BLOCKING); + if (ret < 0) + return ret; + + /* reply written by other core */ + return 1; +} + +void ipc_send_queued_msg(void) +{ + struct ipc *ipc = ipc_get(); + struct ipc_msg *msg; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + + if (ipc->pm_prepare_D3) + goto out; + + /* any messages to send ? */ + if (list_is_empty(&ipc->msg_list)) + goto out; + + msg = list_first_item(&ipc->msg_list, struct ipc_msg, + list); + + if (ipc_platform_send_msg(msg) == 0) { + /* Remove the message from the list if it has been successfully sent. */ + list_item_del(&msg->list); + /* Invoke a callback to notify that the message has been sent. */ + if (msg->callback) + msg->callback(msg); +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_out_msg_count, 1); +#endif + } +out: + k_spin_unlock(&ipc->lock, key); +} + +#ifdef __ZEPHYR__ +static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); + +/* + * Kernel-only: these must not live in struct ipc, which user threads can + * write with CONFIG_SOF_USERSPACE_LL. The work queue embeds the k_thread of + * a supervisor thread and the work item holds the handler it calls. + */ +static struct k_work_q ipc_send_wq; +static struct k_work_delayable ipc_send_work; + +struct k_work_q *ipc_send_wq_get(void) +{ + return &ipc_send_wq; +} +#endif + +static void schedule_ipc_worker(void) +{ + /* + * note: in XTOS builds, this is handled in + * task_main_primary_core() + */ +#ifdef __ZEPHYR__ + k_work_schedule_for_queue(&ipc_send_wq, &ipc_send_work, K_USEC(IPC_PERIOD_USEC)); +#endif +} + +__cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + assert_can_be_cold(); + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + ipc_platform_send_msg_direct(msg); + + k_spin_unlock(&ipc->lock, key); +} + +void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && (msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE) && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + /* + * note: This function can be executed in LL or EDF context, from any core. + * In Zephyr builds, there is IPC queue that is always handled by the primary core, + * whereas submitting to the queue is allowed from any core. Therefore disable option + * of sending IPC immediately by any context/core to secure IPC registers/mailbox + * access. + */ +#ifndef __ZEPHYR__ + /* try to send critical notifications right away */ + if (high_priority) { + ret = ipc_platform_send_msg(msg); + if (!ret) { + k_spin_unlock(&ipc->lock, key); + return; + } + } +#endif + /* add to queue unless already there */ + if (list_is_empty(&msg->list)) { + if (high_priority) + list_item_prepend(&msg->list, &ipc->msg_list); + else + list_item_append(&msg->list, &ipc->msg_list); + } + + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +EXPORT_SYMBOL(ipc_msg_send); + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + /* No caller check: DP modules send notifications, see mod_ipc_msg_init() */ + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); + + K_OOPS(K_SYSCALL_VERIFY(list_is_empty(&msg->list))); + + if (msg->tx_size > 0) { + K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); + } + + /* If data != NULL and tx_size > 0, verify the data buffer */ + if (data && msg->tx_size > 0) + K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); + + z_impl_ipc_msg_send(msg, data, high_priority); +} +#include +#endif /* CONFIG_USERSPACE */ + +void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + list_item_del(&msg->list); + k_spin_unlock(&ipc->lock, key); +} + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) +{ + /* No caller check: called by mod_ipc_msg_free() in DP module threads */ + struct ipc *ipc = ipc_get(); + struct list_item *mlist, *_mlist; + bool found = false; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + /* + * special case: empty list was passed. we can't trust where + * list->prev points to, so do not pass to + * z_impl_ipc_msg_list_remove(), but handle here + */ + if (list_is_empty(&msg->list)) + return; + + list_for_item_safe(mlist, _mlist, &ipc->msg_list) { + if (mlist == &msg->list) { + found = true; + break; + } + } + K_OOPS(K_SYSCALL_VERIFY(found)); + z_impl_ipc_msg_list_remove(msg); +} +#include +#endif /* CONFIG_USERSPACE */ + +#ifdef __ZEPHYR__ +static void ipc_work_handler(struct k_work *work) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + ipc_send_queued_msg(); + + key = k_spin_lock(&ipc->lock); + + if (!list_is_empty(&ipc->msg_list) && !ipc->pm_prepare_D3) + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +#endif + +void ipc_schedule_process(struct ipc *ipc) +{ +#if CONFIG_TWB_IPC_TASK + schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC); +#else + schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC); +#endif +} + +#ifdef CONFIG_SOF_USERSPACE_LL +/* Generic user-space IPC handling thread infrastructure. Protocol-specific + * command interpretation lives in ipc_user_thread_dispatch(), implemented by + * the active IPC major (see src/ipc/ipc4/). + */ + +BUILD_ASSERT(IPC_USER_XCORE_PAYLOAD_SIZE >= IDC_MAX_PAYLOAD_SIZE, + "cross-core request payload smaller than IDC payload"); +#if CONFIG_IPC_MAJOR_4 +BUILD_ASSERT(sizeof(((struct ipc_user_xcore *)0)->ipc4_bind_data) == + sizeof(struct ipc4_module_bind_unbind), + "cross-core request bind data size mismatch"); +#endif + +SOF_DEFINE_REG_UUID(sec_core_init); + +static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); + +/* + * Kernel-only copies of the IPC user thread state. struct ipc_user and the + * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel + * code must use these instead of reading kernel object pointers from there. + */ +static struct ipc_user *ipc_user_kptr; +static struct k_sem *ipc_user_ksem; +static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kaudio_thread[CONFIG_CORE_COUNT]; +static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; + +/* Cross-core request slot semaphores, static so they're never freed */ +static struct k_sem ipc_user_xcore_lock[CONFIG_CORE_COUNT]; +static struct k_sem ipc_user_xcore_done[CONFIG_CORE_COUNT]; + +/** + * @brief Forward an IPC command to the user-space thread. + * + * Called from kernel context (IPC EDF task) to forward the message words + * to the user-space thread for processing. Sets IPC_TASK_IN_THREAD in + * task_mask so the host is not signaled until the user thread completes. + * Blocks until the user thread finishes processing and returns the result. + * + * @param primary Primary message word + * @param extension Extension message word + * @return Result from user thread processing + */ +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc_user_kptr; + k_spinlock_key_t key; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + LOG_DBG("IPC: forward cmd %08x", primary); + + /* Copy message words — original buffer may be reused */ + pdata->ipc_msg_pri = primary; + pdata->ipc_msg_ext = extension; + pdata->ipc = ipc; + + /* Prevent host completion until user thread finishes */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_IN_THREAD; + k_spin_unlock(&ipc->lock, key); + + /* + * Wake the user thread. Post, not set: a cross-core request + * (IPC_USER_EVENT_XCORE) may be pending on the same event. + */ + k_event_post(ipc_user_kevent[core], IPC_USER_EVENT_CMD); + + /* Wait for user thread to complete */ + ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); + if (ret) { + ret = IPC4_MOD_LOAD_CL_FAILED; + LOG_ERR("IPC user: sem error expected %d\n", ret); + /* fall through to complete the cmd */ + } + + /* Clear the task mask bit and check for completion */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_IN_THREAD; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); + + return !ret ? pdata->result : ret; +} + +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) +{ + char thread_name[] = "ll_user0"; + struct k_thread *thread; + struct k_event *event; + int ret; + + assert_can_be_cold(); + + /* + * Build the objects in locals and publish them to struct ipc_user only + * at the end. When a secondary core is brought up, user threads on + * other cores are already running and can overwrite struct ipc_user, + * so its fields must not be read back here. + */ + event = k_object_alloc(K_OBJ_EVENT); + if (!event) { + LOG_ERR("user IPC event alloc failed"); + return -ENOMEM; + } + k_event_init(event); + + thread = k_object_alloc(K_OBJ_THREAD); + if (!thread) { + LOG_ERR("user IPC thread alloc failed"); + ret = -ENOMEM; + goto e_event; + } + + k_thread_create(thread, ipc_user_stack[core], + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, + ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, + -1, K_USER, K_FOREVER); + + k_thread_cpu_pin(thread, core); + thread_name[sizeof(thread_name) - 2] = '0' + core; + k_thread_name_set(thread, thread_name); + + /* + * Each userspace IPC thread must be able to wait on its private event + * and signal completion on the primary core semaphore + */ + k_thread_access_grant(thread, ipc_user_ksem, event); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + user_ll_grant_access(thread, core); + pipeline_posn_grant_access(thread); + + ipc_user_kevent[core] = event; + ipc_user_kthread[core] = thread; + sof_sysuser_thread_set(SOF_SYSUSER_IPC, core, thread); + + /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ + ipc_user->event[core] = event; + ipc_user->thread[core] = thread; + + return 0; + +e_event: + k_object_free(event); + + return ret; +} + +/* + * Grant the threads of the newly initialized core and of the other already + * initialized cores access to each other's cross-core request slots and LL + * scheduler locks. Uses only the kernel-only copies of the thread state. + */ +__cold static void ipc_user_xcore_grant(unsigned int core) +{ + struct k_thread *new_threads[] = { + ipc_user_kthread[core], ipc_user_kaudio_thread[core], + }; + unsigned int c, i; + + assert_can_be_cold(); + + for (c = 0; c < CONFIG_CORE_COUNT; c++) { + struct k_thread *threads[] = { + ipc_user_kthread[c], ipc_user_kaudio_thread[c], + }; + + /* + * Grant the slot semaphores of all cores. Note: braces needed, + * k_thread_access_grant() expands to multiple statements. + */ + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (new_threads[i]) { + k_thread_access_grant(new_threads[i], &ipc_user_xcore_lock[c], + &ipc_user_xcore_done[c]); + } + } + + if (!ipc_user_kthread[c]) + continue; + + for (i = 0; i < ARRAY_SIZE(threads); i++) { + if (!threads[i]) + continue; + k_thread_access_grant(threads[i], ipc_user_kevent[core]); + user_ll_grant_access(threads[i], core); + } + + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (!new_threads[i]) + continue; + k_thread_access_grant(new_threads[i], ipc_user_kevent[c]); + user_ll_grant_access(new_threads[i], c); + } + } +} + +__cold int ipc_user_init_secondary(unsigned int core) +{ + struct ipc_user *ipc_user = ipc_user_kptr; + struct k_thread *audio_thread; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + ret = ipc_user_init_thread(ipc_user, core); + + if (ret < 0) + return ret; + + assert_can_be_cold(); + + k_thread_start(ipc_user_kthread[core]); + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("user LL task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, core, 0); + + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + k_thread_access_grant(ipc_user_kthread[core], audio_thread); + ipc_user_kaudio_thread[core] = audio_thread; + ipc_user->audio_thread[core] = audio_thread; + ipc_user_xcore_grant(core); + + return 0; +} + +struct k_thread *ipc_thread_user(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return NULL; + + /* user threads can not read the kernel-only copy */ + if (k_is_user_context()) + return ipc_get()->ipc_user_pdata->thread[core]; + + return ipc_user_kthread[core]; +} + +struct ipc_user *ipc_user_get(void) +{ + return ipc_user_kptr; +} + +void ipc_user_wait_core_init(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return; + + if (ipc_user_kinit_needed[core]) { + /* wait for IPC thread (ipc_user_thread_fn()) */ + k_sem_take(ipc_user_ksem, K_FOREVER); + ipc_user_kinit_needed[core] = false; + } +} + +/* Primary core only */ +__cold static void ipc_user_init(void) +{ + struct ipc *ipc = ipc_get(); + struct k_thread *audio_thread; + struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc_user), 0); + int ret, core; + + if (!ipc_user) { + LOG_ERR("user IPC pdata alloc failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ + memset(ipc_user, 0, sizeof(*ipc_user)); + + assert_can_be_cold(); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + if (core != PLATFORM_PRIMARY_CORE_ID) + ipc_user_kinit_needed[core] = true; + } + + ipc_user_ksem = k_object_alloc(K_OBJ_SEM); + if (!ipc_user_ksem) { + LOG_ERR("user IPC sem alloc failed"); + k_panic(); + } + ipc_user_kptr = ipc_user; + ipc_user->sem = ipc_user_ksem; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); + if (ret < 0) + LOG_WRN("ipc context partition add failed: %d", ret); + + /* + * Grant user-space read access to the .coldrodata section in IMR. The + * prepare path walks component code that may reference __cold_rodata + * data. + * + * .cold text is deliberately NOT added as a partition: Zephyr already + * maps it executable in the shared ring. + */ +#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM + extern char __coldrodata_start[]; + extern char _imr_end[]; + + if (&_imr_end[0] > &__coldrodata_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__coldrodata_start, + .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); + } +#endif /* CONFIG_COLD_STORE_EXECUTE_DRAM */ + + k_sem_init(ipc_user_ksem, 0, 1); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + struct ipc_user_xcore *xc = &ipc_user->xcore[core]; + + xc->lock = &ipc_user_xcore_lock[core]; + xc->done = &ipc_user_xcore_done[core]; + k_sem_init(xc->lock, 1, 1); + k_sem_init(xc->done, 0, 1); + } + + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); + if (ret < 0) { + LOG_ERR("user IPC thread initialization failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + ret = user_access_to_mailbox(zephyr_ll_mem_domain(), + ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + if (ret < 0) { + LOG_ERR("ipc user: mailbox access grant failed: %d", ret); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* Store references in ipc struct so kernel handler can forward commands */ + ipc->ipc_user_pdata = ipc_user; + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + /* Grant ipc_user thread permission on the audio thread object. + * Needed so user-space dai_common_new() can call + * k_thread_access_grant(audio_thread, dai_mutex) from user context. + */ + k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); + ipc_user_kaudio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + ipc_user_xcore_grant(PLATFORM_PRIMARY_CORE_ID); + + k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + k_sem_take(ipc_user_ksem, K_FOREVER); +} +#else /* CONFIG_SOF_USERSPACE_LL */ +static void ipc_user_init(void) +{ +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc_init(struct sof *sof) +{ + struct k_heap *heap; + struct ipc *ipc; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "entry"); + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + + ipc = ipc_get(); + memset(ipc, 0, sizeof(*ipc)); + ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc->ll_alloc), 0); + if (!ipc->ll_alloc) { + tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); + sof_panic(SOF_IPC_PANIC_IPC); + } + ipc->ll_alloc->heap = heap; + ipc->ll_alloc->vreg = NULL; +#else + heap = NULL; + + /* init ipc data */ + ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); + if (!ipc) { + tr_err(&ipc_tr, "Unable to allocate IPC data"); + return -ENOMEM; + } + sof->ipc = ipc; +#endif /* CONFIG_SOF_USERSPACE_LL */ + + ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc->comp_data) { + tr_err(&ipc_tr, "Unable to allocate IPC component data"); + sof_panic(SOF_IPC_PANIC_IPC); + } + memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); + + k_spinlock_init(&ipc->lock); + list_init(&ipc->msg_list); + list_init(&ipc->comp_list); + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + struct io_perf_data_item init_data = {IO_PERF_IPC_ID, + cpu_get_id(), + IO_PERF_INPUT_DIRECTION, + IO_PERF_POWERED_UP_ENABLED, + IO_PERF_D0IX_POWER_MODE, + 0, 0, 0 }; + io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); + init_data.direction = IO_PERF_OUTPUT_DIRECTION; + io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); +#endif + +#if CONFIG_SOF_BOOT_TEST_STANDALONE + LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); + return 0; +#endif + +#ifdef __ZEPHYR__ + k_tid_t thread; + + k_work_queue_init(&ipc_send_wq); + k_work_queue_start(&ipc_send_wq, ipc_send_wq_stack, + K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); + + thread = k_work_queue_thread_get(&ipc_send_wq); + + k_thread_suspend(thread); + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); +#endif + k_thread_name_set(thread, "ipc_send_wq"); + + k_thread_resume(thread); + + k_work_init_delayable(&ipc_send_work, ipc_work_handler); +#endif /* __ZEPHYR__ */ + + ipc_user_init(); + + return platform_ipc_init(ipc); +} + +/* Locking: call with ipc->lock held and interrupts disabled */ +void ipc_complete_cmd(struct ipc *ipc) +{ + /* + * We have up to three contexts, attempting to complete IPC processing: + * the original IPC EDF task, the IDC EDF task on a secondary core, or + * an LL pipeline thread, running either on the primary or one of + * secondary cores. All these three contexts execute asynchronously. It + * is important to only signal the host that the IPC processing has + * completed after *all* tasks have completed. Therefore only the last + * context should do that. We accomplish this by setting IPC_TASK_* bits + * in ipc->task_mask for each used IPC context and by clearing them when + * each of those contexts completes. Only when the mask is 0 we can + * signal the host. + */ + if (ipc->task_mask) + return; + + ipc_platform_complete_cmd(ipc); +} + +bool ipc_enter_gdb; + +__attribute__((weak)) void ipc_platform_wait_ack(struct ipc *ipc) +{ + k_msleep(1); +} + +static void ipc_complete_task(void *data) +{ + struct ipc *ipc = data; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_INLINE; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); +#if CONFIG_GDBSTUB + if (ipc_enter_gdb) { + ipc_enter_gdb = false; + ipc_platform_wait_ack(ipc); + gdb_init(); + } +#endif +} + +static enum task_state ipc_do_cmd(void *data) +{ + struct ipc *ipc = data; + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_in_msg_count, 1); +#endif + + /* + * 32-bit writes are atomic and at the moment no IPC processing is + * taking place, so, no need for a lock. + */ + ipc->task_mask = IPC_TASK_INLINE; + + return ipc_platform_do_cmd(ipc); +} + +struct task_ops ipc_task_ops = { + .run = ipc_do_cmd, + .complete = ipc_complete_task, + .get_deadline = ipc_task_deadline, +}; From 33556c15a8cf6677f48447e9d6a3fcdebc7aced1 Mon Sep 17 00:00:00 2001 From: "Ganesh, Aditya" Date: Wed, 18 Nov 2026 11:02:05 -0800 Subject: [PATCH 6/7] kernel: ipc: ipc user init thread setup This setups up the ipc user init thread required for module initialization Signed-off-by: Ganesh, Aditya --- src/kernel/ipc/ipc-common-kernel.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/kernel/ipc/ipc-common-kernel.c b/src/kernel/ipc/ipc-common-kernel.c index 832a14a1a9de..c46521aa8313 100644 --- a/src/kernel/ipc/ipc-common-kernel.c +++ b/src/kernel/ipc/ipc-common-kernel.c @@ -469,6 +469,10 @@ SOF_DEFINE_REG_UUID(sec_core_init); static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); +/* Stacks for the per-core helper threads spawned by the IPC user thread */ +static K_THREAD_STACK_ARRAY_DEFINE(ipc_init_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); + /* * Kernel-only copies of the IPC user thread state. struct ipc_user and the * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel @@ -513,7 +517,7 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core pdata->ipc_msg_pri = primary; pdata->ipc_msg_ext = extension; pdata->ipc = ipc; - + pdata->cmd_core = core; /* Prevent host completion until user thread finishes */ key = k_spin_lock(&ipc_ks(ipc)->lock); ipc_ks(ipc)->task_mask |= IPC_TASK_IN_THREAD; @@ -545,6 +549,7 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int c { char thread_name[] = "ll_user0"; struct k_thread *thread; + struct k_thread *init_thread; struct k_event *event; int ret; @@ -570,6 +575,13 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int c goto e_event; } + init_thread = k_object_alloc(K_OBJ_THREAD); + if (!init_thread) { + LOG_ERR("user IPC init thread alloc failed"); + ret = -ENOMEM; + goto e_thread; + } + k_thread_create(thread, ipc_user_stack[core], CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, @@ -584,6 +596,7 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int c * and signal completion on the primary core semaphore */ k_thread_access_grant(thread, ipc_user_ksem, event); + k_thread_access_grant(thread, init_thread, ipc_init_stack[core]); user_grant_dai_access_all(thread); user_grant_dma_access_all(thread); k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); @@ -597,9 +610,11 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int c /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ ipc_user->event[core] = event; ipc_user->thread[core] = thread; - + ipc_user->init_thread[core] = init_thread; + ipc_user->init_stack[core] = ipc_init_stack[core]; return 0; - +e_thread: + k_object_free(thread); e_event: k_object_free(event); From 7e6acbf2ff954c711420c71fc129d7bf47471857 Mon Sep 17 00:00:00 2001 From: "Ganesh, Aditya" Date: Wed, 18 Nov 2026 13:33:19 -0800 Subject: [PATCH 7/7] zephyr: syscall: Abort user thread if kernel oops raised by syscall verified This adds the capability of aborting a user thread if kernel oops is raised from a syscall verifier Signed-off-by: Ganesh, Aditya --- zephyr/wrapper.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/zephyr/wrapper.c b/zephyr/wrapper.c index 2078f9754477..8809188e2fa9 100644 --- a/zephyr/wrapper.c +++ b/zephyr/wrapper.c @@ -369,6 +369,21 @@ void k_sys_fatal_error_handler(unsigned int reason, #endif #if defined(CONFIG_XTENSA) && defined(CONFIG_USERSPACE) + /* + * A kernel oops raised from a syscall verifier (K_OOPS) or + * k_oops() arrives with a syscall stack frame, not an exception + * frame, so xtensa_exception_is_user() cannot validate it and + * would wrongly halt the system. If the faulting thread is a + * user thread, the oops is the user thread's fault: just abort + * that thread (z_fatal_error() does so once we return). + */ + if (reason == K_ERR_KERNEL_OOPS && _current && + k_thread_is_user_thread(_current)) { + LOG_ERR("ERROR: kernel oops from user thread %p; aborting thread.", + _current); + return; + } + if (reason != K_ERR_KERNEL_PANIC && xtensa_exception_is_user(esf)) { LOG_ERR("ERROR: Unexpected fatal error %u in Xtensa user mode; " "aborting thread.\n", reason);