From 57799acab73278562bcc0f1cf0242022a6ec9a81 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 6 Oct 2026 21:30:40 +0000 Subject: [PATCH] multikernel: use per-instance duplex IPI links Children currently publish into a shared parent inbox, while hardirq sends depend on mutex-backed instance lookup. Give each parent/child pair separate rings and serialize local producers in each direction. Use RCU for endpoint and handler lifetime. Bound dispatch to eight links and eight slots per link, with readiness scanning for fair continuation. Keep IRQ-disabled sends and polling available for PCI startup, and defer ordinary interrupt dispatch until built-in consumers register. Attach endpoints during exec and detach before publishing a stopped state. Confirm park before resetting the selected link, and reject deferred halt work from an earlier start. Keep image loading independent of transport activation. Route to the manifest's parent ID, map its halt area separately, and order publication before the APIC doorbell. Signed-off-by: Nikolay Nikolaev --- arch/x86/kernel/smp.c | 2 + drivers/tty/mktty.c | 34 +- include/linux/multikernel.h | 81 +++- kernel/kexec_core.c | 11 + kernel/multikernel/core.c | 86 ++-- kernel/multikernel/instance_dt.c | 70 +++- kernel/multikernel/ipi.c | 680 ++++++++++++++++++------------- kernel/multikernel/kernfs.c | 2 + kernel/multikernel/manifest.c | 5 + 9 files changed, 623 insertions(+), 348 deletions(-) diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index 993766b5a1d1bc..bc175413c40f47 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -342,6 +342,8 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_multikernel) */ void mk_arch_send_ipi(u64 phys_cpu) { + /* Order shared-memory publication before ringing the x2APIC doorbell. */ + weak_wrmsr_fence(); apic_icr_write(APIC_DM_FIXED | APIC_DEST_PHYSICAL | MULTIKERNEL_VECTOR, (u32)phys_cpu); } diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index f544bf342a09d0..cb40cf426d250a 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -337,6 +337,18 @@ static struct tty_driver *mktty_spawn_driver; static struct mktty_spawn_state mktty_spawn; static struct mk_ipi_handler *mktty_spawn_handler; static struct console mktty_spawn_console; + +static int mktty_spawn_parent_id(void) +{ + if (!mk_self) + return -ENODEV; + if (mk_self->id == 0) + return 0; + if (!host_instance) + return -ENODEV; + return READ_ONCE(host_instance->id); +} + static bool mktty_console_registered; static int mktty_spawn_activate(struct tty_port *port, struct tty_struct *tty) @@ -377,10 +389,13 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, { struct mktty_message *msg; size_t sent = 0, chunk; - int ret; + int parent_id, ret; if (tty->index != 0) return -ENODEV; + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + return parent_id; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) @@ -394,9 +409,9 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, msg->reserved = 0; memcpy(msg->data, buf + sent, chunk); - ret = multikernel_send_ipi_data(0, msg, - sizeof(*msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + ret = multikernel_send_ipi_data(parent_id, msg, + sizeof(*msg) - MKTTY_MAX_DATA + chunk, + MKTTY_IPI_TYPE); if (ret < 0) { kfree(msg); return sent > 0 ? sent : ret; @@ -449,8 +464,12 @@ static void mktty_spawn_console_write(struct console *con, const char *s, { unsigned long flags; size_t chunk; + int parent_id; spin_lock_irqsave(&mktty_console_lock, flags); + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + goto out; while (count > 0) { chunk = min_t(size_t, count, MKTTY_MAX_DATA); mktty_console_msg.type = MKTTY_MSG_OUTPUT; @@ -459,12 +478,13 @@ static void mktty_spawn_console_write(struct console *con, const char *s, mktty_console_msg.reserved = 0; memcpy(mktty_console_msg.data, s, chunk); - multikernel_send_ipi_data(0, &mktty_console_msg, - sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + multikernel_send_ipi_data(parent_id, &mktty_console_msg, + sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk, + MKTTY_IPI_TYPE); s += chunk; count -= chunk; } +out: spin_unlock_irqrestore(&mktty_console_lock, flags); } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 76e51d19b91381..6ec882d7df7680 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -6,7 +6,9 @@ #define _LINUX_MULTIKERNEL_H #include -#include +#include +#include +#include #include #include #include @@ -88,6 +90,7 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + u32 ready; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ @@ -96,8 +99,6 @@ struct mk_ipi_data { /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ }; @@ -109,9 +110,10 @@ struct mk_ipi_ring { /* Presence table capacity, in CPUs (see mk_cpu_rank) */ #define MK_PARKED_MAX 512 -/* Shared memory structures - per-instance design */ +/* One duplex link per parent/child pair. Each ring has one kernel producer. */ struct mk_shared_data { - struct mk_ipi_ring ring; /* IPI message ring buffer */ + struct mk_ipi_ring to_child; + struct mk_ipi_ring to_parent; /* * Force-halt marker, host-owned. Armed before the host NMIs the * instance's CPUs and cleared with the rest of this struct when @@ -133,6 +135,22 @@ struct mk_shared_data { u8 parked[MK_PARKED_MAX]; }; +struct mk_instance; + +struct mk_ipi_endpoint { + struct mk_ipi_ring *tx; + struct mk_ipi_ring *rx; + raw_spinlock_t tx_lock; + raw_spinlock_t rx_lock; + u32 tx_head; + u32 rx_tail; + mk_phys_cpu_t doorbell_cpu; + bool tx_enabled; + bool rx_dispatching; + bool registered; + struct list_head rx_node; +}; + /* Function pointer type for IPI callbacks */ typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *data, void *ctx); @@ -140,7 +158,7 @@ struct mk_ipi_handler { mk_ipi_callback_t callback; void *context; unsigned int ipi_type; /* IPI type this handler is registered for */ - struct mk_ipi_handler *next; + struct list_head node; }; /** @@ -156,6 +174,9 @@ struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, /** * multikernel_unregister_handler - Unregister a multikernel IPI callback * @handler: Handler pointer returned from multikernel_register_handler + * + * Must be called from process context and not from a multikernel IPI + * callback. */ void multikernel_unregister_handler(struct mk_ipi_handler *handler); @@ -166,8 +187,8 @@ void multikernel_unregister_handler(struct mk_ipi_handler *handler); * @data_size: Size of data * @type: User-defined type identifier * - * This function copies the data to per-CPU storage and sends an IPI - * to the target CPU. + * This function copies the data into the target instance's duplex ring and + * sends an IPI to its doorbell CPU. * * Returns 0 on success, negative error code on failure */ @@ -183,8 +204,40 @@ struct mk_instance; int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, size_t data_size, unsigned long type); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +#ifdef CONFIG_MULTIKERNEL +int mk_ipi_endpoint_register(struct mk_instance *instance, bool parent_side, + mk_phys_cpu_t doorbell_cpu); +void mk_ipi_endpoint_unregister(struct mk_instance *instance); +int mk_ipi_link_reset(struct mk_instance *instance); +void mk_ipi_handlers_enable(void); +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance); +#else +static inline int mk_ipi_endpoint_register(struct mk_instance *instance, + bool parent_side, + mk_phys_cpu_t doorbell_cpu) +{ + return -EOPNOTSUPP; +} + +static inline void mk_ipi_endpoint_unregister(struct mk_instance *instance) +{ +} + +static inline int mk_ipi_link_reset(struct mk_instance *instance) +{ + return -EOPNOTSUPP; +} + +static inline void mk_ipi_handlers_enable(void) +{ +} + +static inline struct mk_shared_data * +mk_instance_halt_data(struct mk_instance *instance) +{ + return NULL; +} +#endif /* * Multikernel Messaging System @@ -738,6 +791,13 @@ struct mk_instance { struct mk_shared_data *ipi_data; /* IPI shared memory buffer (virtual address) */ phys_addr_t ipi_phys; /* IPI buffer physical address */ u32 ipi_pages; /* IPI buffer size in pages */ + /* + * A spawn's separate mapping of its parent's force-halt area. + * Ordinary child records keep this NULL and use @ipi_data. + */ + struct mk_shared_data *halt_data; + struct mk_ipi_endpoint ipi_endpoint; + u64 ipi_start_sequence; /* Local start identity for deferred halt work */ /* * Host side only: the page the instance's boot tree is written to at * each exec. It and the IPI ring are allocated on the first load and @@ -1157,6 +1217,7 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + #endif /** diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index b7a6fc6eb6ee0d..1aaee8ddade159 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1676,6 +1676,10 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + rc = -EBUSY; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1707,6 +1711,13 @@ int multikernel_kexec_by_id(int mk_id) goto unlock; } + mk_ipi_endpoint_unregister(instance); + rc = mk_ipi_link_reset(instance); + if (rc) { + pr_err("Failed to reset instance %d IPI link: %d\n", mk_id, rc); + goto unlock; + } + /* * Booting consumes the image: the spawn kernel writes its .data and * patches its own text, so the copy in instance memory is spent once diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 7fffac2c1c25aa..688e77cee7a47c 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -2,6 +2,7 @@ /* * Copyright (C) 2025 Multikernel Technologies, Inc. All rights reserved */ +#include #include #include #include @@ -13,9 +14,12 @@ #include #include #include +#include #include #include "internal.h" +static atomic64_t mk_ipi_start_sequence = ATOMIC64_INIT(0); + static void mk_instance_return_all_cpus(struct mk_instance *instance) { if (!instance || mk_cpu_set_empty(instance->cpus)) @@ -217,6 +221,8 @@ static void mk_instance_release(struct kref *kref) pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); + WARN_ON_ONCE(READ_ONCE(instance->ipi_endpoint.registered)); + WARN_ON_ONCE(instance->ipi_endpoint.tx || instance->ipi_endpoint.rx); mk_instance_return_all_cpus(instance); mk_instance_return_pci_devices(instance); @@ -225,6 +231,8 @@ static void mk_instance_release(struct kref *kref) mk_instance_free_boot_pages(instance); mk_instance_track_dump(instance, instance->state, MK_STATE_READY); + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -278,6 +286,9 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) instance->state = MK_STATE_READY; instance->ipi_target = MK_PHYS_CPU_INVALID; + raw_spin_lock_init(&instance->ipi_endpoint.tx_lock); + raw_spin_lock_init(&instance->ipi_endpoint.rx_lock); + INIT_LIST_HEAD(&instance->ipi_endpoint.rx_node); INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); @@ -344,6 +355,8 @@ void mk_instance_free(struct mk_instance *instance) list_del(&plat_dev->list); kfree(plat_dev); } + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -361,12 +374,14 @@ void mk_instance_set_state(struct mk_instance *instance, if (old_state == state) return; + /* Retire transport readers before stopped state permits another exec. */ + if (old_state == MK_STATE_ACTIVE) { + mk_ipi_endpoint_unregister(instance); + mk_pciback_unlend(instance); + } + instance->state = state; mk_instance_track_dump(instance, old_state, state); - - /* Whatever stopped it, its functions are no longer its to reach */ - if (old_state == MK_STATE_ACTIVE) - mk_pciback_unlend(instance); pr_debug("Instance %d (%s) state: %s -> %s\n", instance->id, instance->name, mk_state_to_string(old_state), @@ -427,34 +442,26 @@ int mk_instance_start(struct mk_instance *instance, struct kimage *image, int ret; /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. + * The exec caller has confirmed park, retired old callbacks, and reset + * this link. An old HALTED work item cannot belong to this new start. */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); - - /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. - */ - mk_ipi_ring_drop_pending(); + instance->ipi_start_sequence = atomic64_inc_return(&mk_ipi_start_sequence); + ret = mk_ipi_endpoint_register(instance, true, arch_cpu_physical_id(cpu)); + if (ret) + return ret; ret = mk_pciback_lend(instance); if (ret) { pr_err("Instance %d: its PCI functions cannot be lent: %d\n", instance->id, ret); + mk_ipi_endpoint_unregister(instance); return ret; } ret = mk_arch_spawn_instance(image, instance, cpu); if (ret) { mk_pciback_unlend(instance); + mk_ipi_endpoint_unregister(instance); return ret; } @@ -1629,7 +1636,12 @@ static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) */ void __noreturn mk_halt_to_pool(void) { - mk_notify_down_and_park(0, MK_SYS_HALTED); + int parent_id; + + if (!host_instance) + panic("multikernel: spawned kernel has no parent instance"); + parent_id = READ_ONCE(host_instance->id); + mk_notify_down_and_park(parent_id, MK_SYS_HALTED); } static void mk_shutdown_work_fn(struct work_struct *work) @@ -1658,6 +1670,7 @@ static void mk_instance_settle_halted(struct mk_instance *instance) struct mk_halted_work { struct work_struct work; int instance_id; + u64 received_start_sequence; }; static void mk_halted_work_fn(struct work_struct *work) @@ -1666,14 +1679,19 @@ static void mk_halted_work_fn(struct work_struct *work) container_of(work, struct mk_halted_work, work); struct mk_instance *instance; + /* Exec and the synchronous halt paths already hold this mutex. */ + mutex_lock(&system_transition_mutex); instance = mk_instance_find(aw->instance_id); if (instance) { - mk_instance_settle_halted(instance); + if (instance->state == MK_STATE_ACTIVE && + instance->ipi_start_sequence <= aw->received_start_sequence) + mk_instance_settle_halted(instance); mk_instance_put(instance); } else { pr_warn("Shutdown ACK from unknown instance %d\n", aw->instance_id); } + mutex_unlock(&system_transition_mutex); kfree(aw); } @@ -1738,6 +1756,8 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, INIT_WORK(&aw->work, mk_halted_work_fn); aw->instance_id = ack->resource_id; + /* Exec's unregister grace puts this snapshot before the next start. */ + aw->received_start_sequence = atomic64_read(&mk_ipi_start_sequence); schedule_work(&aw->work); break; } @@ -1875,7 +1895,7 @@ EXPORT_SYMBOL_GPL(mk_cpu_rank); static int mk_fence_missing(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing = 0; @@ -1900,7 +1920,7 @@ static int mk_fence_missing(struct mk_instance *instance, static int mk_confirm_fenced(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing, ret; @@ -2007,8 +2027,10 @@ int multikernel_force_halt_by_id(int mk_id) */ if (instance == host_instance) { ret = mk_confirm_fenced(instance, targets); - if (!ret) + if (!ret) { + mk_ipi_endpoint_unregister(instance); mk_pcifront_host_fenced(); + } } else { mk_instance_settle_halted(instance); } @@ -2018,6 +2040,8 @@ int multikernel_force_halt_by_id(int mk_id) return ret; } +static bool multikernel_initialized __initdata; + static int __init multikernel_init(void) { int ret; @@ -2052,9 +2076,21 @@ static int __init multikernel_init(void) return ret; } + multikernel_initialized = true; + pr_info("Multikernel support initialized\n"); return 0; } +static int __init multikernel_ipi_enable(void) +{ + if (!multikernel_initialized) + return 0; + + mk_ipi_handlers_enable(); + return 0; +} + /* Initialize multikernel after core kernel subsystems are ready */ subsys_initcall(multikernel_init); +late_initcall(multikernel_ipi_enable); diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 8503733372da18..25af5b7ed7afb6 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -297,6 +297,11 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) return 0; } ipi_size = (size_t)ipi_pages << PAGE_SHIFT; + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small: %zu < %zu\n", ipi_size, + sizeof(struct mk_shared_data)); + return -EINVAL; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -316,38 +321,58 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) static int __init mk_restore_host_instance(void) { struct mk_instance *hi; - phys_addr_t host_ipi_phys; - u32 host_ipi_pages; - size_t host_ipi_size; + phys_addr_t halt_phys; + u32 halt_pages; + u32 parent_id; + size_t halt_size; u64 host_ipi_cpu; + int ret; - if (!mk_chosen_ring("host-ipi", &host_ipi_phys, &host_ipi_pages) || + if (!mk_self || !mk_self->ipi_data || + of_property_read_u32(of_chosen, "multikernel,parent-id", &parent_id) || of_property_read_u64(of_chosen, "multikernel,host-ipi-cpu", - &host_ipi_cpu)) { - pr_warn("No host IPI buffer in the boot tree (spawn won't be able to send to host)\n"); + &host_ipi_cpu) || + !mk_chosen_ring("host-ipi", &halt_phys, &halt_pages)) { + pr_warn("Incomplete parent IPI data in the boot tree\n"); return -ENOENT; } - host_ipi_size = (size_t)host_ipi_pages << PAGE_SHIFT; + if (parent_id > INT_MAX || parent_id == mk_self->id || + host_ipi_cpu == MK_PHYS_CPU_INVALID) { + pr_err("Invalid parent endpoint: id=%u, doorbell=%llu\n", + parent_id, host_ipi_cpu); + return -EINVAL; + } + + halt_size = (size_t)halt_pages << PAGE_SHIFT; + if (halt_size < sizeof(struct mk_shared_data) || + halt_phys == mk_self->ipi_phys) { + pr_err("Invalid parent force-halt area: phys=0x%llx, pages=%u\n", + (unsigned long long)halt_phys, halt_pages); + return -EINVAL; + } - hi = mk_instance_alloc(0, "host"); + hi = mk_instance_alloc(parent_id, "host"); if (!hi) return -ENOMEM; hi->ipi_target = host_ipi_cpu; - - hi->ipi_data = memremap(host_ipi_phys, host_ipi_size, MEMREMAP_WB); - if (!hi->ipi_data) { - pr_err("Failed to map host IPI buffer at 0x%llx\n", - (unsigned long long)host_ipi_phys); + hi->ipi_data = mk_self->ipi_data; + hi->ipi_phys = mk_self->ipi_phys; + hi->ipi_pages = mk_self->ipi_pages; + hi->halt_data = memremap(halt_phys, halt_size, MEMREMAP_WB); + if (!hi->halt_data) { + pr_err("Failed to map parent force-halt area at 0x%llx\n", + (unsigned long long)halt_phys); + ret = -ENOMEM; goto err_free; } - hi->ipi_phys = host_ipi_phys; - hi->ipi_pages = host_ipi_pages; - if (mk_instance_publish(hi)) { - memunmap(hi->ipi_data); + ret = mk_ipi_endpoint_register(hi, false, host_ipi_cpu); + if (ret) goto err_free; - } + ret = mk_instance_publish(hi); + if (ret) + goto err_endpoint; /* The host is running, or this kernel would not be */ mk_instance_set_state(hi, MK_STATE_ACTIVE); @@ -362,15 +387,16 @@ static int __init mk_restore_host_instance(void) host_instance = hi; - pr_info("Restored host IPI buffer: phys=0x%llx, pages=%u\n", - (unsigned long long)host_ipi_phys, host_ipi_pages); - pr_info("Registered host instance (ID 0) for spawn→host communication\n"); + pr_info("Registered parent instance %u on the duplex IPI link\n", + parent_id); return 0; +err_endpoint: + mk_ipi_endpoint_unregister(hi); err_free: mk_instance_free(hi); - return -ENOMEM; + return ret; } /** diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 4a6a44f94c88dc..f1db8a9be167ef 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -2,423 +2,535 @@ /* * Copyright (C) 2025 Multikernel Technologies, Inc. All rights reserved */ -#include #include -#include -#include +#include #include -#include #include -#include -#include -#include +#include +#include +#include #include "internal.h" -/* Callback management */ -static struct mk_ipi_handler *mk_handlers; -static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); +#define MK_IPI_ENDPOINT_BATCH 8 +#define MK_IPI_LINK_BUDGET 8 + +static LIST_HEAD(mk_handlers); +static DEFINE_RAW_SPINLOCK(mk_handlers_lock); -static DEFINE_RAW_SPINLOCK(mk_drain_lock); +static LIST_HEAD(mk_ipi_endpoints); +static DEFINE_RAW_SPINLOCK(mk_ipi_endpoints_lock); +static struct list_head *mk_ipi_cursor = &mk_ipi_endpoints; +static unsigned int mk_ipi_endpoint_count; +static bool mk_handlers_ready; static DEFINE_PER_CPU(bool, mk_draining); -static void mk_ipi_drain_ring(void); +static inline unsigned int mk_ring_idx(unsigned int index) +{ + return index & (MK_IPI_RING_SIZE - 1); +} -/* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. +static noinline void mk_ipi_kick_local(void) +{ + mk_arch_send_ipi(arch_cpu_physical_id(0)); +} + +static void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + unsigned int i; + + for (i = 0; i < MK_IPI_RING_SIZE; i++) + WRITE_ONCE(ring->entries[i].ready, 0); +} + +static void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->to_child); + mk_ipi_ring_reset(&shared->to_parent); + WRITE_ONCE(shared->force_halt, 0); + memset(shared->parked, 0, sizeof(shared->parked)); +} + +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance) +{ + struct mk_shared_data *shared = READ_ONCE(instance->halt_data); + + return shared ? shared : READ_ONCE(instance->ipi_data); +} + +/** + * mk_ipi_endpoint_register - Publish one side of a duplex IPI link + * @instance: Record for the peer at the other end of the link + * @parent_side: True in the parent kernel, false in its child + * @doorbell_cpu: Stable physical CPU to notify in the peer + * + * Ordinary detach and re-registration preserve the local ring indices. */ -static inline unsigned int mk_ring_idx(unsigned int i) +int mk_ipi_endpoint_register(struct mk_instance *instance, bool parent_side, + mk_phys_cpu_t doorbell_cpu) { - return i & (MK_IPI_RING_SIZE - 1); + struct mk_ipi_endpoint *endpoint; + struct mk_shared_data *shared; + unsigned long flags; + + if (!instance || doorbell_cpu == MK_PHYS_CPU_INVALID) + return -EINVAL; + shared = READ_ONCE(instance->ipi_data); + if (!shared) + return -ENODEV; + + endpoint = &instance->ipi_endpoint; + if (WARN_ON_ONCE(READ_ONCE(endpoint->registered))) + return -EALREADY; + + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx = parent_side ? &shared->to_child : &shared->to_parent; + endpoint->doorbell_cpu = doorbell_cpu; + endpoint->tx_enabled = true; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx = parent_side ? &shared->to_parent : &shared->to_child; + endpoint->rx_dispatching = false; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + INIT_LIST_HEAD(&endpoint->rx_node); + list_add_tail_rcu(&endpoint->rx_node, &mk_ipi_endpoints); + WRITE_ONCE(endpoint->registered, true); + mk_ipi_endpoint_count++; + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + + /* Pairs with the handler publication in mk_ipi_handlers_enable(). */ + if (smp_load_acquire(&mk_handlers_ready)) + mk_ipi_kick_local(); + return 0; } /** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring + * mk_ipi_endpoint_unregister - Stop and unpublish a duplex endpoint + * @instance: Peer record whose endpoint is being detached * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. + * Must be called from process context and never from an IPI callback. The + * function waits for senders and hardirq callbacks that already found the + * endpoint before returning. Local ring indices are preserved for reattach. */ -void mk_ipi_ring_drop_pending(void) +void mk_ipi_endpoint_unregister(struct mk_instance *instance) { - struct mk_ipi_ring *ring; - unsigned int head, tail; + struct mk_ipi_endpoint *endpoint; + unsigned long flags; + bool removed = false; - if (!mk_self || !mk_self->ipi_data) + if (!instance) return; + endpoint = &instance->ipi_endpoint; + + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx_enabled = false; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + if (endpoint->registered) { + if (mk_ipi_cursor == &endpoint->rx_node) + mk_ipi_cursor = endpoint->rx_node.prev; + list_del_rcu(&endpoint->rx_node); + WRITE_ONCE(endpoint->registered, false); + mk_ipi_endpoint_count--; + removed = true; + } + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + + if (!removed) + return; + + synchronize_rcu(); + + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx = NULL; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx = NULL; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + INIT_LIST_HEAD(&endpoint->rx_node); +} - ring = &mk_self->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); +/** + * mk_ipi_link_reset - Start a confirmed-stopped peer link at generation zero + * @instance: Peer whose CPUs are parked and endpoint is detached + * + * Returns 0 on success or a negative error when the reset preconditions are + * not met. + */ +int mk_ipi_link_reset(struct mk_instance *instance) +{ + struct mk_ipi_endpoint *endpoint; + struct mk_shared_data *shared; + unsigned long flags; - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; + if (!instance) + return -EINVAL; + endpoint = &instance->ipi_endpoint; + shared = READ_ONCE(instance->ipi_data); + if (!shared) + return -ENODEV; - atomic_set(&ring->tail, head); + if (WARN_ON_ONCE(READ_ONCE(endpoint->registered))) + return -EBUSY; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + raw_spin_lock(&endpoint->rx_lock); + mk_shared_data_reset(shared); + endpoint->tx_head = 0; + endpoint->rx_tail = 0; + endpoint->rx_dispatching = false; + raw_spin_unlock(&endpoint->rx_lock); + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + return 0; } /** - * multikernel_register_handler - Register a callback for multikernel IPI - * @callback: Function to call when IPI is received - * @ctx: Context pointer passed to the callback - * @ipi_type: IPI type this handler should process + * multikernel_register_handler - Register a hardirq IPI callback + * @callback: Atomic callback invoked directly from the IPI handler + * @ctx: Opaque callback context + * @ipi_type: Message type delivered to the callback * - * Returns pointer to handler on success, NULL on failure + * The callback must not sleep and must not unregister itself. */ -struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, unsigned int ipi_type) +struct mk_ipi_handler * +multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, + unsigned int ipi_type) { struct mk_ipi_handler *handler; unsigned long flags; if (!callback) return NULL; - handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler) return NULL; - handler->callback = callback; handler->context = ctx; handler->ipi_type = ipi_type; raw_spin_lock_irqsave(&mk_handlers_lock, flags); - handler->next = mk_handlers; - mk_handlers = handler; + list_add_rcu(&handler->node, &mk_handlers); raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - return handler; } EXPORT_SYMBOL(multikernel_register_handler); /** - * multikernel_unregister_handler - Unregister a multikernel IPI callback - * @handler: Handler pointer returned from multikernel_register_handler + * multikernel_unregister_handler - Unregister an IPI callback + * @handler: Handler returned by multikernel_register_handler() + * + * Must be called from process context and outside the callback. It waits for + * hardirq readers that may still be using the callback and its context. */ void multikernel_unregister_handler(struct mk_ipi_handler *handler) { - struct mk_ipi_handler **pp, *p; unsigned long flags; if (!handler) return; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); - pp = &mk_handlers; - while ((p = *pp) != NULL) { - if (p == handler) { - *pp = p->next; - break; - } - pp = &p->next; - } + list_del_rcu(&handler->node); raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - - kfree(p); + synchronize_rcu(); + kfree(handler); } EXPORT_SYMBOL(multikernel_unregister_handler); -/** - * mk_arm_force_halt - Post the force-halt marker for an instance - * @instance: Instance about to be NMIed - * - * The instance's CPUs test the marker from their NMI handlers, so it - * must be armed before the NMIs are sent. It stays armed until the - * kexec path has confirmed every CPU parked and wipes the shared area - * for the next run, which is what makes the NMI rescue idempotent: a - * repeat force halt still reaches CPUs an earlier one missed. - * - * Returns 0 on success, -ENODEV if the instance has no shared IPI area. - */ int mk_arm_force_halt(struct mk_instance *instance) { - struct mk_shared_data *ipi_data = READ_ONCE(instance->ipi_data); + struct mk_shared_data *shared = mk_instance_halt_data(instance); - if (!ipi_data) + if (!shared) return -ENODEV; - - WRITE_ONCE(ipi_data->force_halt, 1); - /* The marker must be visible before the NMIs that test it */ + WRITE_ONCE(shared->force_halt, 1); smp_wmb(); return 0; } /** - * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID - * @data: Pointer to data to send - * @data_size: Size of data - * @type: User-defined type identifier + * mk_ipi_send - Queue data on a peer's duplex link + * @peer: Referenced peer, or NULL for lookup by ID + * @instance_id: Peer instance ID + * @data: Payload, or NULL for a zero-length message + * @data_size: Payload size + * @type: Message type * - * This function enqueues data into the target instance's IPI ring buffer - * and sends an IPI to notify the target CPU. - * - * Returns 0 on success, negative error code on failure + * Safe in IRQ-disabled and hardirq contexts. Endpoint lookup is RCU-protected + * and publication is serialized per link direction. */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +static int mk_ipi_send(struct mk_instance *peer, int instance_id, void *data, + size_t data_size, unsigned long type) { - struct mk_instance *instance = mk_instance_find(instance_id); - int ret; + struct mk_ipi_endpoint *endpoint; + struct mk_ipi_data *slot; + mk_phys_cpu_t target = MK_PHYS_CPU_INVALID; + unsigned long flags; + int ret = -ENODEV; + u32 index; - if (!instance) + if (data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; - ret = multikernel_send_ipi_data_to(instance, data, data_size, type); - mk_instance_put(instance); - return ret; -} + rcu_read_lock(); + list_for_each_entry_rcu(endpoint, &mk_ipi_endpoints, rx_node) { + struct mk_instance *instance; -/* - * The same for a caller that holds a reference to @instance. Looking an - * instance up by ID takes a mutex; this does not sleep, so it is the one - * to use with interrupts off. - */ -int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, - size_t data_size, unsigned long type) -{ - struct mk_shared_data *ipi_data; - struct mk_ipi_data *slot; - int instance_id = instance->id; - unsigned int head, next_head, tail; - mk_phys_cpu_t target; + instance = container_of(endpoint, struct mk_instance, ipi_endpoint); + if (READ_ONCE(instance->id) != instance_id || + (peer && instance != peer)) + continue; - if (data_size > MK_MAX_DATA_SIZE) - return -EINVAL; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled || + !endpoint->tx) { + ret = -ESHUTDOWN; + goto unlock; + } - target = instance->ipi_target; - if (target == MK_PHYS_CPU_INVALID) - target = mk_cpu_set_first(instance->cpus); - if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPU to receive the IPI\n", instance_id); - return -ENODEV; - } + index = mk_ring_idx(endpoint->tx_head); + slot = &endpoint->tx->entries[index]; + /* Observe consumption before reusing this slot. */ + if (smp_load_acquire(&slot->ready)) { + ret = -ENOSPC; + goto unlock; + } - /* The ring lives as long as the instance, which is pinned here */ - ipi_data = READ_ONCE(instance->ipi_data); - if (!ipi_data) { - pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - return -ENODEV; + WRITE_ONCE(slot->sender_cpu, + arch_cpu_physical_id(raw_smp_processor_id())); + WRITE_ONCE(slot->type, type); + WRITE_ONCE(slot->data_size, data_size); + if (data_size) + memcpy(slot->buffer, data, data_size); + /* Publish the payload before the remote consumer sees ready. */ + smp_store_release(&slot->ready, 1); + endpoint->tx_head++; + target = endpoint->doorbell_cpu; + ret = 0; + +unlock: + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + break; } - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - return -ENOSPC; - } + if (!ret) + mk_arch_send_ipi(target); + rcu_read_unlock(); - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&ipi_data->ring.head, head, next_head) != head); + if (ret == -ENOSPC) + printk_deferred(KERN_WARNING + "multikernel: IPI ring full for instance %d\n", + instance_id); + return ret; +} - /* We've claimed slot 'head', now fill it */ - slot = &ipi_data->ring.entries[head]; +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + return mk_ipi_send(NULL, instance_id, data, data_size, type); +} +EXPORT_SYMBOL_GPL(multikernel_send_ipi_data); - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; +/* The caller pins the exact peer; never substitute a reused instance ID. */ +int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) +{ + return mk_ipi_send(instance, instance->id, data, data_size, type); +} - if (data && data_size > 0) - memcpy(slot->buffer, data, data_size); +static bool mk_ipi_dispatch(struct mk_ipi_data *slot) +{ + struct mk_ipi_handler *handler; + size_t data_size = READ_ONCE(slot->data_size); - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); + if (data_size > MK_MAX_DATA_SIZE) { + printk_deferred(KERN_WARNING + "multikernel: oversized IPI payload (%zu bytes)\n", + data_size); + return true; + } - mk_arch_send_ipi(target); - return 0; + list_for_each_entry_rcu(handler, &mk_handlers, node) { + if (handler->ipi_type == READ_ONCE(slot->type)) { + handler->callback(slot, handler->context); + return true; + } + } + /* Early polling must not discard messages for later initcall consumers. */ + return smp_load_acquire(&mk_handlers_ready); +} + +static bool mk_ipi_endpoint_pending(struct mk_ipi_endpoint *endpoint) +{ + struct mk_ipi_ring *ring; + unsigned long flags; + bool pending = false; + u32 index; + + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + if (!READ_ONCE(endpoint->registered)) + goto out; + if (endpoint->rx_dispatching) { + pending = true; + goto out; + } + ring = endpoint->rx; + if (ring) { + index = mk_ring_idx(endpoint->rx_tail); + /* Observe the payload after the producer publishes ready. */ + pending = smp_load_acquire(&ring->entries[index].ready); + } +out: + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return pending; } -static void mk_ipi_drain_ring(void) +static noinline void mk_ipi_drain_endpoint(struct mk_ipi_endpoint *endpoint) { struct mk_ipi_data *slot; - struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; - size_t data_size; - int messages_processed = 0; + struct mk_ipi_ring *ring; + unsigned long flags; + unsigned int drained = 0; + u32 index; - if (!mk_self || !mk_self->ipi_data) + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + if (!endpoint->registered || endpoint->rx_dispatching || !endpoint->rx) { + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); return; - - while (1) { - tail = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.head)); - - if (tail == head) + } + endpoint->rx_dispatching = true; + ring = endpoint->rx; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + + while (drained < MK_IPI_LINK_BUDGET) { + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + index = mk_ring_idx(endpoint->rx_tail); + slot = &ring->entries[index]; + /* Observe the payload after the producer publishes ready. */ + if (!smp_load_acquire(&slot->ready)) { + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); break; + } + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); - slot = &mk_self->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (!mk_ipi_dispatch(slot)) break; + /* Finish callback reads before the producer reuses this slot. */ + smp_store_release(&slot->ready, 0); - if (data_size > MK_MAX_DATA_SIZE) { - pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - continue; - } - - /* Dispatch to registered handler */ - raw_spin_lock(&mk_handlers_lock); - for (handler = mk_handlers; handler; handler = handler->next) { - if (handler->ipi_type == slot->type && handler->callback) { - mk_ipi_callback_t cb = handler->callback; - void *ctx = handler->context; - - raw_spin_unlock(&mk_handlers_lock); - cb(slot, ctx); - goto advance_tail; - } - } - raw_spin_unlock(&mk_handlers_lock); + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_tail++; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + drained++; + } -advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - messages_processed++; + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_dispatching = false; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); +} - if (messages_processed >= MK_IPI_RING_SIZE) +static unsigned int +mk_ipi_select_endpoints(struct mk_ipi_endpoint **selected) +{ + struct mk_ipi_endpoint *endpoint; + struct list_head *node; + unsigned long flags; + unsigned int count = 0; + unsigned int scanned; + + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + node = mk_ipi_cursor; + for (scanned = 0; scanned < mk_ipi_endpoint_count; scanned++) { + node = node->next; + if (node == &mk_ipi_endpoints) + node = node->next; + endpoint = list_entry(node, struct mk_ipi_endpoint, rx_node); + if (!mk_ipi_endpoint_pending(endpoint)) + continue; + selected[count++] = endpoint; + if (count == MK_IPI_ENDPOINT_BATCH) break; } + if (count) + mk_ipi_cursor = &selected[count - 1]->rx_node; + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + return count; } -/** - * multikernel_interrupt_handler - Handle the multikernel IPI - * - * This function is called when a multikernel IPI is received. - * Messages are drained here, in interrupt context. - */ -static void multikernel_interrupt_handler(void) +static noinline bool mk_ipi_any_pending(void) { + struct mk_ipi_endpoint *endpoint; + + list_for_each_entry_rcu(endpoint, &mk_ipi_endpoints, rx_node) { + if (mk_ipi_endpoint_pending(endpoint)) + return true; + } + return false; +} + +void mk_ipi_handlers_enable(void) +{ + /* Publish all handler registrations before enabling dispatch. */ + smp_store_release(&mk_handlers_ready, true); + mk_ipi_kick_local(); +} + +static void mk_ipi_drain(bool poll) +{ + struct mk_ipi_endpoint *selected[MK_IPI_ENDPOINT_BATCH]; + unsigned int count; + unsigned int i; + bool pending; unsigned long flags; - if (!mk_self || !mk_self->ipi_data) + /* Explicit polling also serves PCI requests before late initcalls. */ + if (!poll && !smp_load_acquire(&mk_handlers_ready)) return; - /* - * Drain here rather than from irq_work. We are already in interrupt - * context and every handler is safe to call from it, and irq_work - * brings a failure mode with it: the work is a single static - * instance, so if it is ever left pending - its self-IPI lost while - * the CPU was bringing its APIC up, say - every later queue attempt - * is a no-op and the ring never drains again. - * - * The ring has one consumer at a time. That used to be implied by - * the doorbell going to a single CPU; mk_ipi_poll() is a second way - * in, so it is a lock now. Waiting for it rather than giving up - * matters: a poller that is just leaving may not look again. - */ - raw_spin_lock_irqsave(&mk_drain_lock, flags); + local_irq_save(flags); + if (this_cpu_read(mk_draining)) + goto out; this_cpu_write(mk_draining, true); - mk_ipi_drain_ring(); + rcu_read_lock(); + count = mk_ipi_select_endpoints(selected); + for (i = 0; i < count; i++) + mk_ipi_drain_endpoint(selected[i]); + pending = mk_ipi_any_pending(); + rcu_read_unlock(); + this_cpu_write(mk_draining, false); - raw_spin_unlock_irqrestore(&mk_drain_lock, flags); + /* Pair with handler publication before requesting an interrupt drain. */ + if (pending && smp_load_acquire(&mk_handlers_ready)) + mk_ipi_kick_local(); +out: + local_irq_restore(flags); } -/** - * mk_ipi_in_drain - Whether this CPU is running a message handler - * - * A handler that polled the ring for an answer would wait for the drain - * lock it already holds. - */ bool mk_ipi_in_drain(void) { return this_cpu_read(mk_draining); } -/** - * mk_ipi_poll - Drain the message ring without waiting for the doorbell - * - * For a caller that waits for an answer where the doorbell interrupt - * cannot be taken: with interrupts off on the doorbell CPU, or while - * stop_machine() holds every CPU. Message handlers run in the caller's - * context, which they are fit for, as they normally run in the interrupt. - */ void mk_ipi_poll(void) { - multikernel_interrupt_handler(); + mk_ipi_drain(true); } -/** - * Generic multikernel interrupt handler - called by the IPI vector - * - * This is the function that gets called by the IPI vector handler. - */ void generic_multikernel_interrupt(void) { - multikernel_interrupt_handler(); + mk_ipi_drain(false); } -/** - * mk_has_pending_shutdown - Check if the host demanded a forcible shutdown - * - * Tests the force-halt marker the host arms before NMIing this kernel's - * CPUs. The marker used to be a message peeked in the IPI ring, but a - * ring message is consumed by the doorbell interrupt: on a responsive - * kernel the ordinary message path could eat it before the NMIs landed, - * and every NMI then found nothing to act on. The marker is host-owned - * and stays up until the host has confirmed all CPUs parked, so it - * gives the same answer no matter when each NMI arrives. - * - * A spawn kernel that has no ring cannot be asked anything, so an NMI - * no local source claims is the only voice the host has left; it is - * taken as the demand (the caller sits on the unknown-NMI chain). A - * host without a pool has no ring either, but its NMIs are its own. - * - * Safe to call from NMI context (a single read of shared memory). - * - * Returns: true if shutdown requested, false otherwise - */ bool mk_has_pending_shutdown(void) { if (!mk_self) return false; - if (!mk_self->ipi_data) return mk_manifest_phys() != 0; - return mk_force_halt_armed(); } -/** - * mk_force_halt_armed - Check the force-halt marker alone - * - * Unlike mk_has_pending_shutdown(), never true for a kernel without a - * ring, so it can be tested on NMIs that a local source may own. - */ bool mk_force_halt_armed(void) { return mk_self && mk_self->ipi_data && diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index c8fa6270e1fe6b..1201b25e5cb6f7 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -426,6 +426,7 @@ int mk_instance_destroy(struct mk_instance *instance) return -EBUSY; } + mk_ipi_endpoint_unregister(instance); list_del(&instance->list); idr_remove(&mk_instance_idr, instance->id); if (instance->kn) { @@ -601,6 +602,7 @@ void mk_kernfs_cleanup(void) /* Remove all instances */ mutex_lock(&mk_instance_mutex); list_for_each_entry_safe(instance, tmp, &mk_instance_list, list) { + mk_ipi_endpoint_unregister(instance); list_del(&instance->list); idr_remove(&mk_instance_idr, instance->id); mk_instance_put(instance); diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 307c2a3a1c6cc9..6123e584c1c0bc 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -278,6 +278,11 @@ static int mk_manifest_chosen(void *fdt, void *data) if (ret) return ret; + ret = fdt_property_u32(fdt, "multikernel,parent-id", mk_self->id); + if (ret) + return ret; + + /* The parent's self area carries force-halt and parked-CPU state. */ if (mk_self->ipi_data) { ret = fdt_property_u64(fdt, "multikernel,host-ipi-buffer", mk_self->ipi_phys);