diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index 993766b5a1d1bc..bc175413c40f47 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -342,6 +342,8 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_multikernel) */ void mk_arch_send_ipi(u64 phys_cpu) { + /* Order shared-memory publication before ringing the x2APIC doorbell. */ + weak_wrmsr_fence(); apic_icr_write(APIC_DM_FIXED | APIC_DEST_PHYSICAL | MULTIKERNEL_VECTOR, (u32)phys_cpu); } diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index f544bf342a09d0..cb40cf426d250a 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -337,6 +337,18 @@ static struct tty_driver *mktty_spawn_driver; static struct mktty_spawn_state mktty_spawn; static struct mk_ipi_handler *mktty_spawn_handler; static struct console mktty_spawn_console; + +static int mktty_spawn_parent_id(void) +{ + if (!mk_self) + return -ENODEV; + if (mk_self->id == 0) + return 0; + if (!host_instance) + return -ENODEV; + return READ_ONCE(host_instance->id); +} + static bool mktty_console_registered; static int mktty_spawn_activate(struct tty_port *port, struct tty_struct *tty) @@ -377,10 +389,13 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, { struct mktty_message *msg; size_t sent = 0, chunk; - int ret; + int parent_id, ret; if (tty->index != 0) return -ENODEV; + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + return parent_id; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) @@ -394,9 +409,9 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, msg->reserved = 0; memcpy(msg->data, buf + sent, chunk); - ret = multikernel_send_ipi_data(0, msg, - sizeof(*msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + ret = multikernel_send_ipi_data(parent_id, msg, + sizeof(*msg) - MKTTY_MAX_DATA + chunk, + MKTTY_IPI_TYPE); if (ret < 0) { kfree(msg); return sent > 0 ? sent : ret; @@ -449,8 +464,12 @@ static void mktty_spawn_console_write(struct console *con, const char *s, { unsigned long flags; size_t chunk; + int parent_id; spin_lock_irqsave(&mktty_console_lock, flags); + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + goto out; while (count > 0) { chunk = min_t(size_t, count, MKTTY_MAX_DATA); mktty_console_msg.type = MKTTY_MSG_OUTPUT; @@ -459,12 +478,13 @@ static void mktty_spawn_console_write(struct console *con, const char *s, mktty_console_msg.reserved = 0; memcpy(mktty_console_msg.data, s, chunk); - multikernel_send_ipi_data(0, &mktty_console_msg, - sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + multikernel_send_ipi_data(parent_id, &mktty_console_msg, + sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk, + MKTTY_IPI_TYPE); s += chunk; count -= chunk; } +out: spin_unlock_irqrestore(&mktty_console_lock, flags); } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 76e51d19b91381..6ec882d7df7680 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -6,7 +6,9 @@ #define _LINUX_MULTIKERNEL_H #include -#include +#include +#include +#include #include #include #include @@ -88,6 +90,7 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + u32 ready; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ @@ -96,8 +99,6 @@ struct mk_ipi_data { /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ }; @@ -109,9 +110,10 @@ struct mk_ipi_ring { /* Presence table capacity, in CPUs (see mk_cpu_rank) */ #define MK_PARKED_MAX 512 -/* Shared memory structures - per-instance design */ +/* One duplex link per parent/child pair. Each ring has one kernel producer. */ struct mk_shared_data { - struct mk_ipi_ring ring; /* IPI message ring buffer */ + struct mk_ipi_ring to_child; + struct mk_ipi_ring to_parent; /* * Force-halt marker, host-owned. Armed before the host NMIs the * instance's CPUs and cleared with the rest of this struct when @@ -133,6 +135,22 @@ struct mk_shared_data { u8 parked[MK_PARKED_MAX]; }; +struct mk_instance; + +struct mk_ipi_endpoint { + struct mk_ipi_ring *tx; + struct mk_ipi_ring *rx; + raw_spinlock_t tx_lock; + raw_spinlock_t rx_lock; + u32 tx_head; + u32 rx_tail; + mk_phys_cpu_t doorbell_cpu; + bool tx_enabled; + bool rx_dispatching; + bool registered; + struct list_head rx_node; +}; + /* Function pointer type for IPI callbacks */ typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *data, void *ctx); @@ -140,7 +158,7 @@ struct mk_ipi_handler { mk_ipi_callback_t callback; void *context; unsigned int ipi_type; /* IPI type this handler is registered for */ - struct mk_ipi_handler *next; + struct list_head node; }; /** @@ -156,6 +174,9 @@ struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, /** * multikernel_unregister_handler - Unregister a multikernel IPI callback * @handler: Handler pointer returned from multikernel_register_handler + * + * Must be called from process context and not from a multikernel IPI + * callback. */ void multikernel_unregister_handler(struct mk_ipi_handler *handler); @@ -166,8 +187,8 @@ void multikernel_unregister_handler(struct mk_ipi_handler *handler); * @data_size: Size of data * @type: User-defined type identifier * - * This function copies the data to per-CPU storage and sends an IPI - * to the target CPU. + * This function copies the data into the target instance's duplex ring and + * sends an IPI to its doorbell CPU. * * Returns 0 on success, negative error code on failure */ @@ -183,8 +204,40 @@ struct mk_instance; int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, size_t data_size, unsigned long type); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +#ifdef CONFIG_MULTIKERNEL +int mk_ipi_endpoint_register(struct mk_instance *instance, bool parent_side, + mk_phys_cpu_t doorbell_cpu); +void mk_ipi_endpoint_unregister(struct mk_instance *instance); +int mk_ipi_link_reset(struct mk_instance *instance); +void mk_ipi_handlers_enable(void); +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance); +#else +static inline int mk_ipi_endpoint_register(struct mk_instance *instance, + bool parent_side, + mk_phys_cpu_t doorbell_cpu) +{ + return -EOPNOTSUPP; +} + +static inline void mk_ipi_endpoint_unregister(struct mk_instance *instance) +{ +} + +static inline int mk_ipi_link_reset(struct mk_instance *instance) +{ + return -EOPNOTSUPP; +} + +static inline void mk_ipi_handlers_enable(void) +{ +} + +static inline struct mk_shared_data * +mk_instance_halt_data(struct mk_instance *instance) +{ + return NULL; +} +#endif /* * Multikernel Messaging System @@ -738,6 +791,13 @@ struct mk_instance { struct mk_shared_data *ipi_data; /* IPI shared memory buffer (virtual address) */ phys_addr_t ipi_phys; /* IPI buffer physical address */ u32 ipi_pages; /* IPI buffer size in pages */ + /* + * A spawn's separate mapping of its parent's force-halt area. + * Ordinary child records keep this NULL and use @ipi_data. + */ + struct mk_shared_data *halt_data; + struct mk_ipi_endpoint ipi_endpoint; + u64 ipi_start_sequence; /* Local start identity for deferred halt work */ /* * Host side only: the page the instance's boot tree is written to at * each exec. It and the IPI ring are allocated on the first load and @@ -1157,6 +1217,7 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + #endif /** diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index b7a6fc6eb6ee0d..1aaee8ddade159 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1676,6 +1676,10 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + rc = -EBUSY; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1707,6 +1711,13 @@ int multikernel_kexec_by_id(int mk_id) goto unlock; } + mk_ipi_endpoint_unregister(instance); + rc = mk_ipi_link_reset(instance); + if (rc) { + pr_err("Failed to reset instance %d IPI link: %d\n", mk_id, rc); + goto unlock; + } + /* * Booting consumes the image: the spawn kernel writes its .data and * patches its own text, so the copy in instance memory is spent once diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 7fffac2c1c25aa..688e77cee7a47c 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -2,6 +2,7 @@ /* * Copyright (C) 2025 Multikernel Technologies, Inc. All rights reserved */ +#include #include #include #include @@ -13,9 +14,12 @@ #include #include #include +#include #include #include "internal.h" +static atomic64_t mk_ipi_start_sequence = ATOMIC64_INIT(0); + static void mk_instance_return_all_cpus(struct mk_instance *instance) { if (!instance || mk_cpu_set_empty(instance->cpus)) @@ -217,6 +221,8 @@ static void mk_instance_release(struct kref *kref) pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); + WARN_ON_ONCE(READ_ONCE(instance->ipi_endpoint.registered)); + WARN_ON_ONCE(instance->ipi_endpoint.tx || instance->ipi_endpoint.rx); mk_instance_return_all_cpus(instance); mk_instance_return_pci_devices(instance); @@ -225,6 +231,8 @@ static void mk_instance_release(struct kref *kref) mk_instance_free_boot_pages(instance); mk_instance_track_dump(instance, instance->state, MK_STATE_READY); + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -278,6 +286,9 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) instance->state = MK_STATE_READY; instance->ipi_target = MK_PHYS_CPU_INVALID; + raw_spin_lock_init(&instance->ipi_endpoint.tx_lock); + raw_spin_lock_init(&instance->ipi_endpoint.rx_lock); + INIT_LIST_HEAD(&instance->ipi_endpoint.rx_node); INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); @@ -344,6 +355,8 @@ void mk_instance_free(struct mk_instance *instance) list_del(&plat_dev->list); kfree(plat_dev); } + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -361,12 +374,14 @@ void mk_instance_set_state(struct mk_instance *instance, if (old_state == state) return; + /* Retire transport readers before stopped state permits another exec. */ + if (old_state == MK_STATE_ACTIVE) { + mk_ipi_endpoint_unregister(instance); + mk_pciback_unlend(instance); + } + instance->state = state; mk_instance_track_dump(instance, old_state, state); - - /* Whatever stopped it, its functions are no longer its to reach */ - if (old_state == MK_STATE_ACTIVE) - mk_pciback_unlend(instance); pr_debug("Instance %d (%s) state: %s -> %s\n", instance->id, instance->name, mk_state_to_string(old_state), @@ -427,34 +442,26 @@ int mk_instance_start(struct mk_instance *instance, struct kimage *image, int ret; /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. + * The exec caller has confirmed park, retired old callbacks, and reset + * this link. An old HALTED work item cannot belong to this new start. */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); - - /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. - */ - mk_ipi_ring_drop_pending(); + instance->ipi_start_sequence = atomic64_inc_return(&mk_ipi_start_sequence); + ret = mk_ipi_endpoint_register(instance, true, arch_cpu_physical_id(cpu)); + if (ret) + return ret; ret = mk_pciback_lend(instance); if (ret) { pr_err("Instance %d: its PCI functions cannot be lent: %d\n", instance->id, ret); + mk_ipi_endpoint_unregister(instance); return ret; } ret = mk_arch_spawn_instance(image, instance, cpu); if (ret) { mk_pciback_unlend(instance); + mk_ipi_endpoint_unregister(instance); return ret; } @@ -1629,7 +1636,12 @@ static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) */ void __noreturn mk_halt_to_pool(void) { - mk_notify_down_and_park(0, MK_SYS_HALTED); + int parent_id; + + if (!host_instance) + panic("multikernel: spawned kernel has no parent instance"); + parent_id = READ_ONCE(host_instance->id); + mk_notify_down_and_park(parent_id, MK_SYS_HALTED); } static void mk_shutdown_work_fn(struct work_struct *work) @@ -1658,6 +1670,7 @@ static void mk_instance_settle_halted(struct mk_instance *instance) struct mk_halted_work { struct work_struct work; int instance_id; + u64 received_start_sequence; }; static void mk_halted_work_fn(struct work_struct *work) @@ -1666,14 +1679,19 @@ static void mk_halted_work_fn(struct work_struct *work) container_of(work, struct mk_halted_work, work); struct mk_instance *instance; + /* Exec and the synchronous halt paths already hold this mutex. */ + mutex_lock(&system_transition_mutex); instance = mk_instance_find(aw->instance_id); if (instance) { - mk_instance_settle_halted(instance); + if (instance->state == MK_STATE_ACTIVE && + instance->ipi_start_sequence <= aw->received_start_sequence) + mk_instance_settle_halted(instance); mk_instance_put(instance); } else { pr_warn("Shutdown ACK from unknown instance %d\n", aw->instance_id); } + mutex_unlock(&system_transition_mutex); kfree(aw); } @@ -1738,6 +1756,8 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, INIT_WORK(&aw->work, mk_halted_work_fn); aw->instance_id = ack->resource_id; + /* Exec's unregister grace puts this snapshot before the next start. */ + aw->received_start_sequence = atomic64_read(&mk_ipi_start_sequence); schedule_work(&aw->work); break; } @@ -1875,7 +1895,7 @@ EXPORT_SYMBOL_GPL(mk_cpu_rank); static int mk_fence_missing(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing = 0; @@ -1900,7 +1920,7 @@ static int mk_fence_missing(struct mk_instance *instance, static int mk_confirm_fenced(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing, ret; @@ -2007,8 +2027,10 @@ int multikernel_force_halt_by_id(int mk_id) */ if (instance == host_instance) { ret = mk_confirm_fenced(instance, targets); - if (!ret) + if (!ret) { + mk_ipi_endpoint_unregister(instance); mk_pcifront_host_fenced(); + } } else { mk_instance_settle_halted(instance); } @@ -2018,6 +2040,8 @@ int multikernel_force_halt_by_id(int mk_id) return ret; } +static bool multikernel_initialized __initdata; + static int __init multikernel_init(void) { int ret; @@ -2052,9 +2076,21 @@ static int __init multikernel_init(void) return ret; } + multikernel_initialized = true; + pr_info("Multikernel support initialized\n"); return 0; } +static int __init multikernel_ipi_enable(void) +{ + if (!multikernel_initialized) + return 0; + + mk_ipi_handlers_enable(); + return 0; +} + /* Initialize multikernel after core kernel subsystems are ready */ subsys_initcall(multikernel_init); +late_initcall(multikernel_ipi_enable); diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 8503733372da18..25af5b7ed7afb6 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -297,6 +297,11 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) return 0; } ipi_size = (size_t)ipi_pages << PAGE_SHIFT; + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small: %zu < %zu\n", ipi_size, + sizeof(struct mk_shared_data)); + return -EINVAL; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -316,38 +321,58 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) static int __init mk_restore_host_instance(void) { struct mk_instance *hi; - phys_addr_t host_ipi_phys; - u32 host_ipi_pages; - size_t host_ipi_size; + phys_addr_t halt_phys; + u32 halt_pages; + u32 parent_id; + size_t halt_size; u64 host_ipi_cpu; + int ret; - if (!mk_chosen_ring("host-ipi", &host_ipi_phys, &host_ipi_pages) || + if (!mk_self || !mk_self->ipi_data || + of_property_read_u32(of_chosen, "multikernel,parent-id", &parent_id) || of_property_read_u64(of_chosen, "multikernel,host-ipi-cpu", - &host_ipi_cpu)) { - pr_warn("No host IPI buffer in the boot tree (spawn won't be able to send to host)\n"); + &host_ipi_cpu) || + !mk_chosen_ring("host-ipi", &halt_phys, &halt_pages)) { + pr_warn("Incomplete parent IPI data in the boot tree\n"); return -ENOENT; } - host_ipi_size = (size_t)host_ipi_pages << PAGE_SHIFT; + if (parent_id > INT_MAX || parent_id == mk_self->id || + host_ipi_cpu == MK_PHYS_CPU_INVALID) { + pr_err("Invalid parent endpoint: id=%u, doorbell=%llu\n", + parent_id, host_ipi_cpu); + return -EINVAL; + } + + halt_size = (size_t)halt_pages << PAGE_SHIFT; + if (halt_size < sizeof(struct mk_shared_data) || + halt_phys == mk_self->ipi_phys) { + pr_err("Invalid parent force-halt area: phys=0x%llx, pages=%u\n", + (unsigned long long)halt_phys, halt_pages); + return -EINVAL; + } - hi = mk_instance_alloc(0, "host"); + hi = mk_instance_alloc(parent_id, "host"); if (!hi) return -ENOMEM; hi->ipi_target = host_ipi_cpu; - - hi->ipi_data = memremap(host_ipi_phys, host_ipi_size, MEMREMAP_WB); - if (!hi->ipi_data) { - pr_err("Failed to map host IPI buffer at 0x%llx\n", - (unsigned long long)host_ipi_phys); + hi->ipi_data = mk_self->ipi_data; + hi->ipi_phys = mk_self->ipi_phys; + hi->ipi_pages = mk_self->ipi_pages; + hi->halt_data = memremap(halt_phys, halt_size, MEMREMAP_WB); + if (!hi->halt_data) { + pr_err("Failed to map parent force-halt area at 0x%llx\n", + (unsigned long long)halt_phys); + ret = -ENOMEM; goto err_free; } - hi->ipi_phys = host_ipi_phys; - hi->ipi_pages = host_ipi_pages; - if (mk_instance_publish(hi)) { - memunmap(hi->ipi_data); + ret = mk_ipi_endpoint_register(hi, false, host_ipi_cpu); + if (ret) goto err_free; - } + ret = mk_instance_publish(hi); + if (ret) + goto err_endpoint; /* The host is running, or this kernel would not be */ mk_instance_set_state(hi, MK_STATE_ACTIVE); @@ -362,15 +387,16 @@ static int __init mk_restore_host_instance(void) host_instance = hi; - pr_info("Restored host IPI buffer: phys=0x%llx, pages=%u\n", - (unsigned long long)host_ipi_phys, host_ipi_pages); - pr_info("Registered host instance (ID 0) for spawn→host communication\n"); + pr_info("Registered parent instance %u on the duplex IPI link\n", + parent_id); return 0; +err_endpoint: + mk_ipi_endpoint_unregister(hi); err_free: mk_instance_free(hi); - return -ENOMEM; + return ret; } /** diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 4a6a44f94c88dc..f1db8a9be167ef 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -2,423 +2,535 @@ /* * Copyright (C) 2025 Multikernel Technologies, Inc. All rights reserved */ -#include #include -#include -#include +#include #include -#include #include -#include -#include -#include +#include +#include +#include #include "internal.h" -/* Callback management */ -static struct mk_ipi_handler *mk_handlers; -static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); +#define MK_IPI_ENDPOINT_BATCH 8 +#define MK_IPI_LINK_BUDGET 8 + +static LIST_HEAD(mk_handlers); +static DEFINE_RAW_SPINLOCK(mk_handlers_lock); -static DEFINE_RAW_SPINLOCK(mk_drain_lock); +static LIST_HEAD(mk_ipi_endpoints); +static DEFINE_RAW_SPINLOCK(mk_ipi_endpoints_lock); +static struct list_head *mk_ipi_cursor = &mk_ipi_endpoints; +static unsigned int mk_ipi_endpoint_count; +static bool mk_handlers_ready; static DEFINE_PER_CPU(bool, mk_draining); -static void mk_ipi_drain_ring(void); +static inline unsigned int mk_ring_idx(unsigned int index) +{ + return index & (MK_IPI_RING_SIZE - 1); +} -/* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. +static noinline void mk_ipi_kick_local(void) +{ + mk_arch_send_ipi(arch_cpu_physical_id(0)); +} + +static void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + unsigned int i; + + for (i = 0; i < MK_IPI_RING_SIZE; i++) + WRITE_ONCE(ring->entries[i].ready, 0); +} + +static void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->to_child); + mk_ipi_ring_reset(&shared->to_parent); + WRITE_ONCE(shared->force_halt, 0); + memset(shared->parked, 0, sizeof(shared->parked)); +} + +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance) +{ + struct mk_shared_data *shared = READ_ONCE(instance->halt_data); + + return shared ? shared : READ_ONCE(instance->ipi_data); +} + +/** + * mk_ipi_endpoint_register - Publish one side of a duplex IPI link + * @instance: Record for the peer at the other end of the link + * @parent_side: True in the parent kernel, false in its child + * @doorbell_cpu: Stable physical CPU to notify in the peer + * + * Ordinary detach and re-registration preserve the local ring indices. */ -static inline unsigned int mk_ring_idx(unsigned int i) +int mk_ipi_endpoint_register(struct mk_instance *instance, bool parent_side, + mk_phys_cpu_t doorbell_cpu) { - return i & (MK_IPI_RING_SIZE - 1); + struct mk_ipi_endpoint *endpoint; + struct mk_shared_data *shared; + unsigned long flags; + + if (!instance || doorbell_cpu == MK_PHYS_CPU_INVALID) + return -EINVAL; + shared = READ_ONCE(instance->ipi_data); + if (!shared) + return -ENODEV; + + endpoint = &instance->ipi_endpoint; + if (WARN_ON_ONCE(READ_ONCE(endpoint->registered))) + return -EALREADY; + + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx = parent_side ? &shared->to_child : &shared->to_parent; + endpoint->doorbell_cpu = doorbell_cpu; + endpoint->tx_enabled = true; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx = parent_side ? &shared->to_parent : &shared->to_child; + endpoint->rx_dispatching = false; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + INIT_LIST_HEAD(&endpoint->rx_node); + list_add_tail_rcu(&endpoint->rx_node, &mk_ipi_endpoints); + WRITE_ONCE(endpoint->registered, true); + mk_ipi_endpoint_count++; + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + + /* Pairs with the handler publication in mk_ipi_handlers_enable(). */ + if (smp_load_acquire(&mk_handlers_ready)) + mk_ipi_kick_local(); + return 0; } /** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring + * mk_ipi_endpoint_unregister - Stop and unpublish a duplex endpoint + * @instance: Peer record whose endpoint is being detached * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. + * Must be called from process context and never from an IPI callback. The + * function waits for senders and hardirq callbacks that already found the + * endpoint before returning. Local ring indices are preserved for reattach. */ -void mk_ipi_ring_drop_pending(void) +void mk_ipi_endpoint_unregister(struct mk_instance *instance) { - struct mk_ipi_ring *ring; - unsigned int head, tail; + struct mk_ipi_endpoint *endpoint; + unsigned long flags; + bool removed = false; - if (!mk_self || !mk_self->ipi_data) + if (!instance) return; + endpoint = &instance->ipi_endpoint; + + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx_enabled = false; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + if (endpoint->registered) { + if (mk_ipi_cursor == &endpoint->rx_node) + mk_ipi_cursor = endpoint->rx_node.prev; + list_del_rcu(&endpoint->rx_node); + WRITE_ONCE(endpoint->registered, false); + mk_ipi_endpoint_count--; + removed = true; + } + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + + if (!removed) + return; + + synchronize_rcu(); + + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx = NULL; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx = NULL; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + INIT_LIST_HEAD(&endpoint->rx_node); +} - ring = &mk_self->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); +/** + * mk_ipi_link_reset - Start a confirmed-stopped peer link at generation zero + * @instance: Peer whose CPUs are parked and endpoint is detached + * + * Returns 0 on success or a negative error when the reset preconditions are + * not met. + */ +int mk_ipi_link_reset(struct mk_instance *instance) +{ + struct mk_ipi_endpoint *endpoint; + struct mk_shared_data *shared; + unsigned long flags; - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; + if (!instance) + return -EINVAL; + endpoint = &instance->ipi_endpoint; + shared = READ_ONCE(instance->ipi_data); + if (!shared) + return -ENODEV; - atomic_set(&ring->tail, head); + if (WARN_ON_ONCE(READ_ONCE(endpoint->registered))) + return -EBUSY; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + raw_spin_lock(&endpoint->rx_lock); + mk_shared_data_reset(shared); + endpoint->tx_head = 0; + endpoint->rx_tail = 0; + endpoint->rx_dispatching = false; + raw_spin_unlock(&endpoint->rx_lock); + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + return 0; } /** - * multikernel_register_handler - Register a callback for multikernel IPI - * @callback: Function to call when IPI is received - * @ctx: Context pointer passed to the callback - * @ipi_type: IPI type this handler should process + * multikernel_register_handler - Register a hardirq IPI callback + * @callback: Atomic callback invoked directly from the IPI handler + * @ctx: Opaque callback context + * @ipi_type: Message type delivered to the callback * - * Returns pointer to handler on success, NULL on failure + * The callback must not sleep and must not unregister itself. */ -struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, unsigned int ipi_type) +struct mk_ipi_handler * +multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, + unsigned int ipi_type) { struct mk_ipi_handler *handler; unsigned long flags; if (!callback) return NULL; - handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler) return NULL; - handler->callback = callback; handler->context = ctx; handler->ipi_type = ipi_type; raw_spin_lock_irqsave(&mk_handlers_lock, flags); - handler->next = mk_handlers; - mk_handlers = handler; + list_add_rcu(&handler->node, &mk_handlers); raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - return handler; } EXPORT_SYMBOL(multikernel_register_handler); /** - * multikernel_unregister_handler - Unregister a multikernel IPI callback - * @handler: Handler pointer returned from multikernel_register_handler + * multikernel_unregister_handler - Unregister an IPI callback + * @handler: Handler returned by multikernel_register_handler() + * + * Must be called from process context and outside the callback. It waits for + * hardirq readers that may still be using the callback and its context. */ void multikernel_unregister_handler(struct mk_ipi_handler *handler) { - struct mk_ipi_handler **pp, *p; unsigned long flags; if (!handler) return; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); - pp = &mk_handlers; - while ((p = *pp) != NULL) { - if (p == handler) { - *pp = p->next; - break; - } - pp = &p->next; - } + list_del_rcu(&handler->node); raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - - kfree(p); + synchronize_rcu(); + kfree(handler); } EXPORT_SYMBOL(multikernel_unregister_handler); -/** - * mk_arm_force_halt - Post the force-halt marker for an instance - * @instance: Instance about to be NMIed - * - * The instance's CPUs test the marker from their NMI handlers, so it - * must be armed before the NMIs are sent. It stays armed until the - * kexec path has confirmed every CPU parked and wipes the shared area - * for the next run, which is what makes the NMI rescue idempotent: a - * repeat force halt still reaches CPUs an earlier one missed. - * - * Returns 0 on success, -ENODEV if the instance has no shared IPI area. - */ int mk_arm_force_halt(struct mk_instance *instance) { - struct mk_shared_data *ipi_data = READ_ONCE(instance->ipi_data); + struct mk_shared_data *shared = mk_instance_halt_data(instance); - if (!ipi_data) + if (!shared) return -ENODEV; - - WRITE_ONCE(ipi_data->force_halt, 1); - /* The marker must be visible before the NMIs that test it */ + WRITE_ONCE(shared->force_halt, 1); smp_wmb(); return 0; } /** - * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID - * @data: Pointer to data to send - * @data_size: Size of data - * @type: User-defined type identifier + * mk_ipi_send - Queue data on a peer's duplex link + * @peer: Referenced peer, or NULL for lookup by ID + * @instance_id: Peer instance ID + * @data: Payload, or NULL for a zero-length message + * @data_size: Payload size + * @type: Message type * - * This function enqueues data into the target instance's IPI ring buffer - * and sends an IPI to notify the target CPU. - * - * Returns 0 on success, negative error code on failure + * Safe in IRQ-disabled and hardirq contexts. Endpoint lookup is RCU-protected + * and publication is serialized per link direction. */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +static int mk_ipi_send(struct mk_instance *peer, int instance_id, void *data, + size_t data_size, unsigned long type) { - struct mk_instance *instance = mk_instance_find(instance_id); - int ret; + struct mk_ipi_endpoint *endpoint; + struct mk_ipi_data *slot; + mk_phys_cpu_t target = MK_PHYS_CPU_INVALID; + unsigned long flags; + int ret = -ENODEV; + u32 index; - if (!instance) + if (data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; - ret = multikernel_send_ipi_data_to(instance, data, data_size, type); - mk_instance_put(instance); - return ret; -} + rcu_read_lock(); + list_for_each_entry_rcu(endpoint, &mk_ipi_endpoints, rx_node) { + struct mk_instance *instance; -/* - * The same for a caller that holds a reference to @instance. Looking an - * instance up by ID takes a mutex; this does not sleep, so it is the one - * to use with interrupts off. - */ -int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, - size_t data_size, unsigned long type) -{ - struct mk_shared_data *ipi_data; - struct mk_ipi_data *slot; - int instance_id = instance->id; - unsigned int head, next_head, tail; - mk_phys_cpu_t target; + instance = container_of(endpoint, struct mk_instance, ipi_endpoint); + if (READ_ONCE(instance->id) != instance_id || + (peer && instance != peer)) + continue; - if (data_size > MK_MAX_DATA_SIZE) - return -EINVAL; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled || + !endpoint->tx) { + ret = -ESHUTDOWN; + goto unlock; + } - target = instance->ipi_target; - if (target == MK_PHYS_CPU_INVALID) - target = mk_cpu_set_first(instance->cpus); - if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPU to receive the IPI\n", instance_id); - return -ENODEV; - } + index = mk_ring_idx(endpoint->tx_head); + slot = &endpoint->tx->entries[index]; + /* Observe consumption before reusing this slot. */ + if (smp_load_acquire(&slot->ready)) { + ret = -ENOSPC; + goto unlock; + } - /* The ring lives as long as the instance, which is pinned here */ - ipi_data = READ_ONCE(instance->ipi_data); - if (!ipi_data) { - pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - return -ENODEV; + WRITE_ONCE(slot->sender_cpu, + arch_cpu_physical_id(raw_smp_processor_id())); + WRITE_ONCE(slot->type, type); + WRITE_ONCE(slot->data_size, data_size); + if (data_size) + memcpy(slot->buffer, data, data_size); + /* Publish the payload before the remote consumer sees ready. */ + smp_store_release(&slot->ready, 1); + endpoint->tx_head++; + target = endpoint->doorbell_cpu; + ret = 0; + +unlock: + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + break; } - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - return -ENOSPC; - } + if (!ret) + mk_arch_send_ipi(target); + rcu_read_unlock(); - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&ipi_data->ring.head, head, next_head) != head); + if (ret == -ENOSPC) + printk_deferred(KERN_WARNING + "multikernel: IPI ring full for instance %d\n", + instance_id); + return ret; +} - /* We've claimed slot 'head', now fill it */ - slot = &ipi_data->ring.entries[head]; +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + return mk_ipi_send(NULL, instance_id, data, data_size, type); +} +EXPORT_SYMBOL_GPL(multikernel_send_ipi_data); - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; +/* The caller pins the exact peer; never substitute a reused instance ID. */ +int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) +{ + return mk_ipi_send(instance, instance->id, data, data_size, type); +} - if (data && data_size > 0) - memcpy(slot->buffer, data, data_size); +static bool mk_ipi_dispatch(struct mk_ipi_data *slot) +{ + struct mk_ipi_handler *handler; + size_t data_size = READ_ONCE(slot->data_size); - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); + if (data_size > MK_MAX_DATA_SIZE) { + printk_deferred(KERN_WARNING + "multikernel: oversized IPI payload (%zu bytes)\n", + data_size); + return true; + } - mk_arch_send_ipi(target); - return 0; + list_for_each_entry_rcu(handler, &mk_handlers, node) { + if (handler->ipi_type == READ_ONCE(slot->type)) { + handler->callback(slot, handler->context); + return true; + } + } + /* Early polling must not discard messages for later initcall consumers. */ + return smp_load_acquire(&mk_handlers_ready); +} + +static bool mk_ipi_endpoint_pending(struct mk_ipi_endpoint *endpoint) +{ + struct mk_ipi_ring *ring; + unsigned long flags; + bool pending = false; + u32 index; + + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + if (!READ_ONCE(endpoint->registered)) + goto out; + if (endpoint->rx_dispatching) { + pending = true; + goto out; + } + ring = endpoint->rx; + if (ring) { + index = mk_ring_idx(endpoint->rx_tail); + /* Observe the payload after the producer publishes ready. */ + pending = smp_load_acquire(&ring->entries[index].ready); + } +out: + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return pending; } -static void mk_ipi_drain_ring(void) +static noinline void mk_ipi_drain_endpoint(struct mk_ipi_endpoint *endpoint) { struct mk_ipi_data *slot; - struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; - size_t data_size; - int messages_processed = 0; + struct mk_ipi_ring *ring; + unsigned long flags; + unsigned int drained = 0; + u32 index; - if (!mk_self || !mk_self->ipi_data) + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + if (!endpoint->registered || endpoint->rx_dispatching || !endpoint->rx) { + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); return; - - while (1) { - tail = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.head)); - - if (tail == head) + } + endpoint->rx_dispatching = true; + ring = endpoint->rx; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + + while (drained < MK_IPI_LINK_BUDGET) { + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + index = mk_ring_idx(endpoint->rx_tail); + slot = &ring->entries[index]; + /* Observe the payload after the producer publishes ready. */ + if (!smp_load_acquire(&slot->ready)) { + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); break; + } + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); - slot = &mk_self->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (!mk_ipi_dispatch(slot)) break; + /* Finish callback reads before the producer reuses this slot. */ + smp_store_release(&slot->ready, 0); - if (data_size > MK_MAX_DATA_SIZE) { - pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - continue; - } - - /* Dispatch to registered handler */ - raw_spin_lock(&mk_handlers_lock); - for (handler = mk_handlers; handler; handler = handler->next) { - if (handler->ipi_type == slot->type && handler->callback) { - mk_ipi_callback_t cb = handler->callback; - void *ctx = handler->context; - - raw_spin_unlock(&mk_handlers_lock); - cb(slot, ctx); - goto advance_tail; - } - } - raw_spin_unlock(&mk_handlers_lock); + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_tail++; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + drained++; + } -advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - messages_processed++; + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_dispatching = false; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); +} - if (messages_processed >= MK_IPI_RING_SIZE) +static unsigned int +mk_ipi_select_endpoints(struct mk_ipi_endpoint **selected) +{ + struct mk_ipi_endpoint *endpoint; + struct list_head *node; + unsigned long flags; + unsigned int count = 0; + unsigned int scanned; + + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + node = mk_ipi_cursor; + for (scanned = 0; scanned < mk_ipi_endpoint_count; scanned++) { + node = node->next; + if (node == &mk_ipi_endpoints) + node = node->next; + endpoint = list_entry(node, struct mk_ipi_endpoint, rx_node); + if (!mk_ipi_endpoint_pending(endpoint)) + continue; + selected[count++] = endpoint; + if (count == MK_IPI_ENDPOINT_BATCH) break; } + if (count) + mk_ipi_cursor = &selected[count - 1]->rx_node; + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + return count; } -/** - * multikernel_interrupt_handler - Handle the multikernel IPI - * - * This function is called when a multikernel IPI is received. - * Messages are drained here, in interrupt context. - */ -static void multikernel_interrupt_handler(void) +static noinline bool mk_ipi_any_pending(void) { + struct mk_ipi_endpoint *endpoint; + + list_for_each_entry_rcu(endpoint, &mk_ipi_endpoints, rx_node) { + if (mk_ipi_endpoint_pending(endpoint)) + return true; + } + return false; +} + +void mk_ipi_handlers_enable(void) +{ + /* Publish all handler registrations before enabling dispatch. */ + smp_store_release(&mk_handlers_ready, true); + mk_ipi_kick_local(); +} + +static void mk_ipi_drain(bool poll) +{ + struct mk_ipi_endpoint *selected[MK_IPI_ENDPOINT_BATCH]; + unsigned int count; + unsigned int i; + bool pending; unsigned long flags; - if (!mk_self || !mk_self->ipi_data) + /* Explicit polling also serves PCI requests before late initcalls. */ + if (!poll && !smp_load_acquire(&mk_handlers_ready)) return; - /* - * Drain here rather than from irq_work. We are already in interrupt - * context and every handler is safe to call from it, and irq_work - * brings a failure mode with it: the work is a single static - * instance, so if it is ever left pending - its self-IPI lost while - * the CPU was bringing its APIC up, say - every later queue attempt - * is a no-op and the ring never drains again. - * - * The ring has one consumer at a time. That used to be implied by - * the doorbell going to a single CPU; mk_ipi_poll() is a second way - * in, so it is a lock now. Waiting for it rather than giving up - * matters: a poller that is just leaving may not look again. - */ - raw_spin_lock_irqsave(&mk_drain_lock, flags); + local_irq_save(flags); + if (this_cpu_read(mk_draining)) + goto out; this_cpu_write(mk_draining, true); - mk_ipi_drain_ring(); + rcu_read_lock(); + count = mk_ipi_select_endpoints(selected); + for (i = 0; i < count; i++) + mk_ipi_drain_endpoint(selected[i]); + pending = mk_ipi_any_pending(); + rcu_read_unlock(); + this_cpu_write(mk_draining, false); - raw_spin_unlock_irqrestore(&mk_drain_lock, flags); + /* Pair with handler publication before requesting an interrupt drain. */ + if (pending && smp_load_acquire(&mk_handlers_ready)) + mk_ipi_kick_local(); +out: + local_irq_restore(flags); } -/** - * mk_ipi_in_drain - Whether this CPU is running a message handler - * - * A handler that polled the ring for an answer would wait for the drain - * lock it already holds. - */ bool mk_ipi_in_drain(void) { return this_cpu_read(mk_draining); } -/** - * mk_ipi_poll - Drain the message ring without waiting for the doorbell - * - * For a caller that waits for an answer where the doorbell interrupt - * cannot be taken: with interrupts off on the doorbell CPU, or while - * stop_machine() holds every CPU. Message handlers run in the caller's - * context, which they are fit for, as they normally run in the interrupt. - */ void mk_ipi_poll(void) { - multikernel_interrupt_handler(); + mk_ipi_drain(true); } -/** - * Generic multikernel interrupt handler - called by the IPI vector - * - * This is the function that gets called by the IPI vector handler. - */ void generic_multikernel_interrupt(void) { - multikernel_interrupt_handler(); + mk_ipi_drain(false); } -/** - * mk_has_pending_shutdown - Check if the host demanded a forcible shutdown - * - * Tests the force-halt marker the host arms before NMIing this kernel's - * CPUs. The marker used to be a message peeked in the IPI ring, but a - * ring message is consumed by the doorbell interrupt: on a responsive - * kernel the ordinary message path could eat it before the NMIs landed, - * and every NMI then found nothing to act on. The marker is host-owned - * and stays up until the host has confirmed all CPUs parked, so it - * gives the same answer no matter when each NMI arrives. - * - * A spawn kernel that has no ring cannot be asked anything, so an NMI - * no local source claims is the only voice the host has left; it is - * taken as the demand (the caller sits on the unknown-NMI chain). A - * host without a pool has no ring either, but its NMIs are its own. - * - * Safe to call from NMI context (a single read of shared memory). - * - * Returns: true if shutdown requested, false otherwise - */ bool mk_has_pending_shutdown(void) { if (!mk_self) return false; - if (!mk_self->ipi_data) return mk_manifest_phys() != 0; - return mk_force_halt_armed(); } -/** - * mk_force_halt_armed - Check the force-halt marker alone - * - * Unlike mk_has_pending_shutdown(), never true for a kernel without a - * ring, so it can be tested on NMIs that a local source may own. - */ bool mk_force_halt_armed(void) { return mk_self && mk_self->ipi_data && diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index c8fa6270e1fe6b..1201b25e5cb6f7 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -426,6 +426,7 @@ int mk_instance_destroy(struct mk_instance *instance) return -EBUSY; } + mk_ipi_endpoint_unregister(instance); list_del(&instance->list); idr_remove(&mk_instance_idr, instance->id); if (instance->kn) { @@ -601,6 +602,7 @@ void mk_kernfs_cleanup(void) /* Remove all instances */ mutex_lock(&mk_instance_mutex); list_for_each_entry_safe(instance, tmp, &mk_instance_list, list) { + mk_ipi_endpoint_unregister(instance); list_del(&instance->list); idr_remove(&mk_instance_idr, instance->id); mk_instance_put(instance); diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 307c2a3a1c6cc9..6123e584c1c0bc 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -278,6 +278,11 @@ static int mk_manifest_chosen(void *fdt, void *data) if (ret) return ret; + ret = fdt_property_u32(fdt, "multikernel,parent-id", mk_self->id); + if (ret) + return ret; + + /* The parent's self area carries force-halt and parked-CPU state. */ if (mk_self->ipi_data) { ret = fdt_property_u64(fdt, "multikernel,host-ipi-buffer", mk_self->ipi_phys);