From 73b6a93dcf0c4a8dbf2285422d6fdce5fde24819 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 16:33:56 -0700 Subject: [PATCH 01/30] arm64/multikernel: Add the Kconfig symbol, arch header and stubs Multikernel only builds on x86 so far. Give arm64 the pieces the generic core needs from an architecture, so that the port can grow one function at a time in a tree that links. arm64 selects ARCH_SUPPORTS_MULTIKERNEL when it has PSCI and kexec_file: pool CPUs will be held by firmware in the PSCI OFF state and started with CPU_ON, and instances will be loaded as Image files. It does not select ARCH_HAS_MK_HOST_PARK, as there is no software park area to keep in pool memory. asm/multikernel.h names CPUs by MPIDR through the logical map and sizes the control block for the boot device tree alone: without a trampoline, a park page or identity page tables there is nothing else to carve out of instance memory. Every function of the architecture interface gets a stub. Spawning fails with -EOPNOTSUPP, so CONFIG_MULTIKERNEL=y is harmless on arm64 until the follow-up changes fill these in. Signed-off-by: Cong Wang --- arch/arm64/Kbuild | 1 + arch/arm64/Kconfig | 3 ++ arch/arm64/include/asm/multikernel.h | 38 +++++++++++++++++ arch/arm64/multikernel/Makefile | 6 +++ arch/arm64/multikernel/spawn.c | 63 ++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+) create mode 100644 arch/arm64/include/asm/multikernel.h create mode 100644 arch/arm64/multikernel/Makefile create mode 100644 arch/arm64/multikernel/spawn.c diff --git a/arch/arm64/Kbuild b/arch/arm64/Kbuild index d876bc0e542110..39e3e68cf3d686 100644 --- a/arch/arm64/Kbuild +++ b/arch/arm64/Kbuild @@ -8,6 +8,7 @@ obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_XEN) += xen/ obj-$(subst m,y,$(CONFIG_HYPERV)) += hyperv/ obj-$(CONFIG_CRYPTO) += crypto/ +obj-$(CONFIG_MULTIKERNEL) += multikernel/ # for cleaning subdir- += boot diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 9ea19b74b6c3c9..f8d898f7b15e65 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1622,6 +1622,9 @@ config ARCH_DEFAULT_KEXEC_IMAGE_VERIFY_SIG config ARCH_SUPPORTS_KEXEC_HANDOVER def_bool y +config ARCH_SUPPORTS_MULTIKERNEL + def_bool ARM_PSCI_FW && KEXEC_FILE + config ARCH_SUPPORTS_CRASH_DUMP def_bool y diff --git a/arch/arm64/include/asm/multikernel.h b/arch/arm64/include/asm/multikernel.h new file mode 100644 index 00000000000000..50db849ed7ef79 --- /dev/null +++ b/arch/arm64/include/asm/multikernel.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_ARM64_MULTIKERNEL_H +#define _ASM_ARM64_MULTIKERNEL_H + +#include +#include +#include +#include + +/* + * Pool CPUs are held by firmware in the PSCI OFF state and started with + * CPU_ON, so an instance needs no trampoline, park page or page tables: + * only the host's record of what it handed to CPU_ON. + */ +struct mk_instance_arch { + void *dtb; /* Boot device tree, in the control block */ + phys_addr_t dtb_phys; +}; + +struct mk_pool_arch { +}; + +/* Physical CPU IDs are MPIDR affinity values, as in the logical map */ +static inline u64 arch_cpu_physical_id(int cpu) +{ + return cpu_logical_map(cpu); +} + +static inline int arch_cpu_from_physical_id(u64 phys_id) +{ + return get_logical_index(phys_id); +} + +/* Control block: the boot device tree, reserved from the spawn's allocator */ +#define MK_BOOT_DTB_SIZE SZ_64K +#define MK_CTRL_BLOCK_SIZE ALIGN(MK_BOOT_DTB_SIZE, PAGE_SIZE) + +#endif /* _ASM_ARM64_MULTIKERNEL_H */ diff --git a/arch/arm64/multikernel/Makefile b/arch/arm64/multikernel/Makefile new file mode 100644 index 00000000000000..42c5f0565269ee --- /dev/null +++ b/arch/arm64/multikernel/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for multikernel spawn support +# + +obj-y += spawn.o diff --git a/arch/arm64/multikernel/spawn.c b/arch/arm64/multikernel/spawn.c new file mode 100644 index 00000000000000..474ba4cfb17056 --- /dev/null +++ b/arch/arm64/multikernel/spawn.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arm64 multikernel architecture interface. + * + * Nothing is implemented yet: every entry point refuses, so an arm64 + * kernel builds with CONFIG_MULTIKERNEL=y but cannot spawn. + */ + +#include +#include +#include +#include +#include +#include + +void mk_arch_send_ipi(mk_phys_cpu_t phys_cpu) +{ +} + +void __init mk_arch_register_cpu(mk_phys_cpu_t phys_id) +{ +} + +void __noreturn mk_enter_pool_state(void *info) +{ + local_irq_disable(); + cpu_park_loop(); +} + +void mk_force_stop_cpu(mk_phys_cpu_t phys_cpu) +{ +} + +int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, + int cpu) +{ + return -EOPNOTSUPP; +} + +int mk_arch_release_instance(struct mk_instance *instance) +{ + return 0; +} + +int mk_arch_confirm_parked(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) +{ + return -EOPNOTSUPP; +} + +int mk_repark_instance_to_host(struct mk_instance *instance) +{ + return 0; +} + +int mk_repark_cpu_to_instance(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) +{ + return 0; +} + +int mk_repark_cpu_to_host(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) +{ + return 0; +} From bbb3ec5977620685519936fdef14de7c377478a9 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 16:41:54 -0700 Subject: [PATCH 02/30] firmware/psci: Let a caller pass the CPU_ON context ID CPU_ON hands its third argument, the context ID, to the started CPU in x0. Secondary boot has no use for it and the driver hardcodes 0. Multikernel starts a CPU on a different kernel image, straight at that Image's entry point, and the arm64 boot protocol wants the device tree address in x0 there. CPU_ON can deliver it without any trampoline if the caller gets to choose the context ID. Add psci_cpu_on_context() for that, using whichever CPU_ON function ID the firmware was probed with. psci_ops.cpu_on() is unchanged. Signed-off-by: Cong Wang --- drivers/firmware/psci/psci.c | 32 ++++++++++++++++++++++++++++---- include/linux/psci.h | 3 +++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c index 38ca190d4a22d6..69f129178bd7e7 100644 --- a/drivers/firmware/psci/psci.c +++ b/drivers/firmware/psci/psci.c @@ -214,22 +214,46 @@ static int psci_0_2_cpu_off(u32 state) return __psci_cpu_off(PSCI_0_2_FN_CPU_OFF, state); } -static int __psci_cpu_on(u32 fn, unsigned long cpuid, unsigned long entry_point) +static int __psci_cpu_on(u32 fn, unsigned long cpuid, unsigned long entry_point, + unsigned long context_id) { int err; - err = invoke_psci_fn(fn, cpuid, entry_point, 0); + err = invoke_psci_fn(fn, cpuid, entry_point, context_id); return psci_to_linux_errno(err); } static int psci_0_1_cpu_on(unsigned long cpuid, unsigned long entry_point) { - return __psci_cpu_on(psci_0_1_function_ids.cpu_on, cpuid, entry_point); + return __psci_cpu_on(psci_0_1_function_ids.cpu_on, cpuid, entry_point, 0); } static int psci_0_2_cpu_on(unsigned long cpuid, unsigned long entry_point) { - return __psci_cpu_on(PSCI_FN_NATIVE(0_2, CPU_ON), cpuid, entry_point); + return __psci_cpu_on(PSCI_FN_NATIVE(0_2, CPU_ON), cpuid, entry_point, 0); +} + +/** + * psci_cpu_on_context - CPU_ON with a caller-chosen context ID + * @cpuid: MPIDR of the CPU to start + * @entry_point: physical address it starts executing at + * @context_id: value it finds in x0 (r0) there + * + * The secondary boot path has no use for the context ID and passes 0. + * Starting a CPU on a different kernel image does: the boot protocol + * wants the device tree address in that register. + */ +int psci_cpu_on_context(unsigned long cpuid, unsigned long entry_point, + unsigned long context_id) +{ + u32 fn = PSCI_FN_NATIVE(0_2, CPU_ON); + + if (!psci_ops.cpu_on) + return -EOPNOTSUPP; + if (psci_ops.cpu_on == psci_0_1_cpu_on) + fn = psci_0_1_function_ids.cpu_on; + + return __psci_cpu_on(fn, cpuid, entry_point, context_id); } static int __psci_migrate(u32 fn, unsigned long cpuid) diff --git a/include/linux/psci.h b/include/linux/psci.h index 4ca0060a3fc42b..5996cdd148771e 100644 --- a/include/linux/psci.h +++ b/include/linux/psci.h @@ -43,6 +43,9 @@ struct psci_0_1_function_ids { struct psci_0_1_function_ids get_psci_0_1_function_ids(void); +int psci_cpu_on_context(unsigned long cpuid, unsigned long entry_point, + unsigned long context_id); + #if defined(CONFIG_ARM_PSCI_FW) int __init psci_dt_init(void); #else From 100b2cf5981bbd3a0fc79be4a7a8b80f6b244a3c Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 16:41:55 -0700 Subject: [PATCH 03/30] arm64/multikernel: Spawn, confirm and release CPUs through PSCI x86 has no firmware that controls CPUs after boot, so its multikernel code is mostly a software CPU lifecycle: trampolines, identity page tables, a park loop in host-owned memory and mailboxes to move parked CPUs between them. On arm64 firmware does all of that. A pool CPU is a CPU in the PSCI OFF state. The host gets it there with the stock hotplug path, cpu_psci_cpu_die() and cpu_psci_cpu_kill(), which depart_cpu() already ends in. Spawning is CPU_ON at the Image's entry point with the device tree address as context ID, which is the state head.S asks for, after cleaning what the host wrote to the PoC because the new kernel reads it with the caches off. An off CPU belongs to whoever turns it on next, so there is no slot to repark between. What remains of that bookkeeping is the question the core asks before it rewrites an image or frees instance memory: could a CPU still be executing in there? The instance's kernel turns on its secondaries without telling the host, so every CPU of a spawned instance counts as started, a hot-added one joins them, and a CPU leaves the set once AFFINITY_INFO reports it off. That reuses cpus_on_slot, whose comment claimed it stays empty on such architectures; say what it means there instead. Waiting for OFF matters on hot-remove too: the instance acknowledges the offline before the CPU has reached firmware, and CPU_ON of a CPU that is not off yet fails with ALREADY_ON. Sending the doorbell IPI, forced stop and the halt path of a spawn are still stubs. Signed-off-by: Cong Wang --- arch/arm64/multikernel/spawn.c | 208 +++++++++++++++++++++++++++++++-- include/linux/multikernel.h | 15 +-- 2 files changed, 209 insertions(+), 14 deletions(-) diff --git a/arch/arm64/multikernel/spawn.c b/arch/arm64/multikernel/spawn.c index 474ba4cfb17056..83e0146bf493e7 100644 --- a/arch/arm64/multikernel/spawn.c +++ b/arch/arm64/multikernel/spawn.c @@ -2,16 +2,28 @@ /* * arm64 multikernel architecture interface. * - * Nothing is implemented yet: every entry point refuses, so an arm64 - * kernel builds with CONFIG_MULTIKERNEL=y but cannot spawn. + * Firmware owns CPU state on arm64. A pool CPU is a CPU in the PSCI OFF + * state: it executes nothing, so there is no park loop to keep in memory + * and no mailbox to watch. CPU_ON is the boot trampoline, delivering the + * CPU to an Image entry point in exactly the state head.S requires, and + * AFFINITY_INFO tells whether a CPU has come back. */ +#define pr_fmt(fmt) "mk_spawn: " fmt + #include #include +#include #include #include #include +#include +#include #include +#include + +/* A halting kernel acknowledges first and turns its CPUs off afterwards */ +#define MK_CPU_OFF_TIMEOUT_US USEC_PER_SEC void mk_arch_send_ipi(mk_phys_cpu_t phys_cpu) { @@ -31,33 +43,215 @@ void mk_force_stop_cpu(mk_phys_cpu_t phys_cpu) { } +static struct mk_cpu_set *mk_started_set(struct mk_instance *instance) +{ + if (!instance->cpus_on_slot) + instance->cpus_on_slot = mk_cpu_set_alloc(); + + return instance->cpus_on_slot; +} + +static int mk_wait_cpu_off(mk_phys_cpu_t mpidr) +{ + int state; + + if (!psci_ops.affinity_info) + return -EOPNOTSUPP; + + return read_poll_timeout(psci_ops.affinity_info, state, + state == PSCI_0_2_AFFINITY_LEVEL_OFF, + 1000, MK_CPU_OFF_TIMEOUT_US, false, mpidr, 0); +} + +/* + * The spawn reads these with the MMU and caches off, so what the host + * wrote through its cacheable mapping has to be in memory by then. + */ +static void mk_clean_to_poc(phys_addr_t start, size_t size) +{ + unsigned long va = (unsigned long)phys_to_virt(start); + + dcache_clean_inval_poc(va, va + size); +} + +static void mk_clean_image_to_poc(struct kimage *image, + struct mk_instance *instance) +{ + unsigned long i; + + for (i = 0; i < image->nr_segments; i++) + mk_clean_to_poc(image->segment[i].mem, image->segment[i].memsz); + + if (instance->ctrl_va) + mk_clean_to_poc(instance->ctrl_phys, MK_CTRL_BLOCK_SIZE); +} + +/** + * mk_arch_spawn_instance - Start an instance's boot CPU on its image + * @image: Loaded kimage for the instance + * @instance: Instance being spawned + * @cpu: Logical CPU, off in the pool, to boot the instance on + * + * Once the boot CPU runs, the instance's kernel may turn on any other CPU + * it was given without telling the host, so all of them count as started + * from here on and have to be seen off again before the image is reused. + */ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, int cpu) { - return -EOPNOTSUPP; + struct mk_cpu_set *started = mk_started_set(instance); + mk_phys_cpu_t mpidr = arch_cpu_physical_id(cpu); + mk_phys_cpu_t phys_cpu; + unsigned int i; + int ret; + + if (!mk_pool) + return -ENODEV; + + if (!image->arch.dtb_mem) { + pr_err("Instance %d has no boot device tree\n", instance->id); + return -EINVAL; + } + + /* Make the adds below infallible before anything runs */ + if (!started) + return -ENOMEM; + ret = mk_cpu_set_reserve(started, mk_cpu_set_count(instance->cpus)); + if (ret) + return ret; + + mk_clean_image_to_poc(image, instance); + + guard(mutex)(&mk_pool->park_lock); + + ret = psci_cpu_on_context(mpidr, image->start, image->arch.dtb_mem); + if (ret) { + pr_err("CPU_ON of CPU 0x%llx for instance %d failed: %d\n", + mpidr, instance->id, ret); + return ret; + } + + mk_cpu_set_for_each(i, phys_cpu, instance->cpus) + mk_cpu_set_add(started, phys_cpu); + + return 0; } +/** + * mk_arch_release_instance - Let go of an instance's CPUs before teardown + * @instance: Instance being torn down + * + * Fails if a CPU is still on: it may be executing the instance's image, + * so the caller must keep the instance's memory alive. + */ int mk_arch_release_instance(struct mk_instance *instance) { + int ret; + + ret = mk_repark_instance_to_host(instance); + if (ret) { + char buf[256]; + + mk_cpu_set_format(buf, sizeof(buf), instance->cpus_on_slot); + pr_err("Instance %d (%s): CPUs %s are not off, keeping its memory\n", + instance->id, instance->name, buf); + return ret; + } + + mk_cpu_set_free(instance->cpus_on_slot); + instance->cpus_on_slot = NULL; + instance->arch.dtb = NULL; + instance->arch.dtb_phys = 0; return 0; } +/** + * mk_arch_confirm_parked - Verify a CPU has left the instance's kernel + * @instance: Instance the CPU belongs to + * @phys_cpu: MPIDR of the CPU + * + * Returns 0 once firmware reports the CPU off. + */ int mk_arch_confirm_parked(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) { - return -EOPNOTSUPP; + return mk_wait_cpu_off(phys_cpu); } +/** + * mk_repark_instance_to_host - Return an instance's CPUs to the pool + * @instance: Instance being torn down + * + * An off CPU belongs to whoever turns it on next, so there is nothing to + * move. What is left to do is to see every started CPU off; one that is + * not stays tracked, and the caller must not free the instance's memory. + */ int mk_repark_instance_to_host(struct mk_instance *instance) { - return 0; + struct mk_cpu_set *started = instance->cpus_on_slot; + unsigned int i; + int failed = 0; + + if (mk_cpu_set_empty(started)) + return 0; + + guard(mutex)(&mk_pool->park_lock); + + /* Back to front, so deleting never shifts entries yet to be visited */ + for (i = mk_cpu_set_count(started); i-- > 0; ) { + mk_phys_cpu_t phys_cpu = started->ids[i]; + + if (mk_wait_cpu_off(phys_cpu)) + failed++; + else + mk_cpu_set_del(started, phys_cpu); + } + + return failed ? -ETIMEDOUT : 0; } +/** + * mk_repark_cpu_to_instance - Hand one pool CPU to a live instance + * @instance: Running instance the CPU is being hot-added to + * @phys_cpu: MPIDR of the CPU + * + * The instance's kernel turns the CPU on by itself; the host only starts + * counting it among the CPUs that kernel may be running on. + */ int mk_repark_cpu_to_instance(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) { - return 0; + struct mk_cpu_set *started = mk_started_set(instance); + + if (!started) + return -ENOMEM; + + guard(mutex)(&mk_pool->park_lock); + + if (mk_cpu_set_contains(started, phys_cpu)) + return 0; + + return mk_cpu_set_add(started, phys_cpu); } +/** + * mk_repark_cpu_to_host - Take one CPU back from an instance + * @instance: Instance the CPU was removed from + * @phys_cpu: MPIDR of the CPU + * + * The instance's kernel reports the CPU offline before the CPU has + * reached firmware. Turning it on elsewhere only works once it is off, + * so wait for that; until then it stays accounted to the instance. + */ int mk_repark_cpu_to_host(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) { - return 0; + int ret; + + guard(mutex)(&mk_pool->park_lock); + + if (!mk_cpu_set_contains(instance->cpus_on_slot, phys_cpu)) + return 0; + + ret = mk_wait_cpu_off(phys_cpu); + if (!ret) + mk_cpu_set_del(instance->cpus_on_slot, phys_cpu); + return ret; } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 5b1251687f32b9..d840bf5c955d87 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -709,13 +709,14 @@ struct mk_instance { size_t ctrl_used; /* - * CPUs parked inside this instance's memory (its arch park area) - * rather than in the host pool. They must be confirmed parked before - * the image is rewritten and they pin the pool until they leave. - * Membership is per CPU because it changes one CPU at a time: - * hotplug moves CPUs in and out while the instance runs, and a - * stopped instance can gain CPUs that still sit in the host pool. - * Always empty where firmware holds parked CPUs. + * CPUs the host pool cannot reach yet: parked inside this instance's + * memory (its arch park area), or, where firmware holds parked CPUs, + * handed to this instance's kernel and not seen off since. They must + * be confirmed parked before the image is rewritten and they pin the + * pool until they leave. Membership is per CPU because it changes one + * CPU at a time: hotplug moves CPUs in and out while the instance + * runs, and a stopped instance can gain CPUs that still sit in the + * host pool. */ struct mk_cpu_set *cpus_on_slot; From 1e6279e76a2b9222a0f28ba07fce036627f87ec7 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 16:46:13 -0700 Subject: [PATCH 04/30] arm64/multikernel: Turn every CPU off when a spawn kernel goes down Stock arm64 halts by sending the other CPUs to local_cpu_stop(), which leaves them in cpu_park_loop(), and spinning on the last one. All of them keep executing the kernel image, in a WFI loop with DAIF masked. That is fine when the machine resets next. For a spawn kernel it is the hazard the x86 park page exists to avoid: the host rewrites that image for the next spawn and the CPUs run whatever lands under them. PSCI cannot take a CPU away from outside either, so such a CPU is lost to the host for good. A spawn kernel returns every CPU to firmware instead: - machine_halt(), machine_power_off() and machine_restart() notify the host and turn all CPUs off through mk_halt_to_pool(). A spawn must not reach the PSCI SYSTEM_OFF or SYSTEM_RESET behind them, which would take the whole machine down, host included. - mk_enter_pool_state() is cpu_die() of the calling CPU, the boot CPU included: the host's CPUs are on, so firmware never sees the last CPU go. - local_cpu_stop(), where IPI_CPU_STOP and a parallel panic() put a CPU, turns it off instead of parking it. - panic_timeout defaults to reboot, which is the halt above, as on x86. The crash stop path already ends in cpu_die(). Secondaries need nothing: a spawn brings them up with the stock cpu_psci_cpu_boot() and hot-removes them with cpu_psci_cpu_die(), so there is no counterpart to x86's wakeup_secondary_cpu_64 hook, and the generic code never assumed one. The host learns that a halted instance's CPUs have arrived from AFFINITY_INFO, in mk_arch_confirm_parked(). mk_spawned() tells a kernel whether it booted with a manifest. The CPU_OFF path needs cpu_operations::cpu_die, so arm64 multikernel now depends on HOTPLUG_CPU, which the generic core needs to move CPUs into the pool anyway. Signed-off-by: Cong Wang --- arch/arm64/Kconfig | 2 +- arch/arm64/include/asm/multikernel.h | 20 +++++++++++++ arch/arm64/kernel/process.c | 5 ++++ arch/arm64/kernel/smp.c | 2 ++ arch/arm64/multikernel/spawn.c | 44 +++++++++++++++++++++++++++- include/linux/multikernel.h | 7 +++++ kernel/multikernel/manifest.c | 5 ++++ 7 files changed, 83 insertions(+), 2 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index f8d898f7b15e65..4dd175590a82de 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1623,7 +1623,7 @@ config ARCH_SUPPORTS_KEXEC_HANDOVER def_bool y config ARCH_SUPPORTS_MULTIKERNEL - def_bool ARM_PSCI_FW && KEXEC_FILE + def_bool ARM_PSCI_FW && KEXEC_FILE && HOTPLUG_CPU config ARCH_SUPPORTS_CRASH_DUMP def_bool y diff --git a/arch/arm64/include/asm/multikernel.h b/arch/arm64/include/asm/multikernel.h index 50db849ed7ef79..ec1270b68ff192 100644 --- a/arch/arm64/include/asm/multikernel.h +++ b/arch/arm64/include/asm/multikernel.h @@ -7,6 +7,7 @@ #include #include +#ifdef CONFIG_MULTIKERNEL /* * Pool CPUs are held by firmware in the PSCI OFF state and started with * CPU_ON, so an instance needs no trampoline, park page or page tables: @@ -19,6 +20,7 @@ struct mk_instance_arch { struct mk_pool_arch { }; +#endif /* Physical CPU IDs are MPIDR affinity values, as in the logical map */ static inline u64 arch_cpu_physical_id(int cpu) @@ -31,6 +33,24 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) return get_logical_index(phys_id); } +/* + * The ways down of a spawn kernel. Stock arm64 leaves stopped CPUs in a + * WFI loop inside the kernel image, which the host is about to overwrite; + * a spawn returns them to firmware instead. Both return on a host. + */ +#ifdef CONFIG_MULTIKERNEL +void mk_spawn_machine_halt(void); +void mk_spawn_stop_this_cpu(void); +#else +static inline void mk_spawn_machine_halt(void) +{ +} + +static inline void mk_spawn_stop_this_cpu(void) +{ +} +#endif + /* Control block: the boot device tree, reserved from the spawn's allocator */ #define MK_BOOT_DTB_SIZE SZ_64K #define MK_CTRL_BLOCK_SIZE ALIGN(MK_BOOT_DTB_SIZE, PAGE_SIZE) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 489554931231e6..20a7d00ac8d90b 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -98,6 +99,7 @@ void machine_shutdown(void) */ void machine_halt(void) { + mk_spawn_machine_halt(); local_irq_disable(); smp_send_stop(); while (1); @@ -111,6 +113,7 @@ void machine_halt(void) */ void machine_power_off(void) { + mk_spawn_machine_halt(); local_irq_disable(); smp_send_stop(); do_kernel_power_off(); @@ -127,6 +130,8 @@ void machine_power_off(void) */ void machine_restart(char *cmd) { + mk_spawn_machine_halt(); + /* Disable interrupts first */ local_irq_disable(); smp_send_stop(); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 1aa324104afb44..d5baf75a09c77d 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -867,6 +868,7 @@ static void __noreturn local_cpu_stop(unsigned int cpu) local_daif_mask(); sdei_mask_local_cpu(); + mk_spawn_stop_this_cpu(); cpu_park_loop(); } diff --git a/arch/arm64/multikernel/spawn.c b/arch/arm64/multikernel/spawn.c index 83e0146bf493e7..af2e53358f2bde 100644 --- a/arch/arm64/multikernel/spawn.c +++ b/arch/arm64/multikernel/spawn.c @@ -11,14 +11,18 @@ #define pr_fmt(fmt) "mk_spawn: " fmt +#include #include #include #include #include #include #include +#include #include #include +#include +#include #include #include @@ -33,12 +37,50 @@ void __init mk_arch_register_cpu(mk_phys_cpu_t phys_id) { } +/* + * Return the calling CPU to firmware. The boot CPU is no exception: the + * host's CPUs are on, so it is never the last one firmware knows about. + */ void __noreturn mk_enter_pool_state(void *info) { - local_irq_disable(); + unsigned int cpu = smp_processor_id(); + const struct cpu_operations *ops = get_cpu_ops(cpu); + + local_daif_mask(); + sdei_mask_local_cpu(); + + if (ops && ops->cpu_die) + ops->cpu_die(cpu); + + pr_crit("CPU%u could not be turned off, the host cannot reuse it\n", cpu); cpu_park_loop(); } +void mk_spawn_machine_halt(void) +{ + if (mk_spawned()) + mk_halt_to_pool(); +} + +void mk_spawn_stop_this_cpu(void) +{ + if (mk_spawned()) + mk_enter_pool_state(NULL); +} + +/* + * A panicked spawn kernel spinning in the panic loop holds its CPUs + * hostage. "Reboot on panic" is mk_spawn_machine_halt() here, which + * notifies the host and turns the CPUs off, so make it the default. + */ +static int __init mk_spawn_reboot_init(void) +{ + if (mk_spawned() && !panic_timeout) + panic_timeout = -1; + return 0; +} +core_initcall(mk_spawn_reboot_init); + void mk_force_stop_cpu(mk_phys_cpu_t phys_cpu) { } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index d840bf5c955d87..36b2b08ac68d20 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -968,6 +968,9 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +/* True in a kernel another kernel spawned, i.e. one that booted with a manifest */ +bool mk_spawned(void); + /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); #else @@ -1022,6 +1025,10 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } +static inline bool mk_spawned(void) +{ + return false; +} #endif /** diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 5220ac069e1f3f..23ddd99ca20653 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -34,6 +34,11 @@ phys_addr_t mk_manifest_phys(void) return mk_manifest_fdt_phys; } +bool mk_spawned(void) +{ + return mk_manifest_fdt_phys != 0; +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT From e31a696d8114916768d72a1b15f9a6fb8f31e0ac Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 16:51:17 -0700 Subject: [PATCH 05/30] arm64/multikernel: Load an Image into an instance with kexec_file kexec_file on arm64 already does what loading an instance takes: it relocates an Image to any 2 MB aligned address, and the generic code picks that address inside the instance's memory grant for a KEXEC_TYPE_MULTIKERNEL image. image->start is the Image's entry point and image->arch.dtb_mem the device tree, the two things CPU_ON needs. There never was a purgatory to skip. What has to go is the preparation for replacing the running kernel, which an instance never does: - machine_kexec_post_load() builds the relocation code, a copy of the linear map and the EL2 vectors. A multikernel image is loaded in place and started on another CPU, so it needs none of them. It is not flushed here either: every spawn copies the segments in again, and mk_arch_spawn_instance() cleans them to the PoC afterwards, the one place that covers the first exec and every later one. - machine_kexec_prepare() refuses to load while CPUs are stuck in the kernel, which only matters to a kernel that is about to be left. The device tree the loader builds is a copy of this kernel's own with a new /chosen. It names every CPU, all of memory and every device, so an instance must not boot from it, and what an instance owns can change between load and exec anyway. Its boot tree is therefore rebuilt in place on every spawn. Size the segment for that, and refuse to spawn until the code that builds the tree exists, instead of handing the machine to the new kernel. With the tree a kimage segment, nothing of arm64's lives in the instance control block, so drop the fields reserved for it. Signed-off-by: Cong Wang --- arch/arm64/include/asm/multikernel.h | 16 ++++++++----- arch/arm64/kernel/machine_kexec.c | 10 ++++++++- arch/arm64/kernel/machine_kexec_file.c | 7 ++++++ arch/arm64/multikernel/spawn.c | 31 ++++++++++++++++++++------ 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/multikernel.h b/arch/arm64/include/asm/multikernel.h index ec1270b68ff192..d34aecf7568796 100644 --- a/arch/arm64/include/asm/multikernel.h +++ b/arch/arm64/include/asm/multikernel.h @@ -10,12 +10,10 @@ #ifdef CONFIG_MULTIKERNEL /* * Pool CPUs are held by firmware in the PSCI OFF state and started with - * CPU_ON, so an instance needs no trampoline, park page or page tables: - * only the host's record of what it handed to CPU_ON. + * CPU_ON, so an instance needs no trampoline, park page or page tables, + * and what CPU_ON is given lives in the kimage. */ struct mk_instance_arch { - void *dtb; /* Boot device tree, in the control block */ - phys_addr_t dtb_phys; }; struct mk_pool_arch { @@ -51,8 +49,14 @@ static inline void mk_spawn_stop_this_cpu(void) } #endif -/* Control block: the boot device tree, reserved from the spawn's allocator */ +/* + * Room for an instance's boot device tree, a segment of its kimage. The + * tree is rebuilt in place on every spawn, as the memory grant can have + * changed since the load. + */ #define MK_BOOT_DTB_SIZE SZ_64K -#define MK_CTRL_BLOCK_SIZE ALIGN(MK_BOOT_DTB_SIZE, PAGE_SIZE) + +/* Nothing of arm64's lives in the control block, so it is never allocated */ +#define MK_CTRL_BLOCK_SIZE PAGE_SIZE #endif /* _ASM_ARM64_MULTIKERNEL_H */ diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 239c16e3d02f29..9362268755252d 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -54,7 +54,7 @@ void machine_kexec_cleanup(struct kimage *kimage) */ int machine_kexec_prepare(struct kimage *kimage) { - if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) { + if (kimage->type == KEXEC_TYPE_DEFAULT && cpus_are_stuck_in_kernel()) { pr_err("Can't kexec: CPUs are stuck in the kernel.\n"); return -EBUSY; } @@ -113,6 +113,14 @@ int machine_kexec_post_load(struct kimage *kimage) .trans_alloc_arg = kimage, }; + /* + * A multikernel image is loaded in place and never replaces this + * kernel: no relocation code, no copy of the linear map. It is + * copied in again and flushed on every spawn, not here. + */ + if (kimage->type == KEXEC_TYPE_MULTIKERNEL) + return 0; + /* If in place, relocation is not used, only flush next kernel */ if (kimage->head & IND_DONE) { kexec_segment_flush(kimage); diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c index fba260ad87a967..a499cd747e4dc9 100644 --- a/arch/arm64/kernel/machine_kexec_file.c +++ b/arch/arm64/kernel/machine_kexec_file.c @@ -21,6 +21,7 @@ #include #include #include +#include const struct kexec_file_ops * const kexec_file_loaders[] = { &kexec_image_ops, @@ -174,6 +175,12 @@ int load_other_segments(struct kimage *image, kbuf.bufsz = dtb_len; kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; kbuf.memsz = dtb_len; + /* + * A multikernel instance does not boot from this tree but from one + * rebuilt in its place on every spawn. Leave room for that. + */ + if (image->type == KEXEC_TYPE_MULTIKERNEL) + kbuf.memsz = max(dtb_len, MK_BOOT_DTB_SIZE); /* not across 2MB boundary */ kbuf.buf_align = SZ_2M; kbuf.buf_max = ULONG_MAX; diff --git a/arch/arm64/multikernel/spawn.c b/arch/arm64/multikernel/spawn.c index af2e53358f2bde..adb65d89b4cd33 100644 --- a/arch/arm64/multikernel/spawn.c +++ b/arch/arm64/multikernel/spawn.c @@ -116,16 +116,28 @@ static void mk_clean_to_poc(phys_addr_t start, size_t size) dcache_clean_inval_poc(va, va + size); } -static void mk_clean_image_to_poc(struct kimage *image, - struct mk_instance *instance) +/* + * Everything the new kernel reads before its MMU is on is a segment: the + * Image and the device tree. The manifest and the message ring are only + * touched with the caches on, where both kernels see the same memory. + */ +static void mk_clean_image_to_poc(struct kimage *image) { unsigned long i; for (i = 0; i < image->nr_segments; i++) mk_clean_to_poc(image->segment[i].mem, image->segment[i].memsz); +} - if (instance->ctrl_va) - mk_clean_to_poc(instance->ctrl_phys, MK_CTRL_BLOCK_SIZE); +/* + * The tree the loader left in the device tree segment is a copy of this + * kernel's own with a new /chosen: it names every CPU, all of memory and + * every device. A spawn booting from it would take the machine from + * under the host. + */ +static int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance) +{ + return -EOPNOTSUPP; } /** @@ -162,7 +174,14 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, if (ret) return ret; - mk_clean_image_to_poc(image, instance); + ret = mk_build_boot_dtb(image, instance); + if (ret) { + pr_err("No boot device tree for instance %d: %d\n", + instance->id, ret); + return ret; + } + + mk_clean_image_to_poc(image); guard(mutex)(&mk_pool->park_lock); @@ -202,8 +221,6 @@ int mk_arch_release_instance(struct mk_instance *instance) mk_cpu_set_free(instance->cpus_on_slot); instance->cpus_on_slot = NULL; - instance->arch.dtb = NULL; - instance->arch.dtb_phys = 0; return 0; } From fe10021a41d0340511a7c658bbe6055dc82dc41a Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 17:10:04 -0700 Subject: [PATCH 06/30] arm64/multikernel: Build the device tree an instance boots from On x86 a spawn gets its memory map in boot_params and everything else in the manifest, a device tree on a channel of its own. On arm64 the tree in x0 is the only boot channel, and the boot path reads memory, CPUs, PSCI, the timer and the interrupt controller from it long before any multikernel code runs. So the two become one tree: the instance tree the generic code writes, with /resources and the multikernel /chosen, is the base, and arm64 adds what its boot path needs: - a /memory node per region of the grant, the role mk_e820_fill() has on x86; - /cpus, the instance's own CPUs first so that the boot CPU leads and they get the low logical numbers, then every other CPU of the machine. A kernel can only online CPUs it enumerated at boot, and the generic code already prunes the ones the instance does not own from the present mask before smp_init(), so mk_arch_register_cpu() stays empty; - bootargs, the initrd and the seeds, taken from the /chosen of the tree the kexec_file loader made; - PSCI, the arch timer and the GICv3. The last three come from whichever description the host booted with. A DT host copies its own nodes, translating the GIC frames to CPU addresses as the node moves to the root and leaving its ITS child behind. An ACPI host, the SBSA server case, writes the generic bindings from its tables: the PSCI conduit from the FADT, the timer PPIs from the GTDT, the distributor and redistributor frames from the MADT. Both produce the same nodes on QEMU's virt machine. Either way the spawn is a DT platform: a populated tree disables ACPI on arm64 without any quirk, and no UEFI is advertised. There is no UART node on purpose. earlycon takes an address on the command line, and a probed pl011 would claim an SPI the host owns. ECAM windows for granted PCI segments are left to the device work; without them a spawn sees no PCI bus at all. The tree is written into the kimage's device tree segment on every spawn, since the grant can change between load and exec, and that segment grows to hold the manifest with the boot nodes on top. On the other side, setup_arch() accepts the boot tree as the manifest when its root says so, which is what makes mk_spawned() true. An instance now boots on QEMU virt from a DT and from a UEFI/ACPI host: both of its CPUs come up, it halts by turning them off, and a second exec finds them off and boots again. Until the GIC driver learns not to reset the shared distributor, that boot takes the host's SPIs away. Signed-off-by: Cong Wang --- arch/arm64/include/asm/multikernel.h | 14 +- arch/arm64/kernel/setup.c | 2 + arch/arm64/multikernel/Makefile | 3 +- arch/arm64/multikernel/dtb.c | 219 +++++++++++++++++++++++++++ arch/arm64/multikernel/dtb_acpi.c | 216 ++++++++++++++++++++++++++ arch/arm64/multikernel/dtb_of.c | 119 +++++++++++++++ arch/arm64/multikernel/internal.h | 34 +++++ arch/arm64/multikernel/spawn.c | 24 +-- 8 files changed, 616 insertions(+), 15 deletions(-) create mode 100644 arch/arm64/multikernel/dtb.c create mode 100644 arch/arm64/multikernel/dtb_acpi.c create mode 100644 arch/arm64/multikernel/dtb_of.c create mode 100644 arch/arm64/multikernel/internal.h diff --git a/arch/arm64/include/asm/multikernel.h b/arch/arm64/include/asm/multikernel.h index d34aecf7568796..a7b0c93950a457 100644 --- a/arch/arm64/include/asm/multikernel.h +++ b/arch/arm64/include/asm/multikernel.h @@ -39,7 +39,14 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) #ifdef CONFIG_MULTIKERNEL void mk_spawn_machine_halt(void); void mk_spawn_stop_this_cpu(void); + +/* The boot tree is the manifest when another kernel wrote it */ +void mk_spawn_accept_boot_tree(phys_addr_t dt_phys); #else +static inline void mk_spawn_accept_boot_tree(phys_addr_t dt_phys) +{ +} + static inline void mk_spawn_machine_halt(void) { } @@ -50,11 +57,12 @@ static inline void mk_spawn_stop_this_cpu(void) #endif /* - * Room for an instance's boot device tree, a segment of its kimage. The - * tree is rebuilt in place on every spawn, as the memory grant can have + * Room for an instance's boot device tree, a segment of its kimage: the + * manifest (MK_MANIFEST_SIZE) with the arm64 boot nodes on top. The tree + * is rebuilt in place on every spawn, as the memory grant can have * changed since the load. */ -#define MK_BOOT_DTB_SIZE SZ_64K +#define MK_BOOT_DTB_SIZE SZ_512K /* Nothing of arm64's lives in the control block, so it is never allocated */ #define MK_CTRL_BLOCK_SIZE PAGE_SIZE diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 23c05dc7a8f2ac..ec7960a86c5e9e 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -54,6 +54,7 @@ #include #include #include +#include static int num_standard_resources; static struct resource *standard_resources; @@ -290,6 +291,7 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) early_ioremap_init(); setup_machine_fdt(__fdt_pointer); + mk_spawn_accept_boot_tree(__fdt_pointer); /* * Initialise the static keys early as they may be enabled by the diff --git a/arch/arm64/multikernel/Makefile b/arch/arm64/multikernel/Makefile index 42c5f0565269ee..99809856e173aa 100644 --- a/arch/arm64/multikernel/Makefile +++ b/arch/arm64/multikernel/Makefile @@ -3,4 +3,5 @@ # Makefile for multikernel spawn support # -obj-y += spawn.o +obj-y += spawn.o dtb.o dtb_of.o +obj-$(CONFIG_ACPI) += dtb_acpi.o diff --git a/arch/arm64/multikernel/dtb.c b/arch/arm64/multikernel/dtb.c new file mode 100644 index 00000000000000..234d39c1df873b --- /dev/null +++ b/arch/arm64/multikernel/dtb.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The device tree an arm64 instance boots from. + * + * On arm64 the tree in x0 is the only boot channel, so it carries both + * halves of what x86 splits between boot_params and the manifest: the + * instance tree the generic code wrote, with its /resources and /chosen, + * is the base, and the nodes the arm64 boot path itself reads go on top: + * /memory, /cpus, PSCI, the arch timer and the interrupt controller. + * + * It is rebuilt on every spawn, into the device tree segment the loader + * left room for: the memory grant and the CPUs can change between load + * and exec, and a tree made at load would hand the spawn what it owned + * back then. + */ + +#define pr_fmt(fmt) "mk_dtb: " fmt + +#include +#include +#include +#include +#include + +#include "internal.h" + +static_assert(MK_MANIFEST_SIZE + SZ_64K <= MK_BOOT_DTB_SIZE); + +/* What the loader's tree knows and nobody else does */ +static const char * const mk_chosen_props[] = { + "bootargs", + "linux,initrd-start", + "linux,initrd-end", + "kaslr-seed", + "rng-seed", +}; + +/* @pairs are base,size; the root of the instance tree has two cells for each */ +int mk_dtb_set_reg(void *fdt, int node, const u64 *pairs, int nr) +{ + int i, ret; + + ret = fdt_delprop(fdt, node, "reg"); + if (ret && ret != -FDT_ERR_NOTFOUND) + return ret; + + for (i = 0; i < 2 * nr; i++) { + ret = fdt_appendprop_u64(fdt, node, "reg", pairs[i]); + if (ret) + return ret; + } + return 0; +} + +static int mk_dtb_add_memory(void *fdt, struct mk_instance *instance) +{ + struct mk_memory_region *region; + char name[32]; + int node, ret; + + list_for_each_entry(region, &instance->memory_regions, list) { + u64 reg[2] = { region->res.start, resource_size(®ion->res) }; + + snprintf(name, sizeof(name), "memory@%llx", reg[0]); + node = fdt_add_subnode(fdt, 0, name); + if (node < 0) + return node; + ret = fdt_setprop_string(fdt, node, "device_type", "memory"); + if (!ret) + ret = mk_dtb_set_reg(fdt, node, reg, 1); + if (ret) + return ret; + } + return 0; +} + +static int mk_dtb_add_cpu(void *fdt, int cpus, mk_phys_cpu_t mpidr) +{ + char name[32]; + int node, ret; + + snprintf(name, sizeof(name), "cpu@%llx", mpidr); + node = fdt_add_subnode(fdt, cpus, name); + if (node < 0) + return node; + + ret = fdt_setprop_string(fdt, node, "device_type", "cpu"); + if (!ret) + ret = fdt_setprop_string(fdt, node, "compatible", "arm,armv8"); + if (!ret) + ret = fdt_setprop_string(fdt, node, "enable-method", "psci"); + if (!ret) + ret = fdt_setprop_u64(fdt, node, "reg", mpidr); + return ret; +} + +/* + * The instance's own CPUs come first, the boot CPU leading as it does in + * instance->cpus, so they get the low logical numbers. Every other CPU + * of the machine follows: a kernel can only ever online a CPU it + * enumerated at boot, and the generic code prunes the ones the instance + * does not own from the present mask. + * + * fdt_add_subnode() puts a new node ahead of its siblings, hence the + * walk from the last CPU to the first. + */ +static int mk_dtb_add_cpus(void *fdt, struct mk_instance *instance) +{ + mk_phys_cpu_t mpidr; + unsigned int i; + int cpus, cpu, ret; + + cpus = fdt_add_subnode(fdt, 0, "cpus"); + if (cpus < 0) + return cpus; + ret = fdt_setprop_u32(fdt, cpus, "#address-cells", 2); + if (!ret) + ret = fdt_setprop_u32(fdt, cpus, "#size-cells", 0); + if (ret) + return ret; + + for (cpu = nr_cpu_ids - 1; cpu >= 0; cpu--) { + if (!cpu_possible(cpu)) + continue; + mpidr = arch_cpu_physical_id(cpu); + if (mpidr == INVALID_HWID || + mk_cpu_set_contains(instance->cpus, mpidr)) + continue; + ret = mk_dtb_add_cpu(fdt, cpus, mpidr); + if (ret) + return ret; + } + + for (i = mk_cpu_set_count(instance->cpus); i-- > 0; ) { + ret = mk_dtb_add_cpu(fdt, cpus, instance->cpus->ids[i]); + if (ret) + return ret; + } + return 0; +} + +static int mk_dtb_add_chosen(void *fdt, const void *loader_fdt) +{ + int from, to, i; + + from = fdt_path_offset(loader_fdt, "/chosen"); + if (from < 0) + return 0; + + to = fdt_path_offset(fdt, "/chosen"); + if (to < 0) + return to; + + for (i = 0; i < ARRAY_SIZE(mk_chosen_props); i++) { + const void *val; + int len, ret; + + val = fdt_getprop(loader_fdt, from, mk_chosen_props[i], &len); + if (!val) + continue; + ret = fdt_setprop(fdt, to, mk_chosen_props[i], val, len); + if (ret) + return ret; + } + return 0; +} + +static struct kexec_segment *mk_dtb_segment(struct kimage *image) +{ + unsigned long i; + + for (i = 0; i < image->nr_segments; i++) { + if (image->segment[i].mem == image->arch.dtb_mem) + return &image->segment[i]; + } + return NULL; +} + +/** + * mk_build_boot_dtb - Write an instance's boot tree over the loader's + * @image: Loaded kimage, its manifest already finalized for this spawn + * @instance: Instance being spawned + * + * Everything that adds to the tree returns libfdt error codes, which + * share their values with errnos and must not be mixed with them. + * + * Returns 0, -ENOSPC when the segment is too small, or -EINVAL. + */ +int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance) +{ + struct kexec_segment *seg = mk_dtb_segment(image); + void *fdt; + int ret; + + if (!seg || !image->arch.dtb || !instance->manifest_phys) + return -EINVAL; + + fdt = phys_to_virt(seg->mem); + ret = fdt_open_into(phys_to_virt(instance->manifest_phys), fdt, seg->memsz); + if (!ret) + ret = mk_dtb_add_memory(fdt, instance); + if (!ret) + ret = mk_dtb_add_cpus(fdt, instance); + if (!ret) + ret = mk_dtb_add_chosen(fdt, image->arch.dtb); + if (!ret) + ret = acpi_disabled ? mk_dtb_add_platform_of(fdt) : + mk_dtb_add_platform_acpi(fdt); + if (!ret) + ret = fdt_pack(fdt); + if (ret) { + pr_err("Instance %d: %s\n", instance->id, fdt_strerror(ret)); + return ret == -FDT_ERR_NOSPACE ? -ENOSPC : -EINVAL; + } + + pr_info("Instance %d boots from a %u byte tree at %#lx\n", + instance->id, fdt_totalsize(fdt), seg->mem); + return 0; +} diff --git a/arch/arm64/multikernel/dtb_acpi.c b/arch/arm64/multikernel/dtb_acpi.c new file mode 100644 index 00000000000000..b9ee0196321245 --- /dev/null +++ b/arch/arm64/multikernel/dtb_acpi.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Platform nodes of an instance tree, on a host that booted with ACPI. + * + * A spawn kernel always boots from a device tree, so what the host read + * from its tables is written out in the generic bindings: PSCI's conduit + * from the FADT, the arch timer's PPIs from the GTDT, and the GICv3 + * frames from the MADT. SBSA guarantees the generic drivers bind to it. + */ + +#define pr_fmt(fmt) "mk_dtb: " fmt + +#include +#include +#include +#include + +#include "internal.h" + +/* A redistributor frame pair per CPU, where the MADT has no GICR entries */ +#define MK_GICR_PER_CPU_SIZE (2 * SZ_64K) +#define MK_GICR_MAX_REGIONS 256 + +static int mk_dtb_add_psci(void *fdt) +{ + static const char compatible[] = "arm,psci-1.0\0arm,psci-0.2"; + int node, ret; + + node = fdt_add_subnode(fdt, 0, "psci"); + if (node < 0) + return node; + + ret = fdt_setprop(fdt, node, "compatible", compatible, sizeof(compatible)); + if (!ret) + ret = fdt_setprop_string(fdt, node, "method", + acpi_psci_use_hvc() ? "hvc" : "smc"); + return ret; +} + +static int mk_dtb_append_ppi(void *fdt, int node, u32 gsiv, u32 gtdt_flags) +{ + bool edge = gtdt_flags & ACPI_GTDT_INTERRUPT_MODE; + bool low = gtdt_flags & ACPI_GTDT_INTERRUPT_POLARITY; + u32 type; + int ret; + + if (edge) + type = low ? IRQ_TYPE_EDGE_FALLING : IRQ_TYPE_EDGE_RISING; + else + type = low ? IRQ_TYPE_LEVEL_LOW : IRQ_TYPE_LEVEL_HIGH; + + ret = fdt_appendprop_u32(fdt, node, "interrupts", MK_GIC_PPI); + if (!ret) + ret = fdt_appendprop_u32(fdt, node, "interrupts", gsiv - 16); + if (!ret) + ret = fdt_appendprop_u32(fdt, node, "interrupts", type); + return ret; +} + +/* The binding's order: secure, non-secure, virtual, hypervisor */ +static int mk_dtb_add_timer(void *fdt) +{ + struct acpi_table_header *table; + struct acpi_table_gtdt *gtdt; + int node, ret; + + if (ACPI_FAILURE(acpi_get_table(ACPI_SIG_GTDT, 0, &table))) { + pr_err("No GTDT to describe the arch timer from\n"); + return -FDT_ERR_NOTFOUND; + } + gtdt = (struct acpi_table_gtdt *)table; + + node = fdt_add_subnode(fdt, 0, "timer"); + ret = node < 0 ? node : 0; + if (!ret) + ret = fdt_setprop_string(fdt, node, "compatible", "arm,armv8-timer"); + if (!ret) + ret = mk_dtb_append_ppi(fdt, node, gtdt->secure_el1_interrupt, + gtdt->secure_el1_flags); + if (!ret) + ret = mk_dtb_append_ppi(fdt, node, gtdt->non_secure_el1_interrupt, + gtdt->non_secure_el1_flags); + if (!ret) + ret = mk_dtb_append_ppi(fdt, node, gtdt->virtual_timer_interrupt, + gtdt->virtual_timer_flags); + if (!ret && gtdt->non_secure_el2_interrupt) + ret = mk_dtb_append_ppi(fdt, node, gtdt->non_secure_el2_interrupt, + gtdt->non_secure_el2_flags); + if (!ret && (gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON)) + ret = fdt_setprop_empty(fdt, node, "always-on"); + + acpi_put_table(table); + return ret; +} + +struct mk_gic_frames { + u64 reg[2 * (1 + MK_GICR_MAX_REGIONS)]; /* distributor first */ + int nr_gicr; + int nr_gicc; + bool have_gicd; +}; + +/* + * Redistributors are described either as GICR regions or, without any, + * by each GICC's own frame address; the first kind wins, as in the + * GIC driver's own probing. + */ +static void mk_gic_collect(struct acpi_table_madt *madt, + struct mk_gic_frames *f, u8 type) +{ + unsigned long end = (unsigned long)madt + madt->header.length; + struct acpi_subtable_header *sub; + + for (sub = (void *)(madt + 1); + (unsigned long)sub + sizeof(*sub) <= end && sub->length; + sub = (void *)sub + sub->length) { + u64 *gicr = &f->reg[2 * (1 + f->nr_gicr)]; + + if (sub->type != type) + continue; + + if (type == ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR) { + struct acpi_madt_generic_distributor *gicd = (void *)sub; + + f->reg[0] = gicd->base_address; + f->reg[1] = SZ_64K; + f->have_gicd = gicd->version == ACPI_MADT_GIC_VERSION_V3 || + gicd->version == ACPI_MADT_GIC_VERSION_V4 || + gicd->version == ACPI_MADT_GIC_VERSION_NONE; + return; + } + + if (f->nr_gicr == MK_GICR_MAX_REGIONS) + return; + + if (type == ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR) { + struct acpi_madt_generic_redistributor *r = (void *)sub; + + gicr[0] = r->base_address; + gicr[1] = r->length; + f->nr_gicr++; + } else { + struct acpi_madt_generic_interrupt *gicc = (void *)sub; + + f->nr_gicc++; + if (!gicc->gicr_base_address) + continue; + gicr[0] = gicc->gicr_base_address; + gicr[1] = MK_GICR_PER_CPU_SIZE; + f->nr_gicr++; + } + } +} + +static int mk_dtb_add_gic(void *fdt) +{ + struct acpi_table_header *table; + struct mk_gic_frames *f; + int node, ret; + + if (ACPI_FAILURE(acpi_get_table(ACPI_SIG_MADT, 0, &table))) { + pr_err("No MADT to describe the GIC from\n"); + return -FDT_ERR_NOTFOUND; + } + + f = kzalloc(sizeof(*f), GFP_KERNEL); + if (!f) { + acpi_put_table(table); + return -FDT_ERR_NOSPACE; + } + + mk_gic_collect((void *)table, f, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR); + mk_gic_collect((void *)table, f, ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR); + if (!f->nr_gicr) + mk_gic_collect((void *)table, f, ACPI_MADT_TYPE_GENERIC_INTERRUPT); + acpi_put_table(table); + + if (!f->have_gicd || !f->nr_gicr) { + pr_err("The MADT describes no usable GICv3\n"); + ret = -FDT_ERR_NOTFOUND; + goto out; + } + + node = fdt_add_subnode(fdt, 0, "interrupt-controller"); + ret = node < 0 ? node : 0; + if (!ret) + ret = fdt_setprop_string(fdt, node, "compatible", "arm,gic-v3"); + if (!ret) + ret = fdt_setprop_empty(fdt, node, "interrupt-controller"); + if (!ret) + ret = fdt_setprop_u32(fdt, node, "#interrupt-cells", 3); + if (!ret) + ret = fdt_setprop_u32(fdt, node, "#redistributor-regions", + f->nr_gicr); + if (!ret) + ret = fdt_setprop_u32(fdt, node, "phandle", MK_GIC_PHANDLE); + if (!ret) + ret = mk_dtb_set_reg(fdt, node, f->reg, 1 + f->nr_gicr); + if (!ret) + ret = fdt_setprop_u32(fdt, 0, "interrupt-parent", MK_GIC_PHANDLE); +out: + kfree(f); + return ret; +} + +int mk_dtb_add_platform_acpi(void *fdt) +{ + int ret; + + ret = mk_dtb_add_psci(fdt); + if (!ret) + ret = mk_dtb_add_timer(fdt); + if (!ret) + ret = mk_dtb_add_gic(fdt); + return ret; +} diff --git a/arch/arm64/multikernel/dtb_of.c b/arch/arm64/multikernel/dtb_of.c new file mode 100644 index 00000000000000..1ac193682eb16e --- /dev/null +++ b/arch/arm64/multikernel/dtb_of.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Platform nodes of an instance tree, on a host that booted from a + * device tree: copies of the host's own nodes. + */ + +#define pr_fmt(fmt) "mk_dtb: " fmt + +#include +#include +#include + +#include "internal.h" + +static const struct of_device_id mk_psci_ids[] = { + { .compatible = "arm,psci" }, + { .compatible = "arm,psci-0.2" }, + { .compatible = "arm,psci-1.0" }, + {} +}; + +static const struct of_device_id mk_timer_ids[] = { + { .compatible = "arm,armv8-timer" }, + { .compatible = "arm,armv7-timer" }, + {} +}; + +static bool mk_prop_is_addressing(const char *name) +{ + return !strcmp(name, "reg") || !strcmp(name, "ranges") || + !strcmp(name, "#address-cells") || !strcmp(name, "#size-cells"); +} + +/* + * Copy @np's properties to a new node under the root. The phandle comes + * along, so interrupt-parent references between copied nodes keep + * working. Addressing does not: it is only valid below @np's parent bus. + */ +static int mk_dtb_copy_node(void *fdt, struct device_node *np) +{ + struct property *pp; + int node, ret; + + node = fdt_add_subnode(fdt, 0, np->full_name); + if (node < 0) + return node; + + for_each_property_of_node(np, pp) { + if (!strcmp(pp->name, "name") || mk_prop_is_addressing(pp->name)) + continue; + ret = fdt_setprop(fdt, node, pp->name, pp->value, pp->length); + if (ret) + return ret; + } + return node; +} + +static int mk_dtb_copy_matching(void *fdt, const struct of_device_id *ids, + const char *what) +{ + struct device_node *np = of_find_matching_node(NULL, ids); + int node; + + if (!np) { + pr_err("The host tree has no %s node\n", what); + return -FDT_ERR_NOTFOUND; + } + node = mk_dtb_copy_node(fdt, np); + of_node_put(np); + return node < 0 ? node : 0; +} + +/* + * The distributor and redistributor frames, translated to CPU addresses + * as the node moves to the root. Its children are left behind: an ITS + * has one command queue, which is the host's. + */ +static int mk_dtb_copy_gic(void *fdt) +{ + struct device_node *np; + u64 reg[2 * 16]; + struct resource res; + int node, nr = 0, ret; + + np = of_find_compatible_node(NULL, NULL, "arm,gic-v3"); + if (!np) { + pr_err("The host tree has no GICv3\n"); + return -FDT_ERR_NOTFOUND; + } + + node = mk_dtb_copy_node(fdt, np); + while (node >= 0 && nr < ARRAY_SIZE(reg) / 2 && + !of_address_to_resource(np, nr, &res)) { + reg[2 * nr] = res.start; + reg[2 * nr + 1] = resource_size(&res); + nr++; + } + of_node_put(np); + if (node < 0) + return node; + + ret = mk_dtb_set_reg(fdt, node, reg, nr); + if (!ret) + ret = fdt_setprop_u32(fdt, 0, "interrupt-parent", + fdt_get_phandle(fdt, node)); + return ret; +} + +int mk_dtb_add_platform_of(void *fdt) +{ + int ret; + + ret = mk_dtb_copy_matching(fdt, mk_psci_ids, "PSCI"); + if (!ret) + ret = mk_dtb_copy_matching(fdt, mk_timer_ids, "arch timer"); + if (!ret) + ret = mk_dtb_copy_gic(fdt); + return ret; +} diff --git a/arch/arm64/multikernel/internal.h b/arch/arm64/multikernel/internal.h new file mode 100644 index 00000000000000..efe9f4235bc700 --- /dev/null +++ b/arch/arm64/multikernel/internal.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ARM64_MULTIKERNEL_INTERNAL_H +#define _ARM64_MULTIKERNEL_INTERNAL_H + +#include +#include + +struct kimage; +struct mk_instance; + +/* GIC interrupt specifier cells, as in dt-bindings/interrupt-controller/arm-gic.h */ +#define MK_GIC_PPI 1 +#define MK_GIC_PHANDLE 1 + +int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance); + +int mk_dtb_set_reg(void *fdt, int node, const u64 *pairs, int nr); + +/* + * The part of the machine every kernel drives for itself: PSCI, the arch + * timer and the interrupt controller. Taken from whichever description + * this kernel booted with. + */ +int mk_dtb_add_platform_of(void *fdt); +#ifdef CONFIG_ACPI +int mk_dtb_add_platform_acpi(void *fdt); +#else +static inline int mk_dtb_add_platform_acpi(void *fdt) +{ + return -FDT_ERR_NOTFOUND; +} +#endif + +#endif /* _ARM64_MULTIKERNEL_INTERNAL_H */ diff --git a/arch/arm64/multikernel/spawn.c b/arch/arm64/multikernel/spawn.c index adb65d89b4cd33..0a4c69a135f4ca 100644 --- a/arch/arm64/multikernel/spawn.c +++ b/arch/arm64/multikernel/spawn.c @@ -17,7 +17,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -26,6 +28,8 @@ #include #include +#include "internal.h" + /* A halting kernel acknowledges first and turns its CPUs off afterwards */ #define MK_CPU_OFF_TIMEOUT_US USEC_PER_SEC @@ -37,6 +41,15 @@ void __init mk_arch_register_cpu(mk_phys_cpu_t phys_id) { } +void __init mk_spawn_accept_boot_tree(phys_addr_t dt_phys) +{ + if (!initial_boot_params || + !of_flat_dt_is_compatible(of_get_flat_dt_root(), MK_FDT_COMPATIBLE)) + return; + + mk_manifest_populate(dt_phys, fdt_totalsize(initial_boot_params)); +} + /* * Return the calling CPU to firmware. The boot CPU is no exception: the * host's CPUs are on, so it is never the last one firmware knows about. @@ -129,17 +142,6 @@ static void mk_clean_image_to_poc(struct kimage *image) mk_clean_to_poc(image->segment[i].mem, image->segment[i].memsz); } -/* - * The tree the loader left in the device tree segment is a copy of this - * kernel's own with a new /chosen: it names every CPU, all of memory and - * every device. A spawn booting from it would take the machine from - * under the host. - */ -static int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance) -{ - return -EOPNOTSUPP; -} - /** * mk_arch_spawn_instance - Start an instance's boot CPU on its image * @image: Loaded kimage for the instance From 0f7cf7e1642ef93329b997a26d8bfa84bb7c42f5 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 17:36:01 -0700 Subject: [PATCH 07/30] irqchip/gic-v3: Send an SGI to a CPU named by its MPIDR gic_ipi_send_mask() takes a mask of logical CPUs, which can only name CPUs of the running kernel. The hardware has no such limit: ICC_SGI1R addresses a redistributor by affinity, and an SGI is per-target state there, so it reaches a CPU that a different kernel image runs on with nothing shared between the two. Multikernel needs exactly that for its doorbell. Add gic_v3_send_sgi_to_mpidr(), which refuses a target whose Aff0 needs the range selector when the GIC has none. Signed-off-by: Cong Wang --- drivers/irqchip/irq-gic-v3.c | 26 ++++++++++++++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 1 + 2 files changed, 27 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 20f13b686ab22f..aad4ecbed60bab 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1401,6 +1401,32 @@ static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) isb(); } +/** + * gic_v3_send_sgi_to_mpidr - Send an SGI to a CPU named by its affinity + * @mpidr: MPIDR affinity of the target + * @sgi: SGI number + * + * gic_ipi_send_mask() reaches the CPUs of this kernel, by logical number. + * SGIs are delivered to a redistributor by affinity, though, so they + * cross to a CPU that another kernel image runs on just as well. + * + * Returns 0, or a negative error code if the SGI cannot be sent. + */ +int gic_v3_send_sgi_to_mpidr(u64 mpidr, unsigned int sgi) +{ + if (!gic_data.domain) + return -ENODEV; + if (sgi >= SGI_NR) + return -EINVAL; + if (MPIDR_AFFINITY_LEVEL(mpidr, 0) >= 16 && !gic_data.has_rss) + return -ERANGE; + + dsb(ishst); + gic_send_sgi(MPIDR_TO_SGI_CLUSTER_ID(mpidr), BIT(mpidr & 0xf), sgi); + isb(); + return 0; +} + static void __init gic_smp_init(void) { struct irq_fwspec sgi_fwspec = { diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 0225121f301380..666d9b5e0bdbfd 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -638,6 +638,7 @@ struct irq_domain; struct fwnode_handle; int __init its_lpi_memreserve_init(void); int its_cpu_init(void); +int gic_v3_send_sgi_to_mpidr(u64 mpidr, unsigned int sgi); int its_init(struct fwnode_handle *handle, struct rdists *rdists, struct irq_domain *domain, u8 irq_prio); int mbi_init(struct fwnode_handle *fwnode, struct irq_domain *parent); From 8efd067b7499664b9b64ca93b75489ed57f42075 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 17:36:02 -0700 Subject: [PATCH 08/30] arm64/multikernel: Ring the doorbell with an SGI The message ring between kernels needs a doorbell, the role MULTIKERNEL_VECTOR has on x86. On arm64 that is an SGI sent to the target's MPIDR with gic_v3_send_sgi_to_mpidr(). Its number is ABI between kernels, since the sender picks the INTID the receiver sees, so it cannot simply be the next free enumerator. Nor is there a free one: Linux uses all eight non-secure SGIs, and 8-15 belong to the secure world on the TF-A platforms this port targets. The doorbell therefore takes SGI 7 from the KGDB roundup whenever CONFIG_MULTIKERNEL is set. KGDB keeps working: without the arch kgdb_roundup_cpus() it uses the generic one, which needs no IPI of its own. Unlike the roundup, the doorbell is never an NMI, as it drains a message ring. There is no foreign doorbell to filter on the receiving side. All the handler does is drain the kernel's own ring, which a kernel without an instance relationship cannot have written to. With this a spawn's HALTED notification reaches the host, which moves the instance from active back to loaded, and a graceful halt requested by the host reaches the spawn. The ring itself needed nothing: both sides map it cacheable, as memremap(MEMREMAP_WB) is ioremap_cache() on arm64. Signed-off-by: Cong Wang --- arch/arm64/include/asm/smp.h | 8 ++++++++ arch/arm64/kernel/smp.c | 16 +++++++++++++--- arch/arm64/multikernel/spawn.c | 11 +++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 10ea4f54306903..4a3621b846d2e5 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -64,6 +64,14 @@ enum ipi_msg_type { */ IPI_CPU_BACKTRACE = NR_IPI, IPI_KGDB_ROUNDUP, + /* + * The multikernel doorbell goes to CPUs of other kernels, which + * makes its SGI number ABI between them. All eight non-secure SGIs + * are taken, so with CONFIG_MULTIKERNEL it has the one of the KGDB + * roundup, which can do without: the generic roundup needs no IPI + * of its own. + */ + IPI_MULTIKERNEL = IPI_KGDB_ROUNDUP, MAX_IPI }; diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index d5baf75a09c77d..7edddcfd719cc0 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -822,7 +823,11 @@ static const char *ipi_types[MAX_IPI] __tracepoint_string = { [IPI_TIMER] = "Timer broadcast interrupts", [IPI_IRQ_WORK] = "IRQ work interrupts", [IPI_CPU_BACKTRACE] = "CPU backtrace interrupts", +#ifdef CONFIG_MULTIKERNEL + [IPI_MULTIKERNEL] = "Multikernel doorbell interrupts", +#else [IPI_KGDB_ROUNDUP] = "KGDB roundup interrupts", +#endif }; static void smp_cross_call(const struct cpumask *target, unsigned int ipinr); @@ -938,7 +943,7 @@ void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) nmi_trigger_cpumask_backtrace(mask, exclude_cpu, arm64_backtrace_ipi); } -#ifdef CONFIG_KGDB +#if defined(CONFIG_KGDB) && !defined(CONFIG_MULTIKERNEL) void kgdb_roundup_cpus(void) { int this_cpu = raw_smp_processor_id(); @@ -1004,7 +1009,10 @@ static void do_handle_IPI(int ipinr) break; case IPI_KGDB_ROUNDUP: - kgdb_nmicallback(cpu, get_irq_regs()); + if (IS_ENABLED(CONFIG_MULTIKERNEL)) + generic_multikernel_interrupt(); + else + kgdb_nmicallback(cpu, get_irq_regs()); break; default: @@ -1038,8 +1046,10 @@ static bool ipi_should_be_nmi(enum ipi_msg_type ipi) switch (ipi) { case IPI_CPU_STOP_NMI: case IPI_CPU_BACKTRACE: - case IPI_KGDB_ROUNDUP: return true; + case IPI_KGDB_ROUNDUP: + /* The doorbell drains a message ring, which is no NMI work */ + return !IS_ENABLED(CONFIG_MULTIKERNEL); default: return false; } diff --git a/arch/arm64/multikernel/spawn.c b/arch/arm64/multikernel/spawn.c index 0a4c69a135f4ca..3d97db304ce643 100644 --- a/arch/arm64/multikernel/spawn.c +++ b/arch/arm64/multikernel/spawn.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -33,8 +34,18 @@ /* A halting kernel acknowledges first and turns its CPUs off afterwards */ #define MK_CPU_OFF_TIMEOUT_US USEC_PER_SEC +/* + * Ring the doorbell of a CPU that another kernel runs on. There is no + * foreign doorbell to filter on the receiving side: all it does is + * drain this kernel's own ring, which such a sender cannot have filled. + */ void mk_arch_send_ipi(mk_phys_cpu_t phys_cpu) { + int ret = gic_v3_send_sgi_to_mpidr(phys_cpu, IPI_MULTIKERNEL); + + if (ret) + pr_err_ratelimited("No doorbell SGI to CPU 0x%llx: %d\n", + phys_cpu, ret); } void __init mk_arch_register_cpu(mk_phys_cpu_t phys_id) From bbadb493d243a6749aa8cb41149163fa23274e89 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 18:00:07 -0700 Subject: [PATCH 09/30] irqchip/gic-v3: Add a tenant mode for a kernel that shares the GIC A GICv3 has one distributor. A kernel that multikernel spawns on some of the machine's CPUs finds it in use by the kernel that spawned it, and gic_dist_init() is the wrong thing to do to it: it disables the distributor, puts every SPI into group 1, disables and deactivates them all, and routes them to the boot CPU. The host loses its interrupts the moment the new kernel probes the GIC. Redistributors and CPU interfaces are per CPU, and nothing about them changes: a kernel initialises those of the CPUs it runs on, and the region walk only reads the frames of the others. Add a tenant mode, selected by "multikernel,tenant" in the GIC node of the tree such a kernel boots from: - The distributor is left as it is. It has to be enabled with affinity routing, which is checked, since nothing would work otherwise and the tenant is not the one to fix it. - Only the SPIs granted in "multikernel,spis", pairs of SPI numbers, can be mapped; every other SPI and all ESPIs fail with -EPERM. What the driver does to a mapped SPI is per-interrupt state (enable, priority, routing), apart from the read-modify-write of its ICFGR word, which it shares with 15 neighbours. - No MBIs, which are SPIs the tenant was not granted. SGIs and PPIs are per redistributor and unchanged. There is no ITS in such a tree, so no LPIs either. Signed-off-by: Cong Wang --- drivers/irqchip/irq-gic-v3.c | 80 +++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index aad4ecbed60bab..31a322783bf9bd 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -67,6 +67,9 @@ struct gic_chip_data { unsigned int ppi_nr; struct partition_affinity *parts; unsigned int nr_parts; + bool tenant; + u32 *tenant_spis; + unsigned int nr_tenant_spis; }; struct partition_affinity { @@ -920,6 +923,69 @@ static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) __gic_handle_irq_from_irqson(regs); } +/* + * Tenant mode, for a kernel that shares the GIC with the kernel that + * spawned it ("multikernel,tenant" in the GIC node). + * + * Redistributors and CPU interfaces are per CPU, and a tenant drives + * those of its own CPUs like any other kernel. The distributor exists + * once: initialising it would disable, regroup and reroute every SPI + * of the machine, the host's included. A tenant leaves it as the host + * set it up and only ever touches the per-interrupt state of the SPIs + * it was granted ("multikernel,spis", pairs of SPI + * numbers), so everything else must not even map. + */ +static bool gic_tenant_owns(irq_hw_number_t hw) +{ + unsigned int i; + + if (__get_intid_range(hw) != SPI_RANGE) + return false; + + for (i = 0; i < gic_data.nr_tenant_spis; i += 2) { + u32 first = gic_data.tenant_spis[i] + 32; + + if (hw >= first && hw - first < gic_data.tenant_spis[i + 1]) + return true; + } + return false; +} + +static int __init gic_tenant_check_dist(void) +{ + u32 need = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A; + u32 val = readl_relaxed(gic_data.dist_base + GICD_CTLR); + + if ((val & need) != need) { + pr_err("Tenant of a distributor its owner left unusable (GICD_CTLR %#x)\n", + val); + return -ENXIO; + } + + pr_info("Tenant mode, %u granted SPI range(s)\n", + gic_data.nr_tenant_spis / 2); + return 0; +} + +static void __init gic_of_tenant_init(struct device_node *node) +{ + int n; + + gic_data.tenant = of_property_read_bool(node, "multikernel,tenant"); + if (!gic_data.tenant) + return; + + n = of_property_count_u32_elems(node, "multikernel,spis"); + if (n < 2 || n % 2) + return; + + gic_data.tenant_spis = kcalloc(n, sizeof(u32), GFP_KERNEL); + if (gic_data.tenant_spis && + !of_property_read_u32_array(node, "multikernel,spis", + gic_data.tenant_spis, n)) + gic_data.nr_tenant_spis = n; +} + static void __init gic_dist_init(void) { unsigned int i; @@ -1590,6 +1656,8 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, case SPI_RANGE: case ESPI_RANGE: + if (gic_data.tenant && !gic_tenant_owns(hw)) + return -EPERM; irq_domain_set_info(d, irq, hw, chip, d->host_data, handle_fasteoi_irq, NULL, NULL); irq_set_probe(irq); @@ -2054,7 +2122,8 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gic_data.has_rss = !!(typer & GICD_TYPER_RSS); - if (typer & GICD_TYPER_MBIS) { + /* MBIs are SPIs used as MSIs, and a tenant was granted none for that */ + if ((typer & GICD_TYPER_MBIS) && !gic_data.tenant) { err = mbi_init(handle, gic_data.domain); if (err) pr_err("Failed to initialize MBIs\n"); @@ -2066,7 +2135,13 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gic_cpu_sys_reg_enable(); gic_prio_init(); - gic_dist_init(); + if (gic_data.tenant) { + err = gic_tenant_check_dist(); + if (err) + goto out_free; + } else { + gic_dist_init(); + } gic_cpu_init(); gic_enable_nmi_support(); gic_smp_init(); @@ -2264,6 +2339,7 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare redist_stride = 0; gic_enable_of_quirks(node, gic_quirks, &gic_data); + gic_of_tenant_init(node); err = gic_init_bases(dist_phys_base, dist_base, rdist_regs, nr_redist_regions, redist_stride, &node->fwnode); From 484270a284efb65f086e708ad275b95570891ea4 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 18:00:07 -0700 Subject: [PATCH 10/30] arm64/multikernel: Make an instance a tenant of the GIC Mark the GIC node of every instance tree "multikernel,tenant", so the spawn's GIC driver leaves the distributor to the host. Until now a spawn took all SPIs away from the host as it booted. An instance that owns no device owns no SPI. One that was given a platform device gets the device's node, copied from the host tree with reg translated to CPU addresses and the interrupts rewritten as plain GIC specifiers, and each of those SPIs is granted in "multikernel,spis". A spawn probes what its tree describes, so on arm64 the node is the grant; the name based allowlist that x86 applies in platform_device_add() never sees an OF device. The host gives the SPIs up by unbinding its driver, which disables them at the distributor, and takes them back when a driver binds again, which sets them up from scratch. A device still bound at exec is refused. Providers a node points to (clocks, resets, power domains, pinctrl) are not followed, which limits this to self-contained devices for now, and an ACPI host has no node to copy at all. On QEMU virt a spawn now runs a virtio-mmio disk on SPI 47 while the host keeps its UART and its own virtio devices, and the host binds the disk again once the instance has halted. Signed-off-by: Cong Wang --- arch/arm64/multikernel/Makefile | 2 +- arch/arm64/multikernel/dtb.c | 2 + arch/arm64/multikernel/dtb_devices.c | 133 +++++++++++++++++++++++++++ arch/arm64/multikernel/dtb_of.c | 50 +++++----- arch/arm64/multikernel/internal.h | 9 ++ 5 files changed, 171 insertions(+), 25 deletions(-) create mode 100644 arch/arm64/multikernel/dtb_devices.c diff --git a/arch/arm64/multikernel/Makefile b/arch/arm64/multikernel/Makefile index 99809856e173aa..0cd61e8a40e91e 100644 --- a/arch/arm64/multikernel/Makefile +++ b/arch/arm64/multikernel/Makefile @@ -3,5 +3,5 @@ # Makefile for multikernel spawn support # -obj-y += spawn.o dtb.o dtb_of.o +obj-y += spawn.o dtb.o dtb_of.o dtb_devices.o obj-$(CONFIG_ACPI) += dtb_acpi.o diff --git a/arch/arm64/multikernel/dtb.c b/arch/arm64/multikernel/dtb.c index 234d39c1df873b..7f8b8ecd30f4d2 100644 --- a/arch/arm64/multikernel/dtb.c +++ b/arch/arm64/multikernel/dtb.c @@ -206,6 +206,8 @@ int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance) if (!ret) ret = acpi_disabled ? mk_dtb_add_platform_of(fdt) : mk_dtb_add_platform_acpi(fdt); + if (!ret) + ret = mk_dtb_add_devices(fdt, instance); if (!ret) ret = fdt_pack(fdt); if (ret) { diff --git a/arch/arm64/multikernel/dtb_devices.c b/arch/arm64/multikernel/dtb_devices.c new file mode 100644 index 00000000000000..c9d26c1a2db72e --- /dev/null +++ b/arch/arm64/multikernel/dtb_devices.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Platform devices of an instance tree, and the instance's tenancy of + * the GIC. + * + * A spawn kernel probes what its tree describes, so granting a platform + * device is copying its node from the host tree. That takes a host that + * has one: on an ACPI host the device has no node to copy. Providers the + * node points at (clocks, resets, power domains, pinctrl) stay with the + * host and are not followed, which limits this to self-contained + * devices such as a virtio-mmio transport. + * + * The device's SPIs go with it. The spawn's GIC driver runs in tenant + * mode and maps nothing but the SPIs its node grants, and the host gives + * them up by unbinding its driver, which leaves them disabled at the + * distributor until a driver binds again after the instance is gone. + */ + +#define pr_fmt(fmt) "mk_dtb: " fmt + +#include +#include +#include +#include +#include + +#include "internal.h" + +#define MK_GIC_SPI 0 + +static int mk_dtb_gic_node(void *fdt) +{ + return fdt_node_offset_by_compatible(fdt, -1, "arm,gic-v3"); +} + +/* + * Write the device's interrupts as plain GIC specifiers and grant each + * SPI: whatever interrupt-parent or interrupt-map the host resolves them + * through is not part of the instance tree. + */ +static int mk_dtb_grant_irqs(void *fdt, const char *path, struct device_node *np) +{ + struct of_phandle_args irq; + int i, c, node, ret; + + node = fdt_path_offset(fdt, path); + if (node < 0) + return node; + fdt_delprop(fdt, node, "interrupts"); + fdt_delprop(fdt, node, "interrupts-extended"); + fdt_delprop(fdt, node, "interrupt-parent"); + + for (i = 0; !of_irq_parse_one(np, i, &irq); i++) { + bool spi = irq.args_count == 3 && irq.args[0] == MK_GIC_SPI && + of_device_is_compatible(irq.np, "arm,gic-v3"); + + of_node_put(irq.np); + if (!spi) { + pr_err("%pOF: interrupt %d is no GICv3 SPI\n", np, i); + return -FDT_ERR_BADVALUE; + } + + node = fdt_path_offset(fdt, path); + for (c = 0; c < 3 && node >= 0; c++) { + ret = fdt_appendprop_u32(fdt, node, "interrupts", irq.args[c]); + if (ret) + return ret; + } + + node = mk_dtb_gic_node(fdt); + if (node < 0) + return node; + ret = fdt_appendprop_u32(fdt, node, "multikernel,spis", irq.args[1]); + if (!ret) + ret = fdt_appendprop_u32(fdt, node, "multikernel,spis", 1); + if (ret) + return ret; + } + return 0; +} + +static int mk_dtb_add_device(void *fdt, const char *name) +{ + struct device *dev; + char path[64]; + int node, ret; + + dev = bus_find_device_by_name(&platform_bus_type, NULL, name); + if (!dev || !dev->of_node) { + pr_warn("Platform device %s has no node to describe it to an instance\n", + name); + put_device(dev); + return 0; + } + + if (dev->driver) { + pr_err("Platform device %s is still bound to %s here\n", + name, dev->driver->name); + put_device(dev); + return -FDT_ERR_BADSTATE; + } + + node = mk_dtb_copy_of_node(fdt, dev->of_node); + ret = node < 0 ? node : 0; + if (!ret) { + snprintf(path, sizeof(path), "/%pOFf", dev->of_node); + ret = mk_dtb_grant_irqs(fdt, path, dev->of_node); + } + put_device(dev); + return ret; +} + +int mk_dtb_add_devices(void *fdt, struct mk_instance *instance) +{ + struct mk_platform_device *plat; + int node, ret; + + node = mk_dtb_gic_node(fdt); + if (node < 0) + return node; + ret = fdt_setprop_empty(fdt, node, "multikernel,tenant"); + if (ret || !instance->platform_devices_valid) + return ret; + + list_for_each_entry(plat, &instance->platform_devices, list) { + if (!plat->name[0]) + continue; + ret = mk_dtb_add_device(fdt, plat->name); + if (ret) + return ret; + } + return 0; +} diff --git a/arch/arm64/multikernel/dtb_of.c b/arch/arm64/multikernel/dtb_of.c index 1ac193682eb16e..9f0d07cf565fdb 100644 --- a/arch/arm64/multikernel/dtb_of.c +++ b/arch/arm64/multikernel/dtb_of.c @@ -32,14 +32,17 @@ static bool mk_prop_is_addressing(const char *name) } /* - * Copy @np's properties to a new node under the root. The phandle comes - * along, so interrupt-parent references between copied nodes keep - * working. Addressing does not: it is only valid below @np's parent bus. + * Copy @np to a new node under the root and return its offset. The + * phandle comes along, so interrupt-parent references between copied + * nodes keep working. Addressing is only valid below @np's parent bus, + * so reg is written again, translated to CPU addresses. */ -static int mk_dtb_copy_node(void *fdt, struct device_node *np) +int mk_dtb_copy_of_node(void *fdt, struct device_node *np) { + u64 reg[2 * MK_DTB_MAX_REG]; struct property *pp; - int node, ret; + struct resource res; + int node, nr = 0, ret; node = fdt_add_subnode(fdt, 0, np->full_name); if (node < 0) @@ -52,6 +55,17 @@ static int mk_dtb_copy_node(void *fdt, struct device_node *np) if (ret) return ret; } + + while (nr < MK_DTB_MAX_REG && !of_address_to_resource(np, nr, &res)) { + reg[2 * nr] = res.start; + reg[2 * nr + 1] = resource_size(&res); + nr++; + } + if (nr) { + ret = mk_dtb_set_reg(fdt, node, reg, nr); + if (ret) + return ret; + } return node; } @@ -65,22 +79,19 @@ static int mk_dtb_copy_matching(void *fdt, const struct of_device_id *ids, pr_err("The host tree has no %s node\n", what); return -FDT_ERR_NOTFOUND; } - node = mk_dtb_copy_node(fdt, np); + node = mk_dtb_copy_of_node(fdt, np); of_node_put(np); return node < 0 ? node : 0; } /* - * The distributor and redistributor frames, translated to CPU addresses - * as the node moves to the root. Its children are left behind: an ITS - * has one command queue, which is the host's. + * The GIC's children are left behind: an ITS has one command queue, + * which is the host's. */ static int mk_dtb_copy_gic(void *fdt) { struct device_node *np; - u64 reg[2 * 16]; - struct resource res; - int node, nr = 0, ret; + int node; np = of_find_compatible_node(NULL, NULL, "arm,gic-v3"); if (!np) { @@ -88,22 +99,13 @@ static int mk_dtb_copy_gic(void *fdt) return -FDT_ERR_NOTFOUND; } - node = mk_dtb_copy_node(fdt, np); - while (node >= 0 && nr < ARRAY_SIZE(reg) / 2 && - !of_address_to_resource(np, nr, &res)) { - reg[2 * nr] = res.start; - reg[2 * nr + 1] = resource_size(&res); - nr++; - } + node = mk_dtb_copy_of_node(fdt, np); of_node_put(np); if (node < 0) return node; - ret = mk_dtb_set_reg(fdt, node, reg, nr); - if (!ret) - ret = fdt_setprop_u32(fdt, 0, "interrupt-parent", - fdt_get_phandle(fdt, node)); - return ret; + return fdt_setprop_u32(fdt, 0, "interrupt-parent", + fdt_get_phandle(fdt, node)); } int mk_dtb_add_platform_of(void *fdt) diff --git a/arch/arm64/multikernel/internal.h b/arch/arm64/multikernel/internal.h index efe9f4235bc700..1dc634b3c9ce10 100644 --- a/arch/arm64/multikernel/internal.h +++ b/arch/arm64/multikernel/internal.h @@ -16,6 +16,15 @@ int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance); int mk_dtb_set_reg(void *fdt, int node, const u64 *pairs, int nr); +/* reg entries a copied node may have; a GIC has one per redistributor region */ +#define MK_DTB_MAX_REG 16 + +struct device_node; +int mk_dtb_copy_of_node(void *fdt, struct device_node *np); + +/* Granted platform devices, and the SPIs the GIC node grants with them */ +int mk_dtb_add_devices(void *fdt, struct mk_instance *instance); + /* * The part of the machine every kernel drives for itself: PSCI, the arch * timer and the interrupt controller. Taken from whichever description From 34267155caad5804f15688abeed89532be7814d7 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 18:09:23 -0700 Subject: [PATCH 11/30] multikernel: Let an arm64 instance reach its PCI devices The instance tree already describes the root buses above an instance's PCI devices in the devicetree PCI binding, with the devices as the only children, and the spawn's PCI core already skips every function without a node. What a spawn on arm64 lacks is a way to generate config cycles and a driver for the bridge node: - mk_dt_bridge_ecam() only knew x86's MMCONFIG regions. A root bus on the generic ECAM accessors, which is what both pci-host-generic and an ACPI host on arm64 use, keeps its window in sysdata; take it from there so the node gets its reg. - With a reg the node is an ordinary ECAM host bridge. arm64 names pci-host-ecam-generic as its second compatible, so the stock driver binds; x86 creates the bus from the node by hand because it has no such driver. - Where I/O space is memory mapped, an I/O window's resource holds logical port numbers, and the ranges entry claimed the window sits at CPU address 0. Translate it with pci_pio_to_address(); x86, whose ports are an address space of their own, is unaffected. - BARs and bridge windows stay as the host assigned them: the bridge is shared and the host keeps running on the other devices. Say so with linux,pci-probe-only in /chosen. A spawn on QEMU virt now enumerates the virtio NIC it was given, and nothing else, behind windows identical to the host's. The NIC does not probe yet: INTx is wired to SPIs the host shares between devices, and there is no MSI controller in the tree until the ITS can be used from a spawn. Signed-off-by: Cong Wang --- arch/arm64/multikernel/dtb.c | 30 ++++++++++++++++++++++++++++++ arch/arm64/multikernel/internal.h | 2 ++ kernel/multikernel/dts.c | 29 ++++++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/arch/arm64/multikernel/dtb.c b/arch/arm64/multikernel/dtb.c index 7f8b8ecd30f4d2..40aceb56a4a23b 100644 --- a/arch/arm64/multikernel/dtb.c +++ b/arch/arm64/multikernel/dtb.c @@ -162,6 +162,34 @@ static int mk_dtb_add_chosen(void *fdt, const void *loader_fdt) if (ret) return ret; } + + /* BARs and bridge windows are the host's to assign, for every kernel */ + return fdt_setprop_u32(fdt, to, "linux,pci-probe-only", 1); +} + +/* + * The generic code describes the root buses above the instance's PCI + * devices, with their ECAM window. That makes each of them an ordinary + * ECAM host bridge, so name the driver for it. + */ +static int mk_dtb_name_pci_bridges(void *fdt) +{ + static const char compatible[] = + MK_PCI_HOST_BRIDGE "\0pci-host-ecam-generic"; + int node, ret; + + for (node = fdt_node_offset_by_compatible(fdt, -1, MK_PCI_HOST_BRIDGE); + node >= 0; + node = fdt_node_offset_by_compatible(fdt, node, MK_PCI_HOST_BRIDGE)) { + if (!fdt_getprop(fdt, node, "reg", NULL)) { + pr_warn("A PCI root bus of the instance has no ECAM window\n"); + continue; + } + ret = fdt_setprop(fdt, node, "compatible", compatible, + sizeof(compatible)); + if (ret) + return ret; + } return 0; } @@ -208,6 +236,8 @@ int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance) mk_dtb_add_platform_acpi(fdt); if (!ret) ret = mk_dtb_add_devices(fdt, instance); + if (!ret) + ret = mk_dtb_name_pci_bridges(fdt); if (!ret) ret = fdt_pack(fdt); if (ret) { diff --git a/arch/arm64/multikernel/internal.h b/arch/arm64/multikernel/internal.h index 1dc634b3c9ce10..133f491496cc1a 100644 --- a/arch/arm64/multikernel/internal.h +++ b/arch/arm64/multikernel/internal.h @@ -9,6 +9,8 @@ struct kimage; struct mk_instance; /* GIC interrupt specifier cells, as in dt-bindings/interrupt-controller/arm-gic.h */ +#define MK_PCI_HOST_BRIDGE "multikernel,pci-host-bridge" + #define MK_GIC_PPI 1 #define MK_GIC_PHANDLE 1 diff --git a/kernel/multikernel/dts.c b/kernel/multikernel/dts.c index a62aae701f8ab9..99a514482b0daf 100644 --- a/kernel/multikernel/dts.c +++ b/kernel/multikernel/dts.c @@ -20,6 +20,7 @@ #include #include #include +#include #if defined(CONFIG_X86) && defined(CONFIG_PCI_MMCONFIG) #include #endif @@ -1224,6 +1225,18 @@ static u64 mk_dt_bridge_ecam(struct pci_bus *root, u64 *size) *size = (u64)(root->busn_res.end - root->busn_res.start + 1) << 20; return cfg->address + ((u64)root->busn_res.start << 20); +#elif defined(CONFIG_PCI_ECAM) + struct pci_config_window *cfg = root->sysdata; + + /* Only a bridge on the generic ECAM accessors keeps its window there */ + if (root->ops != &pci_generic_ecam_ops.pci_ops || + cfg->busr.start > root->busn_res.start || + cfg->busr.end < root->busn_res.end) + return 0; + + *size = (u64)(root->busn_res.end - root->busn_res.start + 1) << 20; + return cfg->res.start + + ((u64)(root->busn_res.start - cfg->busr.start) << 20); #else return 0; #endif @@ -1281,7 +1294,7 @@ static int mk_dt_emit_one_host_bridge(struct mk_instance *instance, resource_list_for_each_entry(entry, &bridge->windows) { const struct resource *res = entry->res; - u64 pci_addr, size; + u64 pci_addr, cpu_addr, size; if (resource_type(res) != IORESOURCE_IO && resource_type(res) != IORESOURCE_MEM) @@ -1295,11 +1308,21 @@ static int mk_dt_emit_one_host_bridge(struct mk_instance *instance, pci_addr = res->start - entry->offset; size = resource_size(res); + + /* + * Where I/O space is memory mapped, an I/O resource holds + * logical port numbers, not the CPU address of the window. + */ + cpu_addr = res->start; + if (resource_type(res) == IORESOURCE_IO && + pci_pio_to_address(res->start) != (phys_addr_t)OF_BAD_ADDR) + cpu_addr = pci_pio_to_address(res->start); + ranges[cells++] = cpu_to_fdt32(mk_dt_window_flags(res)); ranges[cells++] = cpu_to_fdt32(upper_32_bits(pci_addr)); ranges[cells++] = cpu_to_fdt32(lower_32_bits(pci_addr)); - ranges[cells++] = cpu_to_fdt32(upper_32_bits(res->start)); - ranges[cells++] = cpu_to_fdt32(lower_32_bits(res->start)); + ranges[cells++] = cpu_to_fdt32(upper_32_bits(cpu_addr)); + ranges[cells++] = cpu_to_fdt32(lower_32_bits(cpu_addr)); ranges[cells++] = cpu_to_fdt32(upper_32_bits(size)); ranges[cells++] = cpu_to_fdt32(lower_32_bits(size)); } From e836fd2228436e2cb5d83624acbab5d995edc1ec Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 18:23:13 -0700 Subject: [PATCH 12/30] irqchip/gic-v3-its: Map LPIs on behalf of another kernel An ITS has one command queue, owned by the kernel that booted the machine. A kernel that multikernel spawns on some of the CPUs cannot run the ITS driver against it, so it cannot map the MSIs of the PCI devices it was given. The host has to do it for it. Add the small API that takes: - its_foreign_map() maps an event of a DeviceID to an LPI from this kernel's allocator and routes it to the collection of a given CPU, or moves it there if it is mapped already. The CPU is one the other kernel runs on. Its collection is the one this kernel mapped while the CPU was its own, and stays valid after the CPU was given away. The same goes for the LPI tables: that redistributor keeps using the ones this kernel enabled, which cannot be taken back, so the configuration byte is written here too and the LPI is enabled from the start. No Linux interrupt exists for such an LPI on this side; it never reaches one of this kernel's CPU interfaces. - its_foreign_release() discards everything mapped for one owner and frees its ITTs and LPIs, for when that kernel is gone, whether it went in order or not. - its_foreign_doorbell() is the address such a device writes to. A device that is mapped for another kernel is not an alias to share an ITT with, so its_msi_prepare() refuses it. Systems with more than one ITS are left out: picking the ITS behind a device, or giving a whole ITS away, can come later. Signed-off-by: Cong Wang --- drivers/irqchip/irq-gic-v3-its.c | 151 +++++++++++++++++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 5 + kernel/multikernel/Kconfig | 4 + 3 files changed, 160 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 291d7668cc8daf..1842ad2aca56dd 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -176,6 +176,7 @@ struct its_device { u32 nr_ites; u32 device_id; bool shared; + void *foreign; /* owner, when mapped for another kernel */ }; static struct { @@ -3601,6 +3602,12 @@ static int its_msi_prepare(struct irq_domain *domain, struct device *dev, mutex_lock(&its->dev_alloc_lock); its_dev = its_find_device(its, dev_id); + if (its_dev && its_dev->foreign) { + /* Mapped for another kernel, which is not an alias to share with */ + its_dev = NULL; + err = -EBUSY; + goto out; + } if (its_dev) { /* * We already have seen this ID, probably through @@ -3771,6 +3778,150 @@ static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, irq_domain_free_irqs_parent(domain, virq, nr_irqs); } +#ifdef CONFIG_MULTIKERNEL_ITS_PROXY +/* + * LPIs on behalf of another kernel. + * + * An ITS has one command queue, so a kernel that multikernel spawned on + * some of the machine's CPUs cannot map the MSIs of its devices itself. + * This kernel does it for it: the events of such a device are mapped to + * LPIs from this kernel's allocator and routed to the collection of a + * CPU the other kernel runs on. That collection is the one this kernel + * mapped for the CPU before it gave it away, and its redistributor still + * uses this kernel's LPI tables, which is why the configuration byte is + * written here as well. No Linux interrupt exists for these LPIs on this + * side: they never reach one of this kernel's CPU interfaces. + * + * @owner keeps the devices of different kernels apart and names what + * its_foreign_release() tears down. + */ +static struct its_node *its_foreign_node(void) +{ + if (!list_is_singular(&its_nodes)) + return NULL; + return list_first_entry(&its_nodes, struct its_node, entry); +} + +static void its_foreign_config(struct its_device *dev, u32 event, bool enable) +{ + u8 *cfg = gic_rdists->prop_table_va + + dev->event_map.lpi_base + event - 8192; + + *cfg = lpi_prop_prio | LPI_PROP_GROUP1 | (enable ? LPI_PROP_ENABLED : 0); + + if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) + gic_flush_dcache_to_poc(cfg, sizeof(*cfg)); + else + dsb(ishst); + its_send_inv(dev, event); +} + +/* A collection is only mapped for a CPU that has been online here */ +static struct its_collection *its_foreign_collection(struct its_node *its, + unsigned int cpu) +{ + if (cpu >= nr_cpu_ids || !cpu_possible(cpu) || + its->collections[cpu].col_id != cpu || + (cpu && !its->collections[cpu].target_address)) + return NULL; + return &its->collections[cpu]; +} + +/** + * its_foreign_map - Route a device's event to a CPU of another kernel + * @owner: the other kernel, as the caller names it + * @dev_id: DeviceID as the ITS sees it + * @event: event (MSI data) to map + * @nvecs: events the device will use at most, fixed by the first call + * @cpu: this kernel's number of the CPU to deliver to + * + * Mapping an event that is mapped already moves it to @cpu. + * + * Returns the LPI, which is the INTID the other kernel takes on @cpu, + * or a negative error code. + */ +int its_foreign_map(void *owner, u32 dev_id, u32 event, u32 nvecs, + unsigned int cpu) +{ + struct its_node *its = its_foreign_node(); + struct its_collection *col; + struct its_device *dev; + + if (!its) + return -ENODEV; + col = its_foreign_collection(its, cpu); + if (!col) + return -EINVAL; + + guard(mutex)(&its->dev_alloc_lock); + + dev = its_find_device(its, dev_id); + if (dev && dev->foreign != owner) + return -EBUSY; + if (!dev) { + dev = its_create_device(its, dev_id, + roundup_pow_of_two(max(nvecs, 1U)), true); + if (!dev) + return -ENOMEM; + dev->foreign = owner; + } + if (event >= dev->event_map.nr_lpis) + return -ERANGE; + + dev->event_map.col_map[event] = cpu; + if (test_and_set_bit(event, dev->event_map.lpi_map)) { + its_send_movi(dev, col, event); + } else { + its_send_mapti(dev, dev->event_map.lpi_base + event, event); + its_foreign_config(dev, event, true); + } + return dev->event_map.lpi_base + event; +} + +/** + * its_foreign_release - Drop everything mapped for another kernel + * @owner: the other kernel, as given to its_foreign_map() + * + * For when that kernel is gone or about to be started afresh, whether it + * shut down in order or not: nothing of it must keep raising LPIs. + */ +void its_foreign_release(void *owner) +{ + struct its_node *its = its_foreign_node(); + struct its_device *dev, *tmp; + unsigned int event; + + if (!its) + return; + + guard(mutex)(&its->dev_alloc_lock); + + list_for_each_entry_safe(dev, tmp, &its->its_device_list, entry) { + if (dev->foreign != owner) + continue; + + for_each_set_bit(event, dev->event_map.lpi_map, + dev->event_map.nr_lpis) { + its_foreign_config(dev, event, false); + its_send_discard(dev, event); + } + + its_lpi_free(dev->event_map.lpi_map, dev->event_map.lpi_base, + dev->event_map.nr_lpis); + its_send_mapd(dev, 0); + its_free_device(dev); + } +} + +/* Where a device of another kernel writes its MSIs: 0 without a usable ITS */ +phys_addr_t its_foreign_doorbell(void) +{ + struct its_node *its = its_foreign_node(); + + return its ? its->phys_base + GITS_TRANSLATER : 0; +} +#endif + static const struct irq_domain_ops its_domain_ops = { .select = msi_lib_irq_domain_select, .alloc = its_irq_domain_alloc, diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 666d9b5e0bdbfd..25c2df7ba55a94 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -639,6 +639,11 @@ struct fwnode_handle; int __init its_lpi_memreserve_init(void); int its_cpu_init(void); int gic_v3_send_sgi_to_mpidr(u64 mpidr, unsigned int sgi); + +int its_foreign_map(void *owner, u32 dev_id, u32 event, u32 nvecs, + unsigned int cpu); +void its_foreign_release(void *owner); +phys_addr_t its_foreign_doorbell(void); int its_init(struct fwnode_handle *handle, struct rdists *rdists, struct irq_domain *domain, u8 irq_prio); int mbi_init(struct fwnode_handle *fwnode, struct irq_domain *parent); diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index cec1ea32d83361..b548f14be847e0 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -33,3 +33,7 @@ config MULTIKERNEL boot channel, describing its resources and message rings If unsure, say N. + +# Map the MSIs of an instance's PCI devices through the host's ITS +config MULTIKERNEL_ITS_PROXY + def_bool MULTIKERNEL && ARM_GIC_V3_ITS && PCI_MSI From 4747c5379908e7657f7209b5a809e6e749eb722e Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 18:23:37 -0700 Subject: [PATCH 13/30] multikernel: Route an instance's MSIs through the host's ITS x86 needs nothing to give an instance the MSIs of its PCI devices: the message address names the destination LAPIC. On a GICv3 they go through the ITS, which only the host can program, so an instance asks the host over the message ring. MK_IO_MSI_MAP names a device by its PCI address, an event and one of the instance's CPUs. The host checks that the instance owns both, looks up the DeviceID behind the device, maps the event with its_foreign_map() from a work item, since messages arrive in interrupt context and the ITS driver sleeps, and answers with the LPI in MK_IO_MSI_ACK. Mapping an event again moves it, which is how an affinity change is sent from under the descriptor lock without waiting for an answer. There is no unmap: the host drops everything an instance had when it halts, however it halts, and once more before it is started, so a crashed instance leaves nothing behind and the host can bind the device again right after a halt. The instance's side is irq-gic-v3-its-mk.c, what is left of an ITS driver when someone else runs the command queue: an MSI parent domain on the GIC's LPI range, reusing the ITS MSI parent ops, that composes messages from the ITS's doorbell address and the event. LPIs are not masked there. The redistributors of the instance's CPUs run on the host's LPI tables, so the enable bit is the host's to write, and the MSI capability of the device can be masked without asking anyone. arm64 puts the proxy node into the instance tree, with the doorbell as its reg, and points the msi-map of the instance's root buses at it with an identity mapping. On QEMU virt a spawn now runs a virtio-net PCI device: it gets a DHCP lease and pings through MSI-X vectors delivered as LPIs, while the host keeps its own virtio devices on the same ITS. After a halt the host binds the NIC and uses it with vectors of its own, gives it back, and the next spawn takes it again. Signed-off-by: Cong Wang --- arch/arm64/multikernel/Makefile | 2 +- arch/arm64/multikernel/dtb.c | 28 +-- arch/arm64/multikernel/dtb_pci.c | 99 ++++++++++ arch/arm64/multikernel/internal.h | 3 + drivers/irqchip/Makefile | 1 + drivers/irqchip/irq-gic-v3-its-mk.c | 268 ++++++++++++++++++++++++++++ include/linux/multikernel.h | 38 ++++ kernel/kexec_core.c | 3 + kernel/multikernel/Makefile | 1 + kernel/multikernel/core.c | 2 + kernel/multikernel/its_proxy.c | 174 ++++++++++++++++++ 11 files changed, 591 insertions(+), 28 deletions(-) create mode 100644 arch/arm64/multikernel/dtb_pci.c create mode 100644 drivers/irqchip/irq-gic-v3-its-mk.c create mode 100644 kernel/multikernel/its_proxy.c diff --git a/arch/arm64/multikernel/Makefile b/arch/arm64/multikernel/Makefile index 0cd61e8a40e91e..95fbd2f416607d 100644 --- a/arch/arm64/multikernel/Makefile +++ b/arch/arm64/multikernel/Makefile @@ -3,5 +3,5 @@ # Makefile for multikernel spawn support # -obj-y += spawn.o dtb.o dtb_of.o dtb_devices.o +obj-y += spawn.o dtb.o dtb_of.o dtb_devices.o dtb_pci.o obj-$(CONFIG_ACPI) += dtb_acpi.o diff --git a/arch/arm64/multikernel/dtb.c b/arch/arm64/multikernel/dtb.c index 40aceb56a4a23b..bb1720c2ed54ca 100644 --- a/arch/arm64/multikernel/dtb.c +++ b/arch/arm64/multikernel/dtb.c @@ -167,32 +167,6 @@ static int mk_dtb_add_chosen(void *fdt, const void *loader_fdt) return fdt_setprop_u32(fdt, to, "linux,pci-probe-only", 1); } -/* - * The generic code describes the root buses above the instance's PCI - * devices, with their ECAM window. That makes each of them an ordinary - * ECAM host bridge, so name the driver for it. - */ -static int mk_dtb_name_pci_bridges(void *fdt) -{ - static const char compatible[] = - MK_PCI_HOST_BRIDGE "\0pci-host-ecam-generic"; - int node, ret; - - for (node = fdt_node_offset_by_compatible(fdt, -1, MK_PCI_HOST_BRIDGE); - node >= 0; - node = fdt_node_offset_by_compatible(fdt, node, MK_PCI_HOST_BRIDGE)) { - if (!fdt_getprop(fdt, node, "reg", NULL)) { - pr_warn("A PCI root bus of the instance has no ECAM window\n"); - continue; - } - ret = fdt_setprop(fdt, node, "compatible", compatible, - sizeof(compatible)); - if (ret) - return ret; - } - return 0; -} - static struct kexec_segment *mk_dtb_segment(struct kimage *image) { unsigned long i; @@ -237,7 +211,7 @@ int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance) if (!ret) ret = mk_dtb_add_devices(fdt, instance); if (!ret) - ret = mk_dtb_name_pci_bridges(fdt); + ret = mk_dtb_add_pci(fdt); if (!ret) ret = fdt_pack(fdt); if (ret) { diff --git a/arch/arm64/multikernel/dtb_pci.c b/arch/arm64/multikernel/dtb_pci.c new file mode 100644 index 00000000000000..dc9e58c4f5d457 --- /dev/null +++ b/arch/arm64/multikernel/dtb_pci.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCI in an instance tree. + * + * The generic code describes the root buses above the instance's PCI + * devices, ECAM window included, which makes each of them an ordinary + * ECAM host bridge; what is left is to name the driver for it. INTx is + * not offered: the lines are SPIs that the host shares between devices. + * MSIs go through the host's ITS, which the instance reaches through a + * proxy node: its reg is the ITS's doorbell, the rest happens over the + * message ring (irq-gic-v3-its-mk.c). + */ + +#define pr_fmt(fmt) "mk_dtb: " fmt + +#include +#include +#include + +#include "internal.h" + +#define MK_ITS_PROXY "multikernel,gic-v3-its" + +static int mk_dtb_add_its_proxy(void *fdt, u32 *phandle) +{ + u64 reg[2] = { its_foreign_doorbell(), sizeof(u32) }; + int node, ret; + + ret = fdt_generate_phandle(fdt, phandle); + if (ret) + return ret; + + node = fdt_add_subnode(fdt, 0, "msi-controller"); + if (node < 0) + return node; + + ret = fdt_setprop_string(fdt, node, "compatible", MK_ITS_PROXY); + if (!ret) + ret = fdt_setprop_empty(fdt, node, "msi-controller"); + if (!ret) + ret = fdt_setprop_u32(fdt, node, "#msi-cells", 1); + if (!ret) + ret = fdt_setprop_u32(fdt, node, "phandle", *phandle); + if (!ret) + ret = mk_dtb_set_reg(fdt, node, reg, 1); + return ret; +} + +/* Every requester ID maps to itself: the host knows the real DeviceID */ +static int mk_dtb_set_msi_map(void *fdt, int node, u32 phandle) +{ + const u32 map[] = { 0, phandle, 0, SZ_64K }; + int i, ret; + + fdt_delprop(fdt, node, "msi-map"); + for (i = 0; i < ARRAY_SIZE(map); i++) { + ret = fdt_appendprop_u32(fdt, node, "msi-map", map[i]); + if (ret) + return ret; + } + return 0; +} + +int mk_dtb_add_pci(void *fdt) +{ + static const char compatible[] = + MK_PCI_HOST_BRIDGE "\0pci-host-ecam-generic"; + bool msi = IS_ENABLED(CONFIG_MULTIKERNEL_ITS_PROXY) && + its_foreign_doorbell(); + u32 phandle = 0; + int node, ret; + + if (fdt_node_offset_by_compatible(fdt, -1, MK_PCI_HOST_BRIDGE) < 0) + return 0; + + if (msi) { + ret = mk_dtb_add_its_proxy(fdt, &phandle); + if (ret) + return ret; + } else { + pr_warn("No ITS to route MSIs through: PCI devices of the instance get no interrupts\n"); + } + + for (node = fdt_node_offset_by_compatible(fdt, -1, MK_PCI_HOST_BRIDGE); + node >= 0; + node = fdt_node_offset_by_compatible(fdt, node, MK_PCI_HOST_BRIDGE)) { + if (!fdt_getprop(fdt, node, "reg", NULL)) { + pr_warn("A PCI root bus of the instance has no ECAM window\n"); + continue; + } + ret = fdt_setprop(fdt, node, "compatible", compatible, + sizeof(compatible)); + if (!ret && msi) + ret = mk_dtb_set_msi_map(fdt, node, phandle); + if (ret) + return ret; + } + return 0; +} diff --git a/arch/arm64/multikernel/internal.h b/arch/arm64/multikernel/internal.h index 133f491496cc1a..134dc6086913ec 100644 --- a/arch/arm64/multikernel/internal.h +++ b/arch/arm64/multikernel/internal.h @@ -27,6 +27,9 @@ int mk_dtb_copy_of_node(void *fdt, struct device_node *np); /* Granted platform devices, and the SPIs the GIC node grants with them */ int mk_dtb_add_devices(void *fdt, struct mk_instance *instance); +/* The driver and the MSI controller for the instance's PCI root buses */ +int mk_dtb_add_pci(void *fdt); + /* * The part of the machine every kernel drives for itself: PSCI, the arch * timer and the interrupt controller. Taken from whichever description diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 26aa3b6ec99fd4..3fb64393e51519 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -35,6 +35,7 @@ obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o obj-$(CONFIG_ARM_GIC_V3) += irq-gic-v3.o irq-gic-v3-mbi.o irq-gic-common.o obj-$(CONFIG_ARM_GIC_ITS_PARENT) += irq-gic-its-msi-parent.o obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o irq-gic-v4.o +obj-$(CONFIG_MULTIKERNEL_ITS_PROXY) += irq-gic-v3-its-mk.o obj-$(CONFIG_ARM_GIC_V3_ITS_FSL_MC) += irq-gic-v3-its-fsl-mc-msi.o obj-$(CONFIG_ARM_GIC_V5) += irq-gic-v5.o irq-gic-v5-irs.o irq-gic-v5-its.o \ irq-gic-v5-iwb.o diff --git a/drivers/irqchip/irq-gic-v3-its-mk.c b/drivers/irqchip/irq-gic-v3-its-mk.c new file mode 100644 index 00000000000000..9f44f395bee267 --- /dev/null +++ b/drivers/irqchip/irq-gic-v3-its-mk.c @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * MSI domain of a multikernel instance on a GICv3 with an ITS. + * + * The ITS belongs to the kernel that spawned this one, so this is what + * is left of an ITS driver when someone else runs the command queue: an + * MSI parent domain on top of the GIC's LPI range that asks the host to + * map each event (mk_msi_proxy_map()) and is told which LPI will arrive. + * The device is programmed from here as usual, with the ITS's doorbell + * as the address and the event as the data. + * + * The redistributors of this kernel's CPUs still run on the host's LPI + * tables, which the host enabled before it gave the CPUs away and which + * nothing can take back. So the host also owns the configuration bytes, + * and an LPI stays enabled from the moment it is mapped: masking is left + * to the MSI capability of the device. + */ + +#define pr_fmt(fmt) "ITS-mk: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "irq-gic-its-msi-parent.h" +#include + +#define MK_ITS_COMPATIBLE "multikernel,gic-v3-its" + +struct mk_its_device { + u32 domain; + u32 rid; + u32 nvecs; + unsigned long *events; +}; + +struct mk_its_irq { + struct mk_its_device *dev; + u32 event; +}; + +static phys_addr_t mk_its_doorbell; + +static void mk_its_nop(struct irq_data *d) +{ +} + +static int mk_its_route(struct mk_its_irq *irq, unsigned int cpu, bool wait) +{ + struct mk_its_device *dev = irq->dev; + + return mk_msi_proxy_map(dev->domain, dev->rid, irq->event, dev->nvecs, + cpu_logical_map(cpu), wait); +} + +/* Called with the descriptor locked: the move is sent, not waited for */ +static int mk_its_set_affinity(struct irq_data *d, const struct cpumask *mask, + bool force) +{ + unsigned int cpu = cpumask_first_and(mask, cpu_online_mask); + int ret; + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + ret = mk_its_route(irq_data_get_irq_chip_data(d), cpu, false); + if (ret) + return ret; + + irq_data_update_effective_affinity(d, cpumask_of(cpu)); + return IRQ_SET_MASK_OK_DONE; +} + +static void mk_its_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct mk_its_irq *irq = irq_data_get_irq_chip_data(d); + + msg->address_lo = lower_32_bits(mk_its_doorbell); + msg->address_hi = upper_32_bits(mk_its_doorbell); + msg->data = irq->event; +} + +static struct irq_chip mk_its_chip = { + .name = "ITS-mk", + .irq_mask = mk_its_nop, + .irq_unmask = mk_its_nop, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = mk_its_set_affinity, + .irq_compose_msi_msg = mk_its_compose_msi_msg, +}; + +static int mk_its_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *info) +{ + struct mk_its_device *mdev; + + if (!dev_is_pci(dev)) + return -EINVAL; + + mdev = kzalloc_obj(*mdev); + if (!mdev) + return -ENOMEM; + + mdev->nvecs = roundup_pow_of_two(nvec); + mdev->events = bitmap_zalloc(mdev->nvecs, GFP_KERNEL); + if (!mdev->events) { + kfree(mdev); + return -ENOMEM; + } + mdev->domain = pci_domain_nr(to_pci_dev(dev)->bus); + mdev->rid = pci_dev_id(to_pci_dev(dev)); + + info->scratchpad[0].ptr = mdev; + return 0; +} + +static void mk_its_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *info) +{ + struct mk_its_device *mdev = info->scratchpad[0].ptr; + + bitmap_free(mdev->events); + kfree(mdev); +} + +static struct msi_domain_ops mk_its_msi_domain_ops = { + .msi_prepare = mk_its_msi_prepare, + .msi_teardown = mk_its_msi_teardown, +}; + +static int mk_its_alloc_one(struct irq_domain *domain, unsigned int virq, + struct mk_its_device *mdev, u32 event) +{ + struct irq_fwspec fwspec = { + .fwnode = domain->parent->fwnode, + .param_count = 3, + .param = { GIC_IRQ_TYPE_LPI, 0, IRQ_TYPE_EDGE_RISING }, + }; + unsigned int cpu = cpumask_first(cpu_online_mask); + struct mk_its_irq *irq; + struct irq_data *irqd; + int lpi, ret; + + irq = kzalloc_obj(*irq); + if (!irq) + return -ENOMEM; + irq->dev = mdev; + irq->event = event; + + lpi = mk_its_route(irq, cpu, true); + if (lpi < 0) { + kfree(irq); + return lpi; + } + + fwspec.param[1] = lpi; + ret = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); + if (ret) { + kfree(irq); + return ret; + } + + irq_domain_set_hwirq_and_chip(domain, virq, lpi, &mk_its_chip, irq); + irqd = irq_get_irq_data(virq); + irqd_set_single_target(irqd); + irq_data_update_effective_affinity(irqd, cpumask_of(cpu)); + return 0; +} + +static int mk_its_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + msi_alloc_info_t *info = args; + struct mk_its_device *mdev = info->scratchpad[0].ptr; + unsigned int i; + int first, ret; + + first = bitmap_find_free_region(mdev->events, mdev->nvecs, + get_count_order(nr_irqs)); + if (first < 0) + return -ENOSPC; + + for (i = 0; i < nr_irqs; i++) { + ret = mk_its_alloc_one(domain, virq + i, mdev, first + i); + if (ret) + return ret; + } + return 0; +} + +/* The host keeps the mapping until this kernel is gone; the event is reusable */ +static void mk_its_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + unsigned int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + struct mk_its_irq *irq = irq_data_get_irq_chip_data(d); + + if (irq) { + bitmap_clear(irq->dev->events, irq->event, 1); + kfree(irq); + } + irq_domain_reset_irq_data(d); + } + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static const struct irq_domain_ops mk_its_domain_ops = { + .select = msi_lib_irq_domain_select, + .alloc = mk_its_domain_alloc, + .free = mk_its_domain_free, +}; + +static struct msi_domain_info mk_its_msi_info = { + .ops = &mk_its_msi_domain_ops, +}; + +/* + * No "interrupt-controller" property, so of_irq_init() does not come by. + * Any time after the GIC and before the first PCI host bridge will do. + */ +static int __init mk_its_init(void) +{ + struct irq_domain_info info = { + .ops = &mk_its_domain_ops, + .host_data = &mk_its_msi_info, + }; + struct device_node *np, *gic; + struct resource res; + int ret = -ENODEV; + + np = of_find_compatible_node(NULL, NULL, MK_ITS_COMPATIBLE); + if (!np) + return 0; + + gic = of_irq_find_parent(np); + info.parent = gic ? irq_find_host(gic) : NULL; + of_node_put(gic); + + if (!info.parent || of_address_to_resource(np, 0, &res)) + goto out; + mk_its_doorbell = res.start; + + info.fwnode = of_fwnode_handle(np); + ret = msi_create_parent_irq_domain(&info, &gic_v3_its_msi_parent_ops) ? + 0 : -ENOMEM; +out: + if (ret) + pr_err("%pOF: no MSI domain: %d\n", np, ret); + else + pr_info("MSIs through the host's ITS, doorbell %pa\n", + &mk_its_doorbell); + of_node_put(np); + return ret; +} +arch_initcall(mk_its_init); diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 36b2b08ac68d20..bcf07fd42d2a92 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -175,6 +175,24 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); +/* Spawn side of MK_IO_MSI_MAP: returns the interrupt number or -errno; @wait false when atomic */ +int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, + mk_phys_cpu_t phys_cpu, bool wait); + +/* + * Host side: drop every MSI routed for @instance. Its kernel is gone or + * about to be started afresh, in order or not, and its devices may be + * bound here next. + */ +struct mk_instance; +#ifdef CONFIG_MULTIKERNEL_ITS_PROXY +void mk_msi_proxy_release(struct mk_instance *instance); +#else +static inline void mk_msi_proxy_release(struct mk_instance *instance) +{ +} +#endif + /* Discard everything queued in this kernel's ring (instance re-spawn) */ void mk_ipi_ring_drop_pending(void); @@ -198,6 +216,8 @@ void mk_ipi_ring_drop_pending(void); #define MK_IO_IRQ_BALANCE (MK_MSG_IO + 2) #define MK_IO_IRQ_MASK (MK_MSG_IO + 3) #define MK_IO_IRQ_UNMASK (MK_MSG_IO + 4) +#define MK_IO_MSI_MAP (MK_MSG_IO + 5) /* Route a device's MSI to one of my CPUs */ +#define MK_IO_MSI_ACK (MK_MSG_IO + 6) /* Answer: the interrupt number, or an error */ /* Resource management subtypes */ #define MK_RES_CPU_ADD (MK_MSG_RESOURCE + 1) @@ -290,6 +310,24 @@ struct mk_device_resource_payload { #define MK_DEVICE_BIND_VFIO 0x01 /* Bind to VFIO driver */ #define MK_DEVICE_BIND_STUB 0x02 /* Bind to pci-stub driver */ +/* + * MSI routing request. Where MSIs go through a translation unit only one + * kernel can program (a GICv3 ITS), a spawn asks the kernel that owns it. + * @rid is the device as the spawn sees it on its PCI bus, @event the MSI + * data it will write. Mapping an event again moves it to @phys_cpu, and + * everything is dropped by the host when the instance is started or torn + * down, so there is no unmap. + */ +struct mk_io_msi_payload { + u32 domain; /* PCI domain */ + u32 rid; /* bus << 8 | devfn */ + u32 event; + u32 nvecs; /* Events the device may use, fixed by the first request */ + u64 phys_cpu; /* Target CPU, one of the sender's */ + int result; /* ACK: interrupt number (LPI INTID), or -errno */ + int sender_instance_id; +}; + /* Resource operation response/ACK */ struct mk_resource_ack { u32 operation; /* Original operation (MK_RES_CPU_ADD, etc.) */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 3641b022311248..b937e7c4738388 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1754,6 +1754,9 @@ int multikernel_kexec_by_id(int mk_id) */ mk_ipi_ring_drop_pending(); + /* A previous run that died without a word left its MSIs routed */ + mk_msi_proxy_release(instance); + rc = mk_arch_spawn_instance(mk_image, instance, cpu); if (rc == 0) { rc = mk_instance_set_kexec_active(mk_image->mk_id); diff --git a/kernel/multikernel/Makefile b/kernel/multikernel/Makefile index cf2e1391038ec7..9130711c84eb6c 100644 --- a/kernel/multikernel/Makefile +++ b/kernel/multikernel/Makefile @@ -5,6 +5,7 @@ obj-y += core.o cpuset.o mem.o contig.o kernfs.o dts.o instance_dt.o manifest.o ipi.o messaging.o overlay.o hotplug.o baseline.o obj-$(CONFIG_OF) += of_devices.o +obj-$(CONFIG_MULTIKERNEL_ITS_PROXY) += its_proxy.o # DMA-BUF heap for multikernel memory allocation obj-$(CONFIG_DMABUF_HEAPS) += dma_heap.o diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 4554a412bd2800..adf35486eaae1a 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -1605,6 +1605,7 @@ static void mk_instance_settle_halted(struct mk_instance *instance) { pr_info("Instance %d (%s) halted, CPUs parking in pool\n", instance->id, instance->name); + mk_msi_proxy_release(instance); mk_instance_set_state(instance, MK_STATE_LOADED); } @@ -1749,6 +1750,7 @@ int multikernel_halt_by_id(int mk_id) pr_warn("Multikernel instance %d halted with CPUs unaccounted for\n", mk_id); + mk_msi_proxy_release(instance); mk_instance_set_state(instance, MK_STATE_LOADED); pr_info("Multikernel instance %d halted (graceful)\n", mk_id); } diff --git a/kernel/multikernel/its_proxy.c b/kernel/multikernel/its_proxy.c new file mode 100644 index 00000000000000..b3e6ecdf1d2ebe --- /dev/null +++ b/kernel/multikernel/its_proxy.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * MSIs for an instance's PCI devices, through the host's ITS. + * + * A GICv3 ITS has a single command queue, which belongs to the kernel + * that booted the machine. An instance therefore cannot map the MSIs of + * the devices it was given, and asks over the message ring instead: + * MK_IO_MSI_MAP names a device, an event and one of the instance's CPUs, + * the host programs its ITS and answers with the LPI that will arrive + * there. The instance writes address and data into the device itself, as + * it would with an ITS of its own. + * + * Both ends live here. The ITS side of the host is its_foreign_*() in the + * ITS driver, the interrupt domain of the instance is irq-gic-v3-its-mk. + */ + +#define pr_fmt(fmt) "mk_its: " fmt + +#include +#include +#include +#include +#include +#include + +#define MK_MSI_TIMEOUT_MS 5000 + +struct mk_msi_work { + struct work_struct work; + struct mk_io_msi_payload req; +}; + +static u64 mk_msi_key(const struct mk_io_msi_payload *p) +{ + return (u64)p->domain << 48 | (u64)p->rid << 32 | p->event; +} + +/* Only a device the instance was given, only a CPU it owns */ +static int mk_msi_map_for(struct mk_instance *instance, + const struct mk_io_msi_payload *req) +{ + struct mk_pci_device *mine; + struct pci_dev *pdev; + bool owned = false; + int cpu, ret; + + list_for_each_entry(mine, &instance->pci_devices, list) { + if (mine->domain == req->domain && mine->bus == req->rid >> 8 && + PCI_DEVFN(mine->slot, mine->func) == (req->rid & 0xff)) { + owned = true; + break; + } + } + if (!owned || !mk_cpu_set_contains(instance->cpus, req->phys_cpu)) + return -EPERM; + + cpu = arch_cpu_from_physical_id(req->phys_cpu); + if (cpu < 0) + return -EINVAL; + + pdev = pci_get_domain_bus_and_slot(req->domain, req->rid >> 8, + req->rid & 0xff); + if (!pdev) + return -ENODEV; + + ret = its_foreign_map(instance, + pci_msi_domain_get_msi_rid(dev_get_msi_domain(&pdev->dev), + pdev), + req->event, req->nvecs, cpu); + pci_dev_put(pdev); + return ret; +} + +/* The ITS driver sleeps, and messages arrive in interrupt context */ +static void mk_msi_map_work(struct work_struct *work) +{ + struct mk_msi_work *w = container_of(work, struct mk_msi_work, work); + struct mk_io_msi_payload *req = &w->req; + struct mk_instance *instance; + int to = req->sender_instance_id; + + instance = mk_instance_find(to); + req->result = instance ? mk_msi_map_for(instance, req) : -ENOENT; + if (instance) + mk_instance_put(instance); + + if (req->result < 0) + pr_err("Instance %d: no MSI for %04x:%02x:%02x.%x event %u: %d\n", + to, req->domain, req->rid >> 8, PCI_SLOT(req->rid), + PCI_FUNC(req->rid), req->event, req->result); + + req->sender_instance_id = mk_self->id; + mk_send_message(to, MK_MSG_IO, MK_IO_MSI_ACK, req, sizeof(*req)); + kfree(w); +} + +static void mk_msi_msg_handler(u32 msg_type, u32 subtype, void *payload, + u32 payload_len, void *ctx) +{ + struct mk_io_msi_payload *p = payload; + struct mk_msi_work *w; + + if (payload_len < sizeof(*p)) + return; + + switch (subtype) { + case MK_IO_MSI_MAP: + w = kzalloc(sizeof(*w), GFP_ATOMIC); + if (!w) + return; + w->req = *p; + INIT_WORK(&w->work, mk_msi_map_work); + schedule_work(&w->work); + break; + case MK_IO_MSI_ACK: + mk_msg_pending_complete(MK_MSG_IO, MK_IO_MSI_MAP, mk_msi_key(p), + p->result); + break; + } +} + +/** + * mk_msi_proxy_map - Ask the host to route a device's MSI to a CPU + * @domain: PCI domain of the device + * @rid: bus << 8 | devfn + * @event: MSI data the device will write + * @nvecs: events the device may use + * @phys_cpu: target, one of this kernel's CPUs + * @wait: wait for the answer; false moves an event that is mapped + * already, from a context that cannot sleep + * + * Returns the interrupt number (0 when not waiting) or a negative error. + */ +int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, + mk_phys_cpu_t phys_cpu, bool wait) +{ + struct mk_io_msi_payload req = { + .domain = domain, + .rid = rid, + .event = event, + .nvecs = nvecs, + .phys_cpu = phys_cpu, + .sender_instance_id = mk_self->id, + }; + struct mk_pending_msg *pending = NULL; + int ret; + + if (wait) { + pending = mk_msg_pending_add(MK_MSG_IO, MK_IO_MSI_MAP, + mk_msi_key(&req)); + if (!pending) + return -ENOMEM; + } + + ret = mk_send_message(0, MK_MSG_IO, MK_IO_MSI_MAP, &req, sizeof(req)); + if (!wait) + return ret; + if (ret < 0) { + mk_msg_pending_wait(pending, 0); + return ret; + } + return mk_msg_pending_wait(pending, MK_MSI_TIMEOUT_MS); +} + +void mk_msi_proxy_release(struct mk_instance *instance) +{ + its_foreign_release(instance); +} + +static int __init mk_msi_proxy_init(void) +{ + return mk_register_msg_handler(MK_MSG_IO, mk_msi_msg_handler, NULL); +} +fs_initcall(mk_msi_proxy_init); From 69356717da187da29c5b8230dc028d8d4b859bbc Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 18:39:18 -0700 Subject: [PATCH 14/30] arm64/multikernel: Force a kernel down with the stop IPI x86 forces a wedged spawn down with an NMI per APIC ID. PSCI has no call to turn another CPU off, so arm64 has to reach the CPU in band and have it turn itself off. The stop IPI already does that in a spawn: IPI_CPU_STOP lands in local_cpu_stop(), which returns the CPU to firmware there. So mk_force_stop_cpu() sends IPI_CPU_STOP_NMI to the MPIDR, and the generic force halt, which arms its marker in the shared ring area first, works unchanged. A kernel running with pseudo-NMIs takes that SGI with interrupts masked; to any other kernel it is an ordinary interrupt, and a CPU spinning with interrupts off is out of reach. That case is not silent: the next exec finds the CPU still on through AFFINITY_INFO and refuses to reload the image under it. SGIs cross kernel boundaries, so the receiving side now tells its own stop from a foreign one: smp_send_stop() and the crash path mark theirs, and anything else is obeyed only if the force-halt marker is up, which only a kernel with an instance relationship can set. x86 filters stop NMIs it did not ask for the same way. Obeying means turning the CPU off, in a host that is being fenced as well, not parking it in an image that is about to be replaced. Tested on QEMU virt with a spawn CPU spinning in lkdtm's HARDLOCKUP: with irqchip.gicv3_pseudo_nmi=1 in the spawn both CPUs go off and the next exec boots; without it the exec is refused with the CPU named. SDEI, which would reach a CPU regardless of how its kernel was booted, is not implemented: it takes an SDEI_EVENT_SIGNAL call the firmware driver does not have yet and TF-A to test against. Signed-off-by: Cong Wang --- arch/arm64/include/asm/multikernel.h | 7 +++++++ arch/arm64/kernel/smp.c | 17 ++++++++++++++- arch/arm64/multikernel/spawn.c | 31 ++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/multikernel.h b/arch/arm64/include/asm/multikernel.h index a7b0c93950a457..dde511c01b3cc8 100644 --- a/arch/arm64/include/asm/multikernel.h +++ b/arch/arm64/include/asm/multikernel.h @@ -40,6 +40,9 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) void mk_spawn_machine_halt(void); void mk_spawn_stop_this_cpu(void); +/* A stop IPI from another kernel: its force halt, or not ours to obey */ +void mk_foreign_cpu_stop(void); + /* The boot tree is the manifest when another kernel wrote it */ void mk_spawn_accept_boot_tree(phys_addr_t dt_phys); #else @@ -54,6 +57,10 @@ static inline void mk_spawn_machine_halt(void) static inline void mk_spawn_stop_this_cpu(void) { } + +static inline void mk_foreign_cpu_stop(void) +{ +} #endif /* diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 7edddcfd719cc0..fe0383e9e6bbfa 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -80,6 +80,8 @@ static DEFINE_PER_CPU_READ_MOSTLY(struct ipi_descs, pcpu_ipi_desc); static bool percpu_ipi_descs __ro_after_init; static bool crash_stop; +/* Set once this kernel starts stopping its own CPUs */ +static unsigned long stop_in_progress; static void ipi_setup(int cpu); @@ -959,6 +961,16 @@ void kgdb_roundup_cpus(void) } #endif +/* + * SGIs cross kernel boundaries, so with multikernel a stop IPI that this + * kernel did not send itself comes from another kernel. + */ +static bool ipi_cpu_stop_is_foreign(void) +{ + return IS_ENABLED(CONFIG_MULTIKERNEL) && + !READ_ONCE(stop_in_progress) && !crash_stop; +} + /* * Main handler for inter-processor interrupts */ @@ -980,6 +992,10 @@ static void do_handle_IPI(int ipinr) case IPI_CPU_STOP: case IPI_CPU_STOP_NMI: + if (ipi_cpu_stop_is_foreign()) { + mk_foreign_cpu_stop(); + break; + } if (IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop) { ipi_cpu_crash_stop(cpu, get_irq_regs()); unreachable(); @@ -1196,7 +1212,6 @@ static inline unsigned int num_other_online_cpus(void) void smp_send_stop(void) { - static unsigned long stop_in_progress; static cpumask_t mask; unsigned long timeout; diff --git a/arch/arm64/multikernel/spawn.c b/arch/arm64/multikernel/spawn.c index 3d97db304ce643..f2ca53b96bb6bb 100644 --- a/arch/arm64/multikernel/spawn.c +++ b/arch/arm64/multikernel/spawn.c @@ -105,8 +105,39 @@ static int __init mk_spawn_reboot_init(void) } core_initcall(mk_spawn_reboot_init); +/* + * PSCI cannot turn another CPU off, so a kernel that no longer answers + * is stopped in band, with the stop IPI its own smp_send_stop() would + * use. It is sent as IPI_CPU_STOP_NMI: a kernel running with pseudo-NMIs + * takes that one even with interrupts masked, and to any other kernel it + * is an ordinary SGI, which a CPU spinning with interrupts off never + * sees. Without SDEI that is as far as force goes on arm64. + */ void mk_force_stop_cpu(mk_phys_cpu_t phys_cpu) { + int ret = gic_v3_send_sgi_to_mpidr(phys_cpu, IPI_CPU_STOP_NMI); + + if (ret) + pr_err("No stop SGI to CPU 0x%llx: %d\n", phys_cpu, ret); +} + +/* + * The receiving end. A force halt is announced in the shared ring area + * before the SGIs go out, and only a kernel this one has a relationship + * with can write there; a stop from anyone else is ignored, the way x86 + * drops a stop NMI it did not ask for. Obeying means leaving for good: + * the CPU is turned off, in a host that is being fenced as well. + */ +void mk_foreign_cpu_stop(void) +{ + if (!mk_has_pending_shutdown()) { + pr_warn_ratelimited("CPU%u: ignoring a stop IPI from another kernel\n", + smp_processor_id()); + return; + } + + set_cpu_online(smp_processor_id(), false); + mk_enter_pool_state(NULL); } static struct mk_cpu_set *mk_started_set(struct mk_instance *instance) From d26f1f0481ac7ca63c29cacccb857f10218015f2 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 18:46:27 -0700 Subject: [PATCH 15/30] multikernel: Confine the DMA of an instance's devices to its memory An instance drives its PCI devices without an IOMMU of its own. On arm64 the SMMU is shared and stays with the host, the instance tree does not mention it, and the instance hands physical addresses to its devices. Behind a translating SMMU that DMA faults, which is what a spawn's virtio NIC does on QEMU virt with iommu=smmuv3: F_TRANSLATION for its stream ID on the first buffer. Behind a bypassing one it works, and a buggy driver in the instance can reach all of the host. While an instance runs, attach each of its devices to a paging domain of its own that maps the instance's memory grant onto itself, plus the page of the MSI doorbell, which a device behind the SMMU cannot write to otherwise. The device works untranslated and reaches nothing else. This is the generic IOMMU API, the way VFIO would do it: claim DMA ownership, which fails while a host driver is bound, allocate, map, attach. The domain is built on every exec, since the grant can change between load and exec, and an instance whose devices cannot be confined is not started. It follows the grant of a running instance: memory is mapped before the instance is told about it and unmapped once the instance has let go of it. It goes away when the instance halts, however it halts, which returns the device to the host's default domain for a host driver to bind. A device without an IOMMU is left alone, its DMA as unrestricted as it always was. Only arm64 gets this for now: x86 runs instance devices without an IOMMU, and changing that is a change of its own. arm64 also marks a root bus dma-coherent in the instance tree when the host treats the devices behind it that way. Without the property every device of the instance is non-coherent, which is correct on SBSA hardware but pays for cache maintenance nobody needs. On QEMU virt with an SMMUv3 the NIC of a spawn now gets its lease and pings with no SMMU event logged, the host binds and uses it after a halt, and the next spawn takes it again. Signed-off-by: Cong Wang --- arch/arm64/multikernel/dtb_pci.c | 26 ++++ include/linux/multikernel.h | 34 +++++ kernel/kexec_core.c | 9 ++ kernel/multikernel/Kconfig | 5 + kernel/multikernel/Makefile | 1 + kernel/multikernel/core.c | 2 + kernel/multikernel/hotplug.c | 16 ++- kernel/multikernel/iommu.c | 224 +++++++++++++++++++++++++++++++ kernel/multikernel/its_proxy.c | 5 + 9 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 kernel/multikernel/iommu.c diff --git a/arch/arm64/multikernel/dtb_pci.c b/arch/arm64/multikernel/dtb_pci.c index dc9e58c4f5d457..bf6aef796d4093 100644 --- a/arch/arm64/multikernel/dtb_pci.c +++ b/arch/arm64/multikernel/dtb_pci.c @@ -13,8 +13,10 @@ #define pr_fmt(fmt) "mk_dtb: " fmt +#include #include #include +#include #include #include "internal.h" @@ -61,6 +63,28 @@ static int mk_dtb_set_msi_map(void *fdt, int node, u32 phandle) return 0; } +/* + * Devices inherit DMA coherency from their host bridge node, and a tree + * that does not say so makes every device behind it non-coherent. Ask a + * device on the host's side of the same root bus. + */ +static bool mk_dtb_root_bus_coherent(void *fdt, int node) +{ + const fdt32_t *domain = fdt_getprop(fdt, node, "linux,pci-domain", NULL); + const fdt32_t *range = fdt_getprop(fdt, node, "bus-range", NULL); + struct pci_bus *bus; + struct pci_dev *pdev; + + if (!range) + return false; + + bus = pci_find_bus(domain ? fdt32_to_cpu(*domain) : 0, + fdt32_to_cpu(range[0])); + pdev = bus ? list_first_entry_or_null(&bus->devices, struct pci_dev, + bus_list) : NULL; + return pdev && dev_is_dma_coherent(&pdev->dev); +} + int mk_dtb_add_pci(void *fdt) { static const char compatible[] = @@ -92,6 +116,8 @@ int mk_dtb_add_pci(void *fdt) sizeof(compatible)); if (!ret && msi) ret = mk_dtb_set_msi_map(fdt, node, phandle); + if (!ret && mk_dtb_root_bus_coherent(fdt, node)) + ret = fdt_setprop_empty(fdt, node, "dma-coherent"); if (ret) return ret; } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index bcf07fd42d2a92..b84a429a0fa8d9 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -187,10 +187,44 @@ int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, struct mk_instance; #ifdef CONFIG_MULTIKERNEL_ITS_PROXY void mk_msi_proxy_release(struct mk_instance *instance); +/* The address an instance's devices write their MSIs to, 0 if none */ +phys_addr_t mk_msi_proxy_doorbell(void); #else static inline void mk_msi_proxy_release(struct mk_instance *instance) { } +static inline phys_addr_t mk_msi_proxy_doorbell(void) +{ + return 0; +} +#endif + +/* + * Host side: keep the DMA of an instance's PCI devices inside its memory + * while it runs, where the devices sit behind an IOMMU. + */ +#ifdef CONFIG_MULTIKERNEL_IOMMU +int mk_iommu_contain(struct mk_instance *instance); +void mk_iommu_release(struct mk_instance *instance); +int mk_iommu_map(struct mk_instance *instance, phys_addr_t start, size_t size); +void mk_iommu_unmap(struct mk_instance *instance, phys_addr_t start, size_t size); +#else +static inline int mk_iommu_contain(struct mk_instance *instance) +{ + return 0; +} +static inline void mk_iommu_release(struct mk_instance *instance) +{ +} +static inline int mk_iommu_map(struct mk_instance *instance, + phys_addr_t start, size_t size) +{ + return 0; +} +static inline void mk_iommu_unmap(struct mk_instance *instance, + phys_addr_t start, size_t size) +{ +} #endif /* Discard everything queued in this kernel's ring (instance re-spawn) */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index b937e7c4738388..67d07c152e5d75 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1757,7 +1757,16 @@ int multikernel_kexec_by_id(int mk_id) /* A previous run that died without a word left its MSIs routed */ mk_msi_proxy_release(instance); + rc = mk_iommu_contain(instance); + if (rc) { + pr_err("Instance %d: devices not confined to its memory: %d\n", + mk_id, rc); + goto unlock; + } + rc = mk_arch_spawn_instance(mk_image, instance, cpu); + if (rc) + mk_iommu_release(instance); if (rc == 0) { rc = mk_instance_set_kexec_active(mk_image->mk_id); if (rc) diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index b548f14be847e0..7cda3199c2c956 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -37,3 +37,8 @@ config MULTIKERNEL # Map the MSIs of an instance's PCI devices through the host's ITS config MULTIKERNEL_ITS_PROXY def_bool MULTIKERNEL && ARM_GIC_V3_ITS && PCI_MSI + +# Confine the DMA of an instance's PCI devices to its memory. x86 runs +# them without an IOMMU so far. +config MULTIKERNEL_IOMMU + def_bool MULTIKERNEL && IOMMU_API && PCI && ARM64 diff --git a/kernel/multikernel/Makefile b/kernel/multikernel/Makefile index 9130711c84eb6c..8ecf6d9f7ac647 100644 --- a/kernel/multikernel/Makefile +++ b/kernel/multikernel/Makefile @@ -6,6 +6,7 @@ obj-y += core.o cpuset.o mem.o contig.o kernfs.o dts.o instance_dt.o manifest.o ipi.o messaging.o overlay.o hotplug.o baseline.o obj-$(CONFIG_OF) += of_devices.o obj-$(CONFIG_MULTIKERNEL_ITS_PROXY) += its_proxy.o +obj-$(CONFIG_MULTIKERNEL_IOMMU) += iommu.o # DMA-BUF heap for multikernel memory allocation obj-$(CONFIG_DMABUF_HEAPS) += dma_heap.o diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index adf35486eaae1a..3d3b57c4b5397e 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -1606,6 +1606,7 @@ static void mk_instance_settle_halted(struct mk_instance *instance) pr_info("Instance %d (%s) halted, CPUs parking in pool\n", instance->id, instance->name); mk_msi_proxy_release(instance); + mk_iommu_release(instance); mk_instance_set_state(instance, MK_STATE_LOADED); } @@ -1751,6 +1752,7 @@ int multikernel_halt_by_id(int mk_id) mk_id); mk_msi_proxy_release(instance); + mk_iommu_release(instance); mk_instance_set_state(instance, MK_STATE_LOADED); pr_info("Multikernel instance %d halted (graceful)\n", mk_id); } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index f393b368da149a..3f84e20f40defc 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -1455,16 +1455,29 @@ int mk_send_mem_add(int instance_id, u64 start_pfn, u64 nr_pages, goto out; } + /* Its devices must reach the memory by the time the instance uses it */ + ret = mk_iommu_map(target_instance, PFN_PHYS(start_pfn), + PFN_PHYS(nr_pages)); + if (ret < 0) { + mk_msg_pending_wait(pending, 0); + goto out; + } + ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_MEM_ADD, &payload, sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ + mk_iommu_unmap(target_instance, PFN_PHYS(start_pfn), + PFN_PHYS(nr_pages)); goto out; } ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) + if (ret < 0) { + mk_iommu_unmap(target_instance, PFN_PHYS(start_pfn), + PFN_PHYS(nr_pages)); goto out; + } ret = mk_instance_add_memory_region(target_instance, PFN_PHYS(nr_pages), (int)numa_node); @@ -1541,6 +1554,7 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) if (ret < 0) goto abort; + mk_iommu_unmap(target_instance, PFN_PHYS(start_pfn), PFN_PHYS(nr_pages)); mk_instance_finish_memory_removal(target_instance, PFN_PHYS(start_pfn), PFN_PHYS(nr_pages)); goto out; diff --git a/kernel/multikernel/iommu.c b/kernel/multikernel/iommu.c new file mode 100644 index 00000000000000..88396447ac0418 --- /dev/null +++ b/kernel/multikernel/iommu.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * DMA containment for the PCI devices of an instance. + * + * An instance drives its devices without an IOMMU of its own: the IOMMU + * stays with the host, the instance tree does not mention it, and the + * instance hands physical addresses to its devices. Behind a translating + * IOMMU that DMA faults, and behind a bypassing one a buggy driver in the + * instance can reach all of the host. + * + * So while an instance runs, each of its devices is attached to a domain + * of its own that maps the instance's memory grant onto itself, and the + * MSI doorbell with it. The device works untranslated, and cannot reach + * anything else. The domain follows the grant as memory is added and + * removed, and goes away when the instance halts, which returns the + * device to the host's default domain for a host driver to bind. + * + * A device without an IOMMU is left alone: its DMA is unrestricted, as + * it always was. + */ + +#define pr_fmt(fmt) "mk_iommu: " fmt + +#include +#include +#include +#include +#include + +struct mk_iommu_dev { + struct list_head list; + struct mk_instance *instance; + struct pci_dev *pdev; + struct iommu_domain *domain; +}; + +static LIST_HEAD(mk_iommu_devs); +static DEFINE_MUTEX(mk_iommu_lock); + +static int mk_iommu_prot(struct mk_iommu_dev *d) +{ + return IOMMU_READ | IOMMU_WRITE | + (dev_is_dma_coherent(&d->pdev->dev) ? IOMMU_CACHE : 0); +} + +static void mk_iommu_free_dev(struct mk_iommu_dev *d) +{ + if (d->domain) { + iommu_detach_device(d->domain, &d->pdev->dev); + iommu_domain_free(d->domain); + } + iommu_device_release_dma_owner(&d->pdev->dev); + pci_dev_put(d->pdev); + list_del(&d->list); + kfree(d); +} + +static int mk_iommu_contain_dev(struct mk_instance *instance, + struct pci_dev *pdev, phys_addr_t doorbell) +{ + struct mk_memory_region *region; + struct mk_iommu_dev *d; + int ret; + + d = kzalloc_obj(*d); + if (!d) + return -ENOMEM; + d->instance = instance; + d->pdev = pci_dev_get(pdev); + list_add(&d->list, &mk_iommu_devs); + + /* Fails while a host driver is bound, which must not share the device */ + ret = iommu_device_claim_dma_owner(&pdev->dev, instance); + if (ret) { + pci_dev_put(d->pdev); + list_del(&d->list); + kfree(d); + return ret; + } + + d->domain = iommu_paging_domain_alloc(&pdev->dev); + if (IS_ERR(d->domain)) { + ret = PTR_ERR(d->domain); + d->domain = NULL; + goto err; + } + + list_for_each_entry(region, &instance->memory_regions, list) { + ret = iommu_map(d->domain, region->res.start, region->res.start, + resource_size(®ion->res), mk_iommu_prot(d), + GFP_KERNEL); + if (ret) + goto err_domain; + } + + if (doorbell) { + ret = iommu_map(d->domain, doorbell & PAGE_MASK, + doorbell & PAGE_MASK, PAGE_SIZE, + IOMMU_WRITE | IOMMU_MMIO, GFP_KERNEL); + if (ret) + goto err_domain; + } + + ret = iommu_attach_device(d->domain, &pdev->dev); + if (!ret) + return 0; + +err_domain: + iommu_domain_free(d->domain); + d->domain = NULL; +err: + mk_iommu_free_dev(d); + return ret; +} + +/** + * mk_iommu_contain - Confine the DMA of an instance's devices to its memory + * @instance: Instance about to be started + * + * Returns 0, or a negative error code, in which case nothing is attached + * and the instance must not be started: its devices would either fault + * or roam. + */ +int mk_iommu_contain(struct mk_instance *instance) +{ + phys_addr_t doorbell = mk_msi_proxy_doorbell(); + struct mk_pci_device *mine; + int ret = 0; + + mk_iommu_release(instance); + + if (!instance->pci_devices_valid) + return 0; + + mutex_lock(&mk_iommu_lock); + list_for_each_entry(mine, &instance->pci_devices, list) { + struct pci_dev *pdev; + + pdev = pci_get_domain_bus_and_slot(mine->domain, mine->bus, + PCI_DEVFN(mine->slot, mine->func)); + if (!pdev) + continue; + + if (device_iommu_mapped(&pdev->dev)) + ret = mk_iommu_contain_dev(instance, pdev, doorbell); + else + pr_info_once("%s has no IOMMU, DMA of instance devices is unrestricted\n", + pci_name(pdev)); + if (ret) + pci_err(pdev, "DMA not confined to instance %d: %d\n", + instance->id, ret); + pci_dev_put(pdev); + if (ret) + break; + } + mutex_unlock(&mk_iommu_lock); + + if (ret) + mk_iommu_release(instance); + return ret; +} + +/** + * mk_iommu_release - Return an instance's devices to the host's domain + * @instance: Instance that has halted, or is about to be started afresh + */ +void mk_iommu_release(struct mk_instance *instance) +{ + struct mk_iommu_dev *d, *tmp; + + mutex_lock(&mk_iommu_lock); + list_for_each_entry_safe(d, tmp, &mk_iommu_devs, list) { + if (d->instance == instance) + mk_iommu_free_dev(d); + } + mutex_unlock(&mk_iommu_lock); +} + +/** + * mk_iommu_map - Let a running instance's devices reach memory it gains + * @instance: Instance the memory is being added to + * @start: Physical address of the range + * @size: Size of the range + * + * To be called before the instance is told about the memory. + */ +int mk_iommu_map(struct mk_instance *instance, phys_addr_t start, size_t size) +{ + struct mk_iommu_dev *d; + int ret = 0; + + mutex_lock(&mk_iommu_lock); + list_for_each_entry(d, &mk_iommu_devs, list) { + if (d->instance != instance) + continue; + ret = iommu_map(d->domain, start, start, size, mk_iommu_prot(d), + GFP_KERNEL); + if (ret) + break; + } + mutex_unlock(&mk_iommu_lock); + + if (ret) + mk_iommu_unmap(instance, start, size); + return ret; +} + +/** + * mk_iommu_unmap - Cut a running instance's devices off memory it lost + * @instance: Instance the memory was removed from + * @start: Physical address of the range + * @size: Size of the range + */ +void mk_iommu_unmap(struct mk_instance *instance, phys_addr_t start, size_t size) +{ + struct mk_iommu_dev *d; + + mutex_lock(&mk_iommu_lock); + list_for_each_entry(d, &mk_iommu_devs, list) { + if (d->instance == instance) + iommu_unmap(d->domain, start, size); + } + mutex_unlock(&mk_iommu_lock); +} diff --git a/kernel/multikernel/its_proxy.c b/kernel/multikernel/its_proxy.c index b3e6ecdf1d2ebe..4f4072df72ad16 100644 --- a/kernel/multikernel/its_proxy.c +++ b/kernel/multikernel/its_proxy.c @@ -167,6 +167,11 @@ void mk_msi_proxy_release(struct mk_instance *instance) its_foreign_release(instance); } +phys_addr_t mk_msi_proxy_doorbell(void) +{ + return its_foreign_doorbell(); +} + static int __init mk_msi_proxy_init(void) { return mk_register_msg_handler(MK_MSG_IO, mk_msi_msg_handler, NULL); From 5e4fdb5cc27c6daff429df6e9c29c6b5e56c5e9d Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 19:07:44 -0700 Subject: [PATCH 16/30] Documentation/multikernel: Describe multikernel on arm64 The user interface is the same on both architectures, so usage.rst only gains a pointer. What differs is below it: firmware holds the pool CPUs, the instance tree is the boot tree, the GIC is shared with the host as a tenant, MSIs go through the host's ITS and the SMMU confines a spawn's devices from the host's side. Document that, the device tree properties it introduces, the command line a spawn wants, and what the platform has to provide. Signed-off-by: Cong Wang --- Documentation/multikernel/arm64.rst | 140 ++++++++++++++++++++++++++++ Documentation/multikernel/usage.rst | 7 ++ 2 files changed, 147 insertions(+) create mode 100644 Documentation/multikernel/arm64.rst diff --git a/Documentation/multikernel/arm64.rst b/Documentation/multikernel/arm64.rst new file mode 100644 index 00000000000000..f88eeb9feff241 --- /dev/null +++ b/Documentation/multikernel/arm64.rst @@ -0,0 +1,140 @@ +.. SPDX-License-Identifier: GPL-2.0 + +===================== +Multikernel on arm64 +===================== + +Everything in ``usage.rst``, ``device-tree.rst`` and ``overlays.rst`` +applies unchanged. This document covers what is different below that +interface, what a platform has to provide, and what the host and a spawn +kernel need on their command lines. + +Requirements +============ + +- PSCI 0.2 or later (``CPU_ON``, ``CPU_OFF``, ``AFFINITY_INFO``), as any + SBSA server and QEMU's ``virt`` machine have. Spin-table platforms are + not supported. +- A GICv3. MSIs for PCI devices of an instance need its ITS; systems with + more than one ITS are not supported for that yet. +- ``CONFIG_MULTIKERNEL=y``, which on arm64 needs ``CONFIG_KEXEC_FILE`` and + ``CONFIG_HOTPLUG_CPU``. An instance is loaded from an ``Image`` file with + ``kexec_file_load()``, not from a ``vmlinux``. + +The host may have booted from a device tree or from UEFI and ACPI. A +spawn kernel always boots from a device tree. + +CPUs +==== + +Firmware holds the pool. A pool CPU is a CPU in the PSCI ``OFF`` state: it +executes nothing, and there is no park loop or mailbox in memory. The +host moves a CPU into the pool by offlining it, a spawn is started with +``CPU_ON`` at its Image's entry point with the device tree address as the +context ID, and it brings up its other CPUs with the stock PSCI enable +method. Physical CPU IDs in the multikernel trees are MPIDR affinity +values. + +A spawn kernel that halts, reboots or panics turns every one of its CPUs +off, instead of leaving them in a WFI loop inside an image the host is +about to overwrite. The host learns that a CPU has left from +``AFFINITY_INFO``, and refuses to load a new image over one that has not. + +**Force halt** sends the stop IPI to each CPU of the instance. A spawn +kernel booted with ``irqchip.gicv3_pseudo_nmi=1`` (and built with +``CONFIG_ARM64_PSEUDO_NMI``) takes it as an NMI, so it reaches a CPU that +spins with interrupts masked. Without pseudo-NMIs it is an ordinary +interrupt and such a CPU stays out of reach; the instance then cannot be +started again until the machine is rebooted. SDEI is not used yet. + +The Boot Tree +============= + +An arm64 kernel reads its memory, its CPUs and its interrupt controller +from the tree in ``x0`` long before any multikernel code runs, so the +instance tree of ``device-tree.rst`` and the boot protocol's tree are one +and the same on arm64. The host rebuilds it on every start, from the +instance's current resources, and adds to the instance tree: + +``/memory@`` + One node per range of the instance's memory. + +``/cpus`` + ``enable-method = "psci"`` nodes for the instance's own CPUs, the boot + CPU first, followed by every other CPU of the machine. A kernel can + only ever online a CPU it enumerated at boot; the ones the instance + does not own are possible but not present until they are hot-added. + +``/psci``, ``/timer``, the GICv3 node + Copied from the host's tree, or written from the FADT, GTDT and MADT + on an ACPI host. + +``/chosen`` + ``bootargs``, the initrd and the seeds from the ``kexec_file_load()`` + call, next to the multikernel properties, and + ``linux,pci-probe-only``: BARs and bridge windows remain the host's to + assign. + +There is no UART node. For boot messages give the spawn an explicit +``earlycon``, for example ``earlycon=pl011,mmio32,0x9000000 keep_bootcon`` +on QEMU's ``virt`` machine, which polls the host's UART without taking +its interrupt. + +Interrupts +========== + +The GIC distributor exists once and stays with the host. The GIC node of +a boot tree carries: + +``multikernel,tenant`` + The GICv3 driver leaves the distributor alone: it only checks that it + is enabled with affinity routing, and initialises the redistributors + and CPU interfaces of its own CPUs. + +``multikernel,spis = ...`` + The SPIs the instance may map, as SPI numbers. Everything else fails + with ``-EPERM``. + +The **doorbell** of the message ring is SGI 7. All eight non-secure SGIs +are in use on arm64, so with ``CONFIG_MULTIKERNEL`` it takes the place of +the KGDB roundup IPI, and KGDB falls back to its generic roundup. + +Devices +======= + +Platform devices + Granted by ``device-name``, the name under + ``/sys/bus/platform/devices``, for instance ``a003e00.virtio_mmio``. + The host copies the device's node into the boot tree and grants its + SPIs, so this takes a host that booted from a device tree, and a + device whose node refers to no clocks, resets, power domains or pin + controllers, which stay with the host. Unbind the host's driver first; + that also disables the SPIs on the host's side. + +PCI devices + The root buses above an instance's devices appear as + ``pci-host-ecam-generic`` bridges with the devices as their only + children. INTx is not available. MSIs go through the host's ITS: the + boot tree has a ``multikernel,gic-v3-its`` node whose ``reg`` is the + ITS's doorbell, the spawn asks the host over the message ring to map + each vector, and the host drops the mappings when the instance halts. + Unbind the host's driver before the instance is started, and bind it + again once the instance is stopped if the host should use the device. + +IOMMU + The SMMU stays with the host and is not described to a spawn, which + hands physical addresses to its devices. While an instance runs, each + of its PCI devices behind an SMMU is attached to a domain that maps + exactly the instance's memory, and the MSI doorbell, onto itself. It + follows memory that is added or removed at run time. A device without + an IOMMU has unrestricted DMA. + +Testing under QEMU +================== + +``qemu-system-aarch64 -M virt,gic-version=3`` exercises all of the above +with its built-in PSCI; ``secure=on,virtualization=on`` with TF-A as the +firmware does so against real EL3 firmware and enters the kernels at EL2; +``iommu=smmuv3`` with ``iommu_platform=on,disable-legacy=on`` on a virtio +PCI device puts that device behind an SMMU. A spawn needs no disk: an +initramfs is enough. diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index 8ed85d8ead7e18..7a5749ffa56fa7 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -66,6 +66,13 @@ Workflow ``device-remove``, and destroy the instance with ``instance-remove`` once it is stopped. +Architectures +============= + +x86-64 and arm64 are supported. ``arm64.rst`` describes what differs on +arm64: how CPUs are held and started, the boot tree, interrupts, and what +a device needs to be given to an instance. + Instance States =============== From 8cb345eeef3baa158f5f0e48af912da4a38d6f27 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 19:30:51 -0700 Subject: [PATCH 17/30] multikernel: Route an instance's MSIs through whichever ITS serves the device The MSI routing for instances only worked on a machine with a single ITS: its_foreign_map() took "the" ITS and gave up when there was more than one, and the instance tree carried that one ITS's doorbell as the reg of the proxy node. Servers have an ITS per socket at least, so on those the PCI devices of an instance got no interrupts at all. The ITS behind a device is named by the device's MSI domain, which the host has for every PCI device whether a driver is bound or not. Pass that domain to its_foreign_map() and its_foreign_doorbell() and take the ITS, with its own DeviceID space, collections and lock, from there; a domain that is not an ITS's yields none. The single-ITS shortcut is gone rather than kept as a fast path, so every machine runs the code a multi-ITS machine needs. its_foreign_release() walks all of them. An ITS with erratum 23144 only reaches the CPUs of its own node, which is checked when the collection is picked. Only the host knows which ITS that is, so the doorbell moves from the device tree into the answer: MK_IO_MSI_ACK carries the address next to the LPI, the spawn's MSI domain keeps it per interrupt, and the proxy node loses its reg. That also covers a root bus whose devices sit behind different ITSs, and the pre-ITS doorbell, since the address now comes from the ITS's own get_msi_base(). The generic pending-message helper returns a single int, so the proxy tracks its few in-flight requests itself to get the whole answer back to the waiter. The instance tree now decides per root bus: a bus gets its msi-map when a device on the host's side of it has an ITS behind it, and a warning otherwise. The SMMU containment maps the doorbell of the ITS behind each device instead of a global one. QEMU has no machine with more than one ITS, so that case itself is untested. The PCI smoke test passes on QEMU virt booted from a device tree, under TF-A, from UEFI/ACPI where the domain is found through IORT, and behind an SMMUv3; with its=off the instance boots, its NIC fails to probe for lack of interrupts, and the host is unaffected. Signed-off-by: Cong Wang --- Documentation/multikernel/arm64.rst | 19 +++-- arch/arm64/multikernel/dtb_pci.c | 87 ++++++++++++++--------- drivers/irqchip/irq-gic-v3-its-mk.c | 33 +++++---- drivers/irqchip/irq-gic-v3-its.c | 86 ++++++++++++++--------- include/linux/irqchip/arm-gic-v3.h | 7 +- include/linux/multikernel.h | 24 ++++--- kernel/multikernel/iommu.c | 17 ++--- kernel/multikernel/its_proxy.c | 103 +++++++++++++++++++++------- 8 files changed, 242 insertions(+), 134 deletions(-) diff --git a/Documentation/multikernel/arm64.rst b/Documentation/multikernel/arm64.rst index f88eeb9feff241..be8e05121cf1a3 100644 --- a/Documentation/multikernel/arm64.rst +++ b/Documentation/multikernel/arm64.rst @@ -15,8 +15,8 @@ Requirements - PSCI 0.2 or later (``CPU_ON``, ``CPU_OFF``, ``AFFINITY_INFO``), as any SBSA server and QEMU's ``virt`` machine have. Spin-table platforms are not supported. -- A GICv3. MSIs for PCI devices of an instance need its ITS; systems with - more than one ITS are not supported for that yet. +- A GICv3. MSIs for PCI devices of an instance need an ITS behind those + devices; a machine may have any number of them. - ``CONFIG_MULTIKERNEL=y``, which on arm64 needs ``CONFIG_KEXEC_FILE`` and ``CONFIG_HOTPLUG_CPU``. An instance is loaded from an ``Image`` file with ``kexec_file_load()``, not from a ``vmlinux``. @@ -114,10 +114,14 @@ Platform devices PCI devices The root buses above an instance's devices appear as ``pci-host-ecam-generic`` bridges with the devices as their only - children. INTx is not available. MSIs go through the host's ITS: the - boot tree has a ``multikernel,gic-v3-its`` node whose ``reg`` is the - ITS's doorbell, the spawn asks the host over the message ring to map - each vector, and the host drops the mappings when the instance halts. + children. INTx is not available. MSIs go through the host's ITSs: the + boot tree has a ``multikernel,gic-v3-its`` node, the spawn asks the + host over the message ring to map each vector, and the host answers + with the LPI that will arrive and the doorbell address to program into + the device. The host picks the ITS behind each device, so nothing about + the machine's ITSs appears in the tree. It drops the mappings when the + instance halts. A root bus with no ITS behind it gets no ``msi-map``, + and its devices no interrupts. Unbind the host's driver before the instance is started, and bind it again once the instance is stopped if the host should use the device. @@ -125,7 +129,8 @@ IOMMU The SMMU stays with the host and is not described to a spawn, which hands physical addresses to its devices. While an instance runs, each of its PCI devices behind an SMMU is attached to a domain that maps - exactly the instance's memory, and the MSI doorbell, onto itself. It + exactly the instance's memory, and the doorbell of the ITS behind that + device, onto itself. It follows memory that is added or removed at run time. A device without an IOMMU has unrestricted DMA. diff --git a/arch/arm64/multikernel/dtb_pci.c b/arch/arm64/multikernel/dtb_pci.c index bf6aef796d4093..09ea86affcfa05 100644 --- a/arch/arm64/multikernel/dtb_pci.c +++ b/arch/arm64/multikernel/dtb_pci.c @@ -6,9 +6,10 @@ * devices, ECAM window included, which makes each of them an ordinary * ECAM host bridge; what is left is to name the driver for it. INTx is * not offered: the lines are SPIs that the host shares between devices. - * MSIs go through the host's ITS, which the instance reaches through a - * proxy node: its reg is the ITS's doorbell, the rest happens over the - * message ring (irq-gic-v3-its-mk.c). + * MSIs go through the host's ITSs, which the instance reaches through a + * proxy node. The node has no address: everything, the doorbell of the + * ITS behind a device included, comes over the message ring + * (irq-gic-v3-its-mk.c). */ #define pr_fmt(fmt) "mk_dtb: " fmt @@ -25,7 +26,6 @@ static int mk_dtb_add_its_proxy(void *fdt, u32 *phandle) { - u64 reg[2] = { its_foreign_doorbell(), sizeof(u32) }; int node, ret; ret = fdt_generate_phandle(fdt, phandle); @@ -43,8 +43,6 @@ static int mk_dtb_add_its_proxy(void *fdt, u32 *phandle) ret = fdt_setprop_u32(fdt, node, "#msi-cells", 1); if (!ret) ret = fdt_setprop_u32(fdt, node, "phandle", *phandle); - if (!ret) - ret = mk_dtb_set_reg(fdt, node, reg, 1); return ret; } @@ -64,59 +62,80 @@ static int mk_dtb_set_msi_map(void *fdt, int node, u32 phandle) } /* - * Devices inherit DMA coherency from their host bridge node, and a tree - * that does not say so makes every device behind it non-coherent. Ask a - * device on the host's side of the same root bus. + * What a root bus node has to say about the devices behind it, asked of a + * device on the host's side of the same bus. Devices inherit DMA coherency + * from their host bridge node, and a tree that is silent about it makes + * them all non-coherent. MSIs need an ITS behind the bus; which one does + * not matter here, the host picks it per device when an event is mapped. */ -static bool mk_dtb_root_bus_coherent(void *fdt, int node) +struct mk_root_bus_traits { + bool coherent; + bool msi; +}; + +static int mk_dtb_first_dev_traits(struct pci_dev *pdev, void *data) { - const fdt32_t *domain = fdt_getprop(fdt, node, "linux,pci-domain", NULL); - const fdt32_t *range = fdt_getprop(fdt, node, "bus-range", NULL); - struct pci_bus *bus; - struct pci_dev *pdev; + struct mk_root_bus_traits *traits = data; - if (!range) - return false; + traits->coherent = dev_is_dma_coherent(&pdev->dev); + traits->msi = IS_ENABLED(CONFIG_MULTIKERNEL_ITS_PROXY) && + its_foreign_doorbell(dev_get_msi_domain(&pdev->dev)); + return 1; +} - bus = pci_find_bus(domain ? fdt32_to_cpu(*domain) : 0, - fdt32_to_cpu(range[0])); - pdev = bus ? list_first_entry_or_null(&bus->devices, struct pci_dev, - bus_list) : NULL; - return pdev && dev_is_dma_coherent(&pdev->dev); +static struct mk_root_bus_traits mk_dtb_root_bus_traits(void *fdt, int node) +{ + const fdt32_t *domain = fdt_getprop(fdt, node, "linux,pci-domain", NULL); + const fdt32_t *range = fdt_getprop(fdt, node, "bus-range", NULL); + struct mk_root_bus_traits traits = {}; + struct pci_bus *bus = NULL; + + if (range) + bus = pci_find_bus(domain ? fdt32_to_cpu(*domain) : 0, + fdt32_to_cpu(range[0])); + if (bus) + pci_walk_bus(bus, mk_dtb_first_dev_traits, &traits); + return traits; } +#define mk_dtb_for_each_root_bus(fdt, node) \ + for (node = fdt_node_offset_by_compatible(fdt, -1, MK_PCI_HOST_BRIDGE); \ + node >= 0; \ + node = fdt_node_offset_by_compatible(fdt, node, MK_PCI_HOST_BRIDGE)) + int mk_dtb_add_pci(void *fdt) { static const char compatible[] = MK_PCI_HOST_BRIDGE "\0pci-host-ecam-generic"; - bool msi = IS_ENABLED(CONFIG_MULTIKERNEL_ITS_PROXY) && - its_foreign_doorbell(); + struct mk_root_bus_traits traits; + bool any_msi = false; u32 phandle = 0; int node, ret; - if (fdt_node_offset_by_compatible(fdt, -1, MK_PCI_HOST_BRIDGE) < 0) - return 0; - - if (msi) { + /* Adding the proxy node moves the root buses, so look before writing */ + mk_dtb_for_each_root_bus(fdt, node) + any_msi |= mk_dtb_root_bus_traits(fdt, node).msi; + if (any_msi) { ret = mk_dtb_add_its_proxy(fdt, &phandle); if (ret) return ret; - } else { - pr_warn("No ITS to route MSIs through: PCI devices of the instance get no interrupts\n"); } - for (node = fdt_node_offset_by_compatible(fdt, -1, MK_PCI_HOST_BRIDGE); - node >= 0; - node = fdt_node_offset_by_compatible(fdt, node, MK_PCI_HOST_BRIDGE)) { + mk_dtb_for_each_root_bus(fdt, node) { if (!fdt_getprop(fdt, node, "reg", NULL)) { pr_warn("A PCI root bus of the instance has no ECAM window\n"); continue; } + + traits = mk_dtb_root_bus_traits(fdt, node); + if (!traits.msi) + pr_warn("No ITS behind a PCI root bus of the instance: its devices get no interrupts\n"); + ret = fdt_setprop(fdt, node, "compatible", compatible, sizeof(compatible)); - if (!ret && msi) + if (!ret && traits.msi) ret = mk_dtb_set_msi_map(fdt, node, phandle); - if (!ret && mk_dtb_root_bus_coherent(fdt, node)) + if (!ret && traits.coherent) ret = fdt_setprop_empty(fdt, node, "dma-coherent"); if (ret) return ret; diff --git a/drivers/irqchip/irq-gic-v3-its-mk.c b/drivers/irqchip/irq-gic-v3-its-mk.c index 9f44f395bee267..da57872ce3ae63 100644 --- a/drivers/irqchip/irq-gic-v3-its-mk.c +++ b/drivers/irqchip/irq-gic-v3-its-mk.c @@ -5,9 +5,11 @@ * The ITS belongs to the kernel that spawned this one, so this is what * is left of an ITS driver when someone else runs the command queue: an * MSI parent domain on top of the GIC's LPI range that asks the host to - * map each event (mk_msi_proxy_map()) and is told which LPI will arrive. - * The device is programmed from here as usual, with the ITS's doorbell - * as the address and the event as the data. + * map each event (mk_msi_proxy_map()) and is told which LPI will arrive + * and where the device has to write the event. That address comes with + * every answer rather than with the device tree, because a machine can + * have several ITSs and only the host knows which one sits behind a + * device. The device is programmed from here as usual. * * The redistributors of this kernel's CPUs still run on the host's LPI * tables, which the host enabled before it gave the CPUs away and which @@ -24,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -48,20 +49,21 @@ struct mk_its_device { struct mk_its_irq { struct mk_its_device *dev; u32 event; + phys_addr_t doorbell; }; -static phys_addr_t mk_its_doorbell; - static void mk_its_nop(struct irq_data *d) { } -static int mk_its_route(struct mk_its_irq *irq, unsigned int cpu, bool wait) +/* @doorbell NULL: an event that is mapped already moves, and nobody waits */ +static int mk_its_route(struct mk_its_irq *irq, unsigned int cpu, + phys_addr_t *doorbell) { struct mk_its_device *dev = irq->dev; return mk_msi_proxy_map(dev->domain, dev->rid, irq->event, dev->nvecs, - cpu_logical_map(cpu), wait); + cpu_logical_map(cpu), doorbell); } /* Called with the descriptor locked: the move is sent, not waited for */ @@ -74,7 +76,7 @@ static int mk_its_set_affinity(struct irq_data *d, const struct cpumask *mask, if (cpu >= nr_cpu_ids) return -EINVAL; - ret = mk_its_route(irq_data_get_irq_chip_data(d), cpu, false); + ret = mk_its_route(irq_data_get_irq_chip_data(d), cpu, NULL); if (ret) return ret; @@ -86,8 +88,8 @@ static void mk_its_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) { struct mk_its_irq *irq = irq_data_get_irq_chip_data(d); - msg->address_lo = lower_32_bits(mk_its_doorbell); - msg->address_hi = upper_32_bits(mk_its_doorbell); + msg->address_lo = lower_32_bits(irq->doorbell); + msg->address_hi = upper_32_bits(irq->doorbell); msg->data = irq->event; } @@ -157,7 +159,7 @@ static int mk_its_alloc_one(struct irq_domain *domain, unsigned int virq, irq->dev = mdev; irq->event = event; - lpi = mk_its_route(irq, cpu, true); + lpi = mk_its_route(irq, cpu, &irq->doorbell); if (lpi < 0) { kfree(irq); return lpi; @@ -238,7 +240,6 @@ static int __init mk_its_init(void) .host_data = &mk_its_msi_info, }; struct device_node *np, *gic; - struct resource res; int ret = -ENODEV; np = of_find_compatible_node(NULL, NULL, MK_ITS_COMPATIBLE); @@ -249,9 +250,8 @@ static int __init mk_its_init(void) info.parent = gic ? irq_find_host(gic) : NULL; of_node_put(gic); - if (!info.parent || of_address_to_resource(np, 0, &res)) + if (!info.parent) goto out; - mk_its_doorbell = res.start; info.fwnode = of_fwnode_handle(np); ret = msi_create_parent_irq_domain(&info, &gic_v3_its_msi_parent_ops) ? @@ -260,8 +260,7 @@ static int __init mk_its_init(void) if (ret) pr_err("%pOF: no MSI domain: %d\n", np, ret); else - pr_info("MSIs through the host's ITS, doorbell %pa\n", - &mk_its_doorbell); + pr_info("MSIs through the host's ITS\n"); of_node_put(np); return ret; } diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 1842ad2aca56dd..f742908c122a55 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3778,6 +3778,14 @@ static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, irq_domain_free_irqs_parent(domain, virq, nr_irqs); } +static const struct irq_domain_ops its_domain_ops = { + .select = msi_lib_irq_domain_select, + .alloc = its_irq_domain_alloc, + .free = its_irq_domain_free, + .activate = its_irq_domain_activate, + .deactivate = its_irq_domain_deactivate, +}; + #ifdef CONFIG_MULTIKERNEL_ITS_PROXY /* * LPIs on behalf of another kernel. @@ -3793,13 +3801,15 @@ static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, * side: they never reach one of this kernel's CPU interfaces. * * @owner keeps the devices of different kernels apart and names what - * its_foreign_release() tears down. + * its_foreign_release() tears down. The ITS is the one behind the device, + * named by the device's MSI domain: a machine has one per socket or more, + * each with DeviceIDs, collections and a doorbell of its own. */ -static struct its_node *its_foreign_node(void) +static struct its_node *its_foreign_its(struct irq_domain *domain) { - if (!list_is_singular(&its_nodes)) + if (!domain || domain->ops != &its_domain_ops) return NULL; - return list_first_entry(&its_nodes, struct its_node, entry); + return msi_get_domain_info(domain)->data; } static void its_foreign_config(struct its_device *dev, u32 event, bool enable) @@ -3816,7 +3826,10 @@ static void its_foreign_config(struct its_device *dev, u32 event, bool enable) its_send_inv(dev, event); } -/* A collection is only mapped for a CPU that has been online here */ +/* + * A collection is only mapped for a CPU that has been online here, and an + * ITS with erratum 23144 only reaches the CPUs of its own node. + */ static struct its_collection *its_foreign_collection(struct its_node *its, unsigned int cpu) { @@ -3824,26 +3837,34 @@ static struct its_collection *its_foreign_collection(struct its_node *its, its->collections[cpu].col_id != cpu || (cpu && !its->collections[cpu].target_address)) return NULL; + + if ((its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) && + its->numa_node != NUMA_NO_NODE && cpu_to_node(cpu) != its->numa_node) + return NULL; + return &its->collections[cpu]; } /** * its_foreign_map - Route a device's event to a CPU of another kernel * @owner: the other kernel, as the caller names it - * @dev_id: DeviceID as the ITS sees it + * @domain: MSI domain of the device, which names the ITS behind it + * @dev_id: DeviceID as that ITS sees it * @event: event (MSI data) to map * @nvecs: events the device will use at most, fixed by the first call * @cpu: this kernel's number of the CPU to deliver to + * @doorbell: set to the address the device has to write the event to * * Mapping an event that is mapped already moves it to @cpu. * * Returns the LPI, which is the INTID the other kernel takes on @cpu, * or a negative error code. */ -int its_foreign_map(void *owner, u32 dev_id, u32 event, u32 nvecs, - unsigned int cpu) +int its_foreign_map(void *owner, struct irq_domain *domain, u32 dev_id, + u32 event, u32 nvecs, unsigned int cpu, + phys_addr_t *doorbell) { - struct its_node *its = its_foreign_node(); + struct its_node *its = its_foreign_its(domain); struct its_collection *col; struct its_device *dev; @@ -3875,25 +3896,15 @@ int its_foreign_map(void *owner, u32 dev_id, u32 event, u32 nvecs, its_send_mapti(dev, dev->event_map.lpi_base + event, event); its_foreign_config(dev, event, true); } + *doorbell = its->get_msi_base(dev); return dev->event_map.lpi_base + event; } -/** - * its_foreign_release - Drop everything mapped for another kernel - * @owner: the other kernel, as given to its_foreign_map() - * - * For when that kernel is gone or about to be started afresh, whether it - * shut down in order or not: nothing of it must keep raising LPIs. - */ -void its_foreign_release(void *owner) +static void its_foreign_release_on(struct its_node *its, void *owner) { - struct its_node *its = its_foreign_node(); struct its_device *dev, *tmp; unsigned int event; - if (!its) - return; - guard(mutex)(&its->dev_alloc_lock); list_for_each_entry_safe(dev, tmp, &its->its_device_list, entry) { @@ -3913,23 +3924,34 @@ void its_foreign_release(void *owner) } } -/* Where a device of another kernel writes its MSIs: 0 without a usable ITS */ -phys_addr_t its_foreign_doorbell(void) +/** + * its_foreign_release - Drop everything mapped for another kernel + * @owner: the other kernel, as given to its_foreign_map() + * + * For when that kernel is gone or about to be started afresh, whether it + * shut down in order or not: nothing of it must keep raising LPIs. + */ +void its_foreign_release(void *owner) +{ + struct its_node *its; + + /* ITSs are only ever added, and that during boot */ + list_for_each_entry(its, &its_nodes, entry) + its_foreign_release_on(its, owner); +} + +/* + * The doorbell of the ITS behind a device with MSI domain @domain, which + * an IOMMU in front of the device has to let through. 0 if there is none. + */ +phys_addr_t its_foreign_doorbell(struct irq_domain *domain) { - struct its_node *its = its_foreign_node(); + struct its_node *its = its_foreign_its(domain); return its ? its->phys_base + GITS_TRANSLATER : 0; } #endif -static const struct irq_domain_ops its_domain_ops = { - .select = msi_lib_irq_domain_select, - .alloc = its_irq_domain_alloc, - .free = its_irq_domain_free, - .activate = its_irq_domain_activate, - .deactivate = its_irq_domain_deactivate, -}; - /* * This is insane. * diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 25c2df7ba55a94..b8db3ca26d8353 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -640,10 +640,11 @@ int __init its_lpi_memreserve_init(void); int its_cpu_init(void); int gic_v3_send_sgi_to_mpidr(u64 mpidr, unsigned int sgi); -int its_foreign_map(void *owner, u32 dev_id, u32 event, u32 nvecs, - unsigned int cpu); +int its_foreign_map(void *owner, struct irq_domain *domain, u32 dev_id, + u32 event, u32 nvecs, unsigned int cpu, + phys_addr_t *doorbell); void its_foreign_release(void *owner); -phys_addr_t its_foreign_doorbell(void); +phys_addr_t its_foreign_doorbell(struct irq_domain *domain); int its_init(struct fwnode_handle *handle, struct rdists *rdists, struct irq_domain *domain, u8 irq_prio); int mbi_init(struct fwnode_handle *fwnode, struct irq_domain *parent); diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index b84a429a0fa8d9..b917b703a8dcba 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -175,9 +175,13 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); -/* Spawn side of MK_IO_MSI_MAP: returns the interrupt number or -errno; @wait false when atomic */ +/* + * Spawn side of MK_IO_MSI_MAP: returns the interrupt number or -errno and + * sets @doorbell. A NULL @doorbell sends the request without waiting for + * the answer, for a context that cannot sleep. + */ int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, - mk_phys_cpu_t phys_cpu, bool wait); + mk_phys_cpu_t phys_cpu, phys_addr_t *doorbell); /* * Host side: drop every MSI routed for @instance. Its kernel is gone or @@ -185,15 +189,16 @@ int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, * bound here next. */ struct mk_instance; +struct pci_dev; #ifdef CONFIG_MULTIKERNEL_ITS_PROXY void mk_msi_proxy_release(struct mk_instance *instance); -/* The address an instance's devices write their MSIs to, 0 if none */ -phys_addr_t mk_msi_proxy_doorbell(void); +/* The address @pdev writes its MSIs to when an instance owns it, 0 if none */ +phys_addr_t mk_msi_proxy_doorbell(struct pci_dev *pdev); #else static inline void mk_msi_proxy_release(struct mk_instance *instance) { } -static inline phys_addr_t mk_msi_proxy_doorbell(void) +static inline phys_addr_t mk_msi_proxy_doorbell(struct pci_dev *pdev) { return 0; } @@ -348,9 +353,11 @@ struct mk_device_resource_payload { * MSI routing request. Where MSIs go through a translation unit only one * kernel can program (a GICv3 ITS), a spawn asks the kernel that owns it. * @rid is the device as the spawn sees it on its PCI bus, @event the MSI - * data it will write. Mapping an event again moves it to @phys_cpu, and - * everything is dropped by the host when the instance is started or torn - * down, so there is no unmap. + * data it will write. The answer names the interrupt that will arrive and + * the address to write @event to, which depends on the unit behind the + * device when the machine has several. Mapping an event again moves it to + * @phys_cpu, and everything is dropped by the host when the instance is + * started or torn down, so there is no unmap. */ struct mk_io_msi_payload { u32 domain; /* PCI domain */ @@ -358,6 +365,7 @@ struct mk_io_msi_payload { u32 event; u32 nvecs; /* Events the device may use, fixed by the first request */ u64 phys_cpu; /* Target CPU, one of the sender's */ + u64 doorbell; /* ACK: address the device writes the event to */ int result; /* ACK: interrupt number (LPI INTID), or -errno */ int sender_instance_id; }; diff --git a/kernel/multikernel/iommu.c b/kernel/multikernel/iommu.c index 88396447ac0418..7ac0eae567ce3d 100644 --- a/kernel/multikernel/iommu.c +++ b/kernel/multikernel/iommu.c @@ -9,11 +9,12 @@ * instance can reach all of the host. * * So while an instance runs, each of its devices is attached to a domain - * of its own that maps the instance's memory grant onto itself, and the - * MSI doorbell with it. The device works untranslated, and cannot reach - * anything else. The domain follows the grant as memory is added and - * removed, and goes away when the instance halts, which returns the - * device to the host's default domain for a host driver to bind. + * of its own that maps the instance's memory grant onto itself, and with + * it the MSI doorbell of the ITS behind that device. The device works + * untranslated, and cannot reach anything else. The domain follows the + * grant as memory is added and removed, and goes away when the instance + * halts, which returns the device to the host's default domain for a host + * driver to bind. * * A device without an IOMMU is left alone: its DMA is unrestricted, as * it always was. @@ -56,8 +57,9 @@ static void mk_iommu_free_dev(struct mk_iommu_dev *d) } static int mk_iommu_contain_dev(struct mk_instance *instance, - struct pci_dev *pdev, phys_addr_t doorbell) + struct pci_dev *pdev) { + phys_addr_t doorbell = mk_msi_proxy_doorbell(pdev); struct mk_memory_region *region; struct mk_iommu_dev *d; int ret; @@ -123,7 +125,6 @@ static int mk_iommu_contain_dev(struct mk_instance *instance, */ int mk_iommu_contain(struct mk_instance *instance) { - phys_addr_t doorbell = mk_msi_proxy_doorbell(); struct mk_pci_device *mine; int ret = 0; @@ -142,7 +143,7 @@ int mk_iommu_contain(struct mk_instance *instance) continue; if (device_iommu_mapped(&pdev->dev)) - ret = mk_iommu_contain_dev(instance, pdev, doorbell); + ret = mk_iommu_contain_dev(instance, pdev); else pr_info_once("%s has no IOMMU, DMA of instance devices is unrestricted\n", pci_name(pdev)); diff --git a/kernel/multikernel/its_proxy.c b/kernel/multikernel/its_proxy.c index 4f4072df72ad16..b161e0deb904c9 100644 --- a/kernel/multikernel/its_proxy.c +++ b/kernel/multikernel/its_proxy.c @@ -16,7 +16,9 @@ #define pr_fmt(fmt) "mk_its: " fmt +#include #include +#include #include #include #include @@ -30,6 +32,20 @@ struct mk_msi_work { struct mk_io_msi_payload req; }; +/* + * A request this kernel waits on. The answer is more than a result code, + * so it is kept here until the waiter picks it up. + */ +struct mk_msi_request { + struct list_head list; + u64 key; + struct completion done; + struct mk_io_msi_payload answer; +}; + +static LIST_HEAD(mk_msi_requests); +static DEFINE_SPINLOCK(mk_msi_requests_lock); + static u64 mk_msi_key(const struct mk_io_msi_payload *p) { return (u64)p->domain << 48 | (u64)p->rid << 32 | p->event; @@ -37,9 +53,11 @@ static u64 mk_msi_key(const struct mk_io_msi_payload *p) /* Only a device the instance was given, only a CPU it owns */ static int mk_msi_map_for(struct mk_instance *instance, - const struct mk_io_msi_payload *req) + struct mk_io_msi_payload *req) { + struct irq_domain *msi_domain; struct mk_pci_device *mine; + phys_addr_t doorbell = 0; struct pci_dev *pdev; bool owned = false; int cpu, ret; @@ -63,11 +81,14 @@ static int mk_msi_map_for(struct mk_instance *instance, if (!pdev) return -ENODEV; - ret = its_foreign_map(instance, - pci_msi_domain_get_msi_rid(dev_get_msi_domain(&pdev->dev), - pdev), - req->event, req->nvecs, cpu); + /* The device's MSI domain names the ITS behind it, of possibly several */ + msi_domain = dev_get_msi_domain(&pdev->dev); + ret = its_foreign_map(instance, msi_domain, + pci_msi_domain_get_msi_rid(msi_domain, pdev), + req->event, req->nvecs, cpu, &doorbell); pci_dev_put(pdev); + if (ret >= 0) + req->doorbell = doorbell; return ret; } @@ -94,6 +115,22 @@ static void mk_msi_map_work(struct work_struct *work) kfree(w); } +static void mk_msi_answer(const struct mk_io_msi_payload *answer) +{ + struct mk_msi_request *r; + unsigned long flags; + + spin_lock_irqsave(&mk_msi_requests_lock, flags); + list_for_each_entry(r, &mk_msi_requests, list) { + if (r->key == mk_msi_key(answer)) { + r->answer = *answer; + complete(&r->done); + break; + } + } + spin_unlock_irqrestore(&mk_msi_requests_lock, flags); +} + static void mk_msi_msg_handler(u32 msg_type, u32 subtype, void *payload, u32 payload_len, void *ctx) { @@ -113,8 +150,7 @@ static void mk_msi_msg_handler(u32 msg_type, u32 subtype, void *payload, schedule_work(&w->work); break; case MK_IO_MSI_ACK: - mk_msg_pending_complete(MK_MSG_IO, MK_IO_MSI_MAP, mk_msi_key(p), - p->result); + mk_msi_answer(p); break; } } @@ -126,13 +162,14 @@ static void mk_msi_msg_handler(u32 msg_type, u32 subtype, void *payload, * @event: MSI data the device will write * @nvecs: events the device may use * @phys_cpu: target, one of this kernel's CPUs - * @wait: wait for the answer; false moves an event that is mapped - * already, from a context that cannot sleep + * @doorbell: set to the address to write @event to. NULL moves an event + * that is mapped already without waiting for the answer, from a + * context that cannot sleep * * Returns the interrupt number (0 when not waiting) or a negative error. */ int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, - mk_phys_cpu_t phys_cpu, bool wait) + mk_phys_cpu_t phys_cpu, phys_addr_t *doorbell) { struct mk_io_msi_payload req = { .domain = domain, @@ -142,24 +179,40 @@ int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, .phys_cpu = phys_cpu, .sender_instance_id = mk_self->id, }; - struct mk_pending_msg *pending = NULL; + struct mk_msi_request *r; + unsigned long flags; int ret; - if (wait) { - pending = mk_msg_pending_add(MK_MSG_IO, MK_IO_MSI_MAP, - mk_msi_key(&req)); - if (!pending) - return -ENOMEM; - } + if (!doorbell) + return mk_send_message(0, MK_MSG_IO, MK_IO_MSI_MAP, &req, + sizeof(req)); + + r = kzalloc_obj(*r); + if (!r) + return -ENOMEM; + r->key = mk_msi_key(&req); + init_completion(&r->done); + + spin_lock_irqsave(&mk_msi_requests_lock, flags); + list_add(&r->list, &mk_msi_requests); + spin_unlock_irqrestore(&mk_msi_requests_lock, flags); ret = mk_send_message(0, MK_MSG_IO, MK_IO_MSI_MAP, &req, sizeof(req)); - if (!wait) - return ret; - if (ret < 0) { - mk_msg_pending_wait(pending, 0); - return ret; + if (!ret && !wait_for_completion_timeout(&r->done, + msecs_to_jiffies(MK_MSI_TIMEOUT_MS))) + ret = -ETIMEDOUT; + + /* Off the list before the answer is read: a late one must not land in it */ + spin_lock_irqsave(&mk_msi_requests_lock, flags); + list_del(&r->list); + spin_unlock_irqrestore(&mk_msi_requests_lock, flags); + + if (!ret) { + ret = r->answer.result; + *doorbell = r->answer.doorbell; } - return mk_msg_pending_wait(pending, MK_MSI_TIMEOUT_MS); + kfree(r); + return ret; } void mk_msi_proxy_release(struct mk_instance *instance) @@ -167,9 +220,9 @@ void mk_msi_proxy_release(struct mk_instance *instance) its_foreign_release(instance); } -phys_addr_t mk_msi_proxy_doorbell(void) +phys_addr_t mk_msi_proxy_doorbell(struct pci_dev *pdev) { - return its_foreign_doorbell(); + return its_foreign_doorbell(dev_get_msi_domain(&pdev->dev)); } static int __init mk_msi_proxy_init(void) From aef84052d62d506a88bd753069b53664b94e132d Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 19:40:49 -0700 Subject: [PATCH 18/30] multikernel: Let a message be sent and answered with interrupts off Two things stand in the way of a caller that has to exchange a message with another kernel from a context that cannot sleep, such as an interrupt chip callback under the descriptor lock. Sending looks the instance up by ID, and mk_instance_find() takes a mutex. Everything else on that path is fit for atomic context, down to the GFP_ATOMIC allocation, and the console already gets there with interrupts off. Add mk_send_message_to() and multikernel_send_ipi_data_to() for a caller that holds a reference to the instance, and build the ID variants on top of them. Receiving depends on the doorbell interrupt, which such a caller may be unable to take: it may be the doorbell CPU itself, or stop_machine() may hold every CPU, as it does while a dying CPU migrates its interrupts. Add mk_ipi_poll() to drain the ring from the caller's context. The ring has a single consumer, which used to be implied by the doorbell going to one CPU, so the drain now runs under a lock. The interrupt waits for that lock instead of giving up: a poller that is just leaving may not look at the ring again. No change for existing callers. Signed-off-by: Cong Wang --- include/linux/multikernel.h | 16 ++++++++++ kernel/multikernel/ipi.c | 56 +++++++++++++++++++++++++++------- kernel/multikernel/messaging.c | 27 ++++++++++++++-- 3 files changed, 85 insertions(+), 14 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index b917b703a8dcba..0257f69c5ae518 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -175,6 +175,13 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); +/* Drain the message ring where the doorbell interrupt cannot be taken */ +void mk_ipi_poll(void); + +struct mk_instance; +int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); + /* * Spawn side of MK_IO_MSI_MAP: returns the interrupt number or -errno and * sets @doorbell. A NULL @doorbell sends the request without waiting for @@ -411,6 +418,15 @@ struct mk_pending_msg; int mk_send_message(int instance_id, u32 msg_type, u32 subtype, void *payload, u32 payload_len); +/* + * The same for a caller that holds a reference to @instance. Looking an + * instance up by ID sleeps; this does not, so it is the one to use with + * interrupts off. + */ +struct mk_instance; +int mk_send_message_to(struct mk_instance *instance, u32 msg_type, u32 subtype, + void *payload, u32 payload_len); + /** * mk_register_msg_handler - Register handler for specific message type * @msg_type: Message type to handle diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index d7e9739a448dff..2720ee66ef5983 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -18,6 +18,8 @@ static struct mk_ipi_handler *mk_handlers; static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); +static DEFINE_RAW_SPINLOCK(mk_drain_lock); + static void mk_ipi_drain_ring(void); /* @@ -156,25 +158,39 @@ int mk_arm_force_halt(struct mk_instance *instance) */ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) { - struct mk_ipi_data *slot; struct mk_instance *instance = mk_instance_find(instance_id); + int ret; + + if (!instance) + return -EINVAL; + + ret = multikernel_send_ipi_data_to(instance, data, data_size, type); + mk_instance_put(instance); + return ret; +} + +/* + * The same for a caller that holds a reference to @instance. Looking an + * instance up by ID takes a mutex; this does not sleep, so it is the one + * to use with interrupts off. + */ +int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) +{ struct mk_shared_data *ipi_data; + struct mk_ipi_data *slot; + int instance_id = instance->id; unsigned int head, next_head, tail; mk_phys_cpu_t target; - if (!instance) + if (data_size > MK_MAX_DATA_SIZE) return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); - return -EINVAL; - } target = instance->ipi_target; if (target == MK_PHYS_CPU_INVALID) target = mk_cpu_set_first(instance->cpus); if (target == MK_PHYS_CPU_INVALID) { pr_err("Instance %d has no CPU to receive the IPI\n", instance_id); - mk_instance_put(instance); return -ENODEV; } @@ -182,7 +198,6 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns ipi_data = READ_ONCE(instance->ipi_data); if (!ipi_data) { pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); return -ENODEV; } @@ -202,7 +217,6 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns printk_deferred(KERN_WARNING "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", instance_id, head, tail); - mk_instance_put(instance); return -ENOSPC; } @@ -227,8 +241,6 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns smp_store_release(&slot->data_size, data_size); mk_arch_send_ipi(target); - - mk_instance_put(instance); return 0; } @@ -310,6 +322,8 @@ static void mk_ipi_drain_ring(void) */ static void multikernel_interrupt_handler(void) { + unsigned long flags; + if (!mk_self || !mk_self->ipi_data) return; @@ -320,8 +334,28 @@ static void multikernel_interrupt_handler(void) * instance, so if it is ever left pending - its self-IPI lost while * the CPU was bringing its APIC up, say - every later queue attempt * is a no-op and the ring never drains again. + * + * The ring has one consumer at a time. That used to be implied by + * the doorbell going to a single CPU; mk_ipi_poll() is a second way + * in, so it is a lock now. Waiting for it rather than giving up + * matters: a poller that is just leaving may not look again. */ + raw_spin_lock_irqsave(&mk_drain_lock, flags); mk_ipi_drain_ring(); + raw_spin_unlock_irqrestore(&mk_drain_lock, flags); +} + +/** + * mk_ipi_poll - Drain the message ring without waiting for the doorbell + * + * For a caller that waits for an answer where the doorbell interrupt + * cannot be taken: with interrupts off on the doorbell CPU, or while + * stop_machine() holds every CPU. Message handlers run in the caller's + * context, which they are fit for, as they normally run in the interrupt. + */ +void mk_ipi_poll(void) +{ + multikernel_interrupt_handler(); } /** diff --git a/kernel/multikernel/messaging.c b/kernel/multikernel/messaging.c index 2cd2b7682f8d3e..210f1d3d88618e 100644 --- a/kernel/multikernel/messaging.c +++ b/kernel/multikernel/messaging.c @@ -190,8 +190,9 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms * * Returns 0 on success, negative error code on failure */ -int mk_send_message(int instance_id, u32 msg_type, u32 subtype, - void *payload, u32 payload_len) +static int mk_build_and_send(int instance_id, struct mk_instance *instance, + u32 msg_type, u32 subtype, void *payload, + u32 payload_len) { struct mk_message *msg; size_t total_size; @@ -223,7 +224,12 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, memcpy(msg->payload, payload, payload_len); /* Send via IPI using the message type as IPI type */ - ret = multikernel_send_ipi_data(instance_id, msg, total_size, msg_type); + if (instance) + ret = multikernel_send_ipi_data_to(instance, msg, total_size, + msg_type); + else + ret = multikernel_send_ipi_data(instance_id, msg, total_size, + msg_type); /* Clean up temporary buffer */ kfree(msg); @@ -238,8 +244,23 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, return 0; } + +int mk_send_message(int instance_id, u32 msg_type, u32 subtype, + void *payload, u32 payload_len) +{ + return mk_build_and_send(instance_id, NULL, msg_type, subtype, payload, + payload_len); +} EXPORT_SYMBOL(mk_send_message); +int mk_send_message_to(struct mk_instance *instance, u32 msg_type, u32 subtype, + void *payload, u32 payload_len) +{ + return mk_build_and_send(instance->id, instance, msg_type, subtype, + payload, payload_len); +} +EXPORT_SYMBOL(mk_send_message_to); + /** * mk_register_msg_handler - Register handler for specific message type * @msg_type: Message type to handle From 5bfc89776864239051c256c676f3e2375e5fc8a9 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 19:45:49 -0700 Subject: [PATCH 19/30] arm64/multikernel: Move the ITS proxy out of the generic code its_proxy.c sat in kernel/multikernel/ and called straight into the GICv3 ITS driver. Nothing about it is generic: the problem it solves, one command queue per ITS, belongs to the GIC, and the only other interrupt controller with the same shape is GICv5's, which is arm64 as well. x86, and anything that writes its MSIs straight at a CPU, never needs it. Move it to arch/arm64/multikernel/, together with its Kconfig symbol and the message payload, which only its two ends read. The message numbers stay in the generic header, which is the one registry for them. What the generic code needs from it becomes part of the architecture interface, behind ARCH_HAS_MK_MSI_PROXY the way the host park area sits behind ARCH_HAS_MK_HOST_PARK: mk_arch_msi_release(), called when an instance halts or is about to be started, and mk_arch_msi_doorbell(), which tells the IOMMU containment what a device has to be able to write to. The request function the instance's MSI domain calls is declared in asm/multikernel.h. No functional change. Signed-off-by: Cong Wang --- arch/arm64/Kconfig | 5 ++ arch/arm64/include/asm/multikernel.h | 10 +++ arch/arm64/multikernel/Makefile | 1 + .../arm64}/multikernel/its_proxy.c | 29 +++++++- include/linux/multikernel.h | 74 ++++++------------- kernel/kexec_core.c | 2 +- kernel/multikernel/Kconfig | 7 +- kernel/multikernel/Makefile | 1 - kernel/multikernel/core.c | 4 +- kernel/multikernel/iommu.c | 2 +- 10 files changed, 73 insertions(+), 62 deletions(-) rename {kernel => arch/arm64}/multikernel/its_proxy.c (83%) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 4dd175590a82de..a1f9604580aeba 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1625,6 +1625,11 @@ config ARCH_SUPPORTS_KEXEC_HANDOVER config ARCH_SUPPORTS_MULTIKERNEL def_bool ARM_PSCI_FW && KEXEC_FILE && HOTPLUG_CPU +# An ITS has one command queue, so the host maps MSIs for its instances +config MULTIKERNEL_ITS_PROXY + def_bool MULTIKERNEL && ARM_GIC_V3_ITS && PCI_MSI + select ARCH_HAS_MK_MSI_PROXY + config ARCH_SUPPORTS_CRASH_DUMP def_bool y diff --git a/arch/arm64/include/asm/multikernel.h b/arch/arm64/include/asm/multikernel.h index dde511c01b3cc8..a499a7bdd5e837 100644 --- a/arch/arm64/include/asm/multikernel.h +++ b/arch/arm64/include/asm/multikernel.h @@ -63,6 +63,16 @@ static inline void mk_foreign_cpu_stop(void) } #endif +/* + * Ask the host to route a device's MSI to one of this kernel's CPUs, for + * the instance's MSI domain (irq-gic-v3-its-mk.c). Returns the interrupt + * number or -errno and sets @doorbell; a NULL @doorbell does not wait. + */ +#ifdef CONFIG_MULTIKERNEL_ITS_PROXY +int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, + u64 phys_cpu, phys_addr_t *doorbell); +#endif + /* * Room for an instance's boot device tree, a segment of its kimage: the * manifest (MK_MANIFEST_SIZE) with the arm64 boot nodes on top. The tree diff --git a/arch/arm64/multikernel/Makefile b/arch/arm64/multikernel/Makefile index 95fbd2f416607d..7928a5dfdea413 100644 --- a/arch/arm64/multikernel/Makefile +++ b/arch/arm64/multikernel/Makefile @@ -5,3 +5,4 @@ obj-y += spawn.o dtb.o dtb_of.o dtb_devices.o dtb_pci.o obj-$(CONFIG_ACPI) += dtb_acpi.o +obj-$(CONFIG_MULTIKERNEL_ITS_PROXY) += its_proxy.o diff --git a/kernel/multikernel/its_proxy.c b/arch/arm64/multikernel/its_proxy.c similarity index 83% rename from kernel/multikernel/its_proxy.c rename to arch/arm64/multikernel/its_proxy.c index b161e0deb904c9..80ff800d80aeea 100644 --- a/kernel/multikernel/its_proxy.c +++ b/arch/arm64/multikernel/its_proxy.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0 /* * MSIs for an instance's PCI devices, through the host's ITS. * @@ -27,6 +27,27 @@ #define MK_MSI_TIMEOUT_MS 5000 +/* + * MSI routing request. Where MSIs go through a translation unit only one + * kernel can program (a GICv3 ITS), a spawn asks the kernel that owns it. + * @rid is the device as the spawn sees it on its PCI bus, @event the MSI + * data it will write. The answer names the interrupt that will arrive and + * the address to write @event to, which depends on the unit behind the + * device when the machine has several. Mapping an event again moves it to + * @phys_cpu, and everything is dropped by the host when the instance is + * started or torn down, so there is no unmap. + */ +struct mk_io_msi_payload { + u32 domain; /* PCI domain */ + u32 rid; /* bus << 8 | devfn */ + u32 event; + u32 nvecs; /* Events the device may use, fixed by the first request */ + u64 phys_cpu; /* Target CPU, one of the sender's */ + u64 doorbell; /* ACK: address the device writes the event to */ + int result; /* ACK: interrupt number (LPI INTID), or -errno */ + int sender_instance_id; +}; + struct mk_msi_work { struct work_struct work; struct mk_io_msi_payload req; @@ -169,7 +190,7 @@ static void mk_msi_msg_handler(u32 msg_type, u32 subtype, void *payload, * Returns the interrupt number (0 when not waiting) or a negative error. */ int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, - mk_phys_cpu_t phys_cpu, phys_addr_t *doorbell) + u64 phys_cpu, phys_addr_t *doorbell) { struct mk_io_msi_payload req = { .domain = domain, @@ -215,12 +236,12 @@ int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, return ret; } -void mk_msi_proxy_release(struct mk_instance *instance) +void mk_arch_msi_release(struct mk_instance *instance) { its_foreign_release(instance); } -phys_addr_t mk_msi_proxy_doorbell(struct pci_dev *pdev) +phys_addr_t mk_arch_msi_doorbell(struct pci_dev *pdev) { return its_foreign_doorbell(dev_get_msi_domain(&pdev->dev)); } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 0257f69c5ae518..e6ac4264d4e775 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -182,35 +182,6 @@ struct mk_instance; int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, size_t data_size, unsigned long type); -/* - * Spawn side of MK_IO_MSI_MAP: returns the interrupt number or -errno and - * sets @doorbell. A NULL @doorbell sends the request without waiting for - * the answer, for a context that cannot sleep. - */ -int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, - mk_phys_cpu_t phys_cpu, phys_addr_t *doorbell); - -/* - * Host side: drop every MSI routed for @instance. Its kernel is gone or - * about to be started afresh, in order or not, and its devices may be - * bound here next. - */ -struct mk_instance; -struct pci_dev; -#ifdef CONFIG_MULTIKERNEL_ITS_PROXY -void mk_msi_proxy_release(struct mk_instance *instance); -/* The address @pdev writes its MSIs to when an instance owns it, 0 if none */ -phys_addr_t mk_msi_proxy_doorbell(struct pci_dev *pdev); -#else -static inline void mk_msi_proxy_release(struct mk_instance *instance) -{ -} -static inline phys_addr_t mk_msi_proxy_doorbell(struct pci_dev *pdev) -{ - return 0; -} -#endif - /* * Host side: keep the DMA of an instance's PCI devices inside its memory * while it runs, where the devices sit behind an IOMMU. @@ -262,6 +233,7 @@ void mk_ipi_ring_drop_pending(void); #define MK_IO_IRQ_BALANCE (MK_MSG_IO + 2) #define MK_IO_IRQ_MASK (MK_MSG_IO + 3) #define MK_IO_IRQ_UNMASK (MK_MSG_IO + 4) +/* MSI routing through the host, where the arch needs it: payload is the arch's */ #define MK_IO_MSI_MAP (MK_MSG_IO + 5) /* Route a device's MSI to one of my CPUs */ #define MK_IO_MSI_ACK (MK_MSG_IO + 6) /* Answer: the interrupt number, or an error */ @@ -356,27 +328,6 @@ struct mk_device_resource_payload { #define MK_DEVICE_BIND_VFIO 0x01 /* Bind to VFIO driver */ #define MK_DEVICE_BIND_STUB 0x02 /* Bind to pci-stub driver */ -/* - * MSI routing request. Where MSIs go through a translation unit only one - * kernel can program (a GICv3 ITS), a spawn asks the kernel that owns it. - * @rid is the device as the spawn sees it on its PCI bus, @event the MSI - * data it will write. The answer names the interrupt that will arrive and - * the address to write @event to, which depends on the unit behind the - * device when the machine has several. Mapping an event again moves it to - * @phys_cpu, and everything is dropped by the host when the instance is - * started or torn down, so there is no unmap. - */ -struct mk_io_msi_payload { - u32 domain; /* PCI domain */ - u32 rid; /* bus << 8 | devfn */ - u32 event; - u32 nvecs; /* Events the device may use, fixed by the first request */ - u64 phys_cpu; /* Target CPU, one of the sender's */ - u64 doorbell; /* ACK: address the device writes the event to */ - int result; /* ACK: interrupt number (LPI INTID), or -errno */ - int sender_instance_id; -}; - /* Resource operation response/ACK */ struct mk_resource_ack { u32 operation; /* Original operation (MK_RES_CPU_ADD, etc.) */ @@ -1223,6 +1174,8 @@ void *mk_instance_ctrl_alloc(struct mk_instance *instance, size_t size, * - CONFIG_ARCH_HAS_MK_HOST_PARK and the pool park functions declared * with the pool chunk API above, for architectures that park CPUs in * software rather than in firmware. + * - CONFIG_ARCH_HAS_MK_MSI_PROXY and the two functions below, for + * architectures where an instance cannot route its own MSIs. */ /* Doorbell for the message ring: IPI a CPU owned by another kernel */ @@ -1270,4 +1223,25 @@ int mk_repark_instance_to_host(struct mk_instance *instance); int mk_repark_cpu_to_instance(struct mk_instance *instance, mk_phys_cpu_t phys_cpu); int mk_repark_cpu_to_host(struct mk_instance *instance, mk_phys_cpu_t phys_cpu); +/* + * MSIs that the host has to route for an instance (a GICv3 ITS has one + * command queue). The generic code only needs to know when to let go and + * what an IOMMU in front of a device has to let through. + */ +#ifdef CONFIG_ARCH_HAS_MK_MSI_PROXY +/* Drop every MSI routed for @instance: it halted, or is about to be started */ +void mk_arch_msi_release(struct mk_instance *instance); +/* The address @pdev writes its MSIs to when an instance owns it, 0 if none */ +phys_addr_t mk_arch_msi_doorbell(struct pci_dev *pdev); +#else +static inline void mk_arch_msi_release(struct mk_instance *instance) +{ +} + +static inline phys_addr_t mk_arch_msi_doorbell(struct pci_dev *pdev) +{ + return 0; +} +#endif + #endif /* _LINUX_MULTIKERNEL_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 67d07c152e5d75..d5f42504eddf27 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1755,7 +1755,7 @@ int multikernel_kexec_by_id(int mk_id) mk_ipi_ring_drop_pending(); /* A previous run that died without a word left its MSIs routed */ - mk_msi_proxy_release(instance); + mk_arch_msi_release(instance); rc = mk_iommu_contain(instance); if (rc) { diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index 7cda3199c2c956..fe4e1ca461831c 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -10,6 +10,10 @@ config ARCH_SUPPORTS_MULTIKERNEL config ARCH_HAS_MK_HOST_PARK bool +# The arch routes the MSIs of an instance's devices on the instance's behalf +config ARCH_HAS_MK_MSI_PROXY + bool + config MULTIKERNEL bool "Multikernel support" depends on ARCH_SUPPORTS_MULTIKERNEL @@ -34,9 +38,6 @@ config MULTIKERNEL If unsure, say N. -# Map the MSIs of an instance's PCI devices through the host's ITS -config MULTIKERNEL_ITS_PROXY - def_bool MULTIKERNEL && ARM_GIC_V3_ITS && PCI_MSI # Confine the DMA of an instance's PCI devices to its memory. x86 runs # them without an IOMMU so far. diff --git a/kernel/multikernel/Makefile b/kernel/multikernel/Makefile index 8ecf6d9f7ac647..725b4b2db8ce4d 100644 --- a/kernel/multikernel/Makefile +++ b/kernel/multikernel/Makefile @@ -5,7 +5,6 @@ obj-y += core.o cpuset.o mem.o contig.o kernfs.o dts.o instance_dt.o manifest.o ipi.o messaging.o overlay.o hotplug.o baseline.o obj-$(CONFIG_OF) += of_devices.o -obj-$(CONFIG_MULTIKERNEL_ITS_PROXY) += its_proxy.o obj-$(CONFIG_MULTIKERNEL_IOMMU) += iommu.o # DMA-BUF heap for multikernel memory allocation diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 3d3b57c4b5397e..4361c1e54ac29b 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -1605,7 +1605,7 @@ static void mk_instance_settle_halted(struct mk_instance *instance) { pr_info("Instance %d (%s) halted, CPUs parking in pool\n", instance->id, instance->name); - mk_msi_proxy_release(instance); + mk_arch_msi_release(instance); mk_iommu_release(instance); mk_instance_set_state(instance, MK_STATE_LOADED); } @@ -1751,7 +1751,7 @@ int multikernel_halt_by_id(int mk_id) pr_warn("Multikernel instance %d halted with CPUs unaccounted for\n", mk_id); - mk_msi_proxy_release(instance); + mk_arch_msi_release(instance); mk_iommu_release(instance); mk_instance_set_state(instance, MK_STATE_LOADED); pr_info("Multikernel instance %d halted (graceful)\n", mk_id); diff --git a/kernel/multikernel/iommu.c b/kernel/multikernel/iommu.c index 7ac0eae567ce3d..e66e22b379f588 100644 --- a/kernel/multikernel/iommu.c +++ b/kernel/multikernel/iommu.c @@ -59,7 +59,7 @@ static void mk_iommu_free_dev(struct mk_iommu_dev *d) static int mk_iommu_contain_dev(struct mk_instance *instance, struct pci_dev *pdev) { - phys_addr_t doorbell = mk_msi_proxy_doorbell(pdev); + phys_addr_t doorbell = mk_arch_msi_doorbell(pdev); struct mk_memory_region *region; struct mk_iommu_dev *d; int ret; From d761784853b3fef91a7ca1edb94f06b2276a82a4 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 19:58:15 -0700 Subject: [PATCH 20/30] irqchip/gic-v3-its: Move an LPI of another kernel from any context its_foreign_map() moves an event that is mapped already, but it takes the device allocation mutex, so it can only run in process context. The kernel that owns the interrupt does not have that luxury: its irq_set_affinity() callback runs with the descriptor locked, possibly on a CPU that is on its way out and gets turned off right after. This kernel's own its_set_affinity() is done when it returns, MOVI sent and a pending interrupt taken along, and the other kernel needs the same from its host. Add its_foreign_move() for that. It may run in an interrupt handler of this kernel, so what keeps the device from going away under it is a raw spinlock: a move holds it from the lookup to the MOVI, and its_foreign_release() takes it to disown a device before tearing it down, after which no move finds it and none is under way. The mapping path holds it around its ITS commands as well, so that a map and a move of the same event do not interleave. Signed-off-by: Cong Wang --- drivers/irqchip/irq-gic-v3-its.c | 65 +++++++++++++++++++++++++++--- include/linux/irqchip/arm-gic-v3.h | 2 + 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index f742908c122a55..c20bf2ab5e634e 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3800,11 +3800,19 @@ static const struct irq_domain_ops its_domain_ops = { * written here as well. No Linux interrupt exists for these LPIs on this * side: they never reach one of this kernel's CPU interfaces. * + * Mapping sleeps. Moving a mapped event must not, because the other kernel + * waits for it with interrupts off, like this kernel's own its_set_affinity() + * waits for its MOVI; it may run in this kernel's interrupt handlers. + * its_foreign_lock is what such a move holds against the device going away + * under it. + * * @owner keeps the devices of different kernels apart and names what * its_foreign_release() tears down. The ITS is the one behind the device, * named by the device's MSI domain: a machine has one per socket or more, * each with DeviceIDs, collections and a doorbell of its own. */ +static DEFINE_RAW_SPINLOCK(its_foreign_lock); + static struct its_node *its_foreign_its(struct irq_domain *domain) { if (!domain || domain->ops != &its_domain_ops) @@ -3845,6 +3853,44 @@ static struct its_collection *its_foreign_collection(struct its_node *its, return &its->collections[cpu]; } +/** + * its_foreign_move - Move a mapped event to another CPU of the other kernel + * @owner: the other kernel, as given to its_foreign_map() + * @domain: MSI domain of the device + * @dev_id: DeviceID as that ITS sees it + * @event: event to move, which its_foreign_map() has mapped + * @cpu: this kernel's number of the CPU to deliver to + * + * Returns once the ITS has moved the event, pending state included, so the + * CPU it came from may go away. Safe in any context. + * + * Returns 0 or a negative error code. + */ +int its_foreign_move(void *owner, struct irq_domain *domain, u32 dev_id, + u32 event, unsigned int cpu) +{ + struct its_node *its = its_foreign_its(domain); + struct its_collection *col; + struct its_device *dev; + + if (!its) + return -ENODEV; + col = its_foreign_collection(its, cpu); + if (!col) + return -EINVAL; + + guard(raw_spinlock_irqsave)(&its_foreign_lock); + + dev = its_find_device(its, dev_id); + if (!dev || dev->foreign != owner || event >= dev->event_map.nr_lpis || + !test_bit(event, dev->event_map.lpi_map)) + return -ENOENT; + + dev->event_map.col_map[event] = cpu; + its_send_movi(dev, col, event); + return 0; +} + /** * its_foreign_map - Route a device's event to a CPU of another kernel * @owner: the other kernel, as the caller names it @@ -3889,12 +3935,15 @@ int its_foreign_map(void *owner, struct irq_domain *domain, u32 dev_id, if (event >= dev->event_map.nr_lpis) return -ERANGE; - dev->event_map.col_map[event] = cpu; - if (test_and_set_bit(event, dev->event_map.lpi_map)) { - its_send_movi(dev, col, event); - } else { - its_send_mapti(dev, dev->event_map.lpi_base + event, event); - its_foreign_config(dev, event, true); + scoped_guard(raw_spinlock_irqsave, &its_foreign_lock) { + dev->event_map.col_map[event] = cpu; + if (test_and_set_bit(event, dev->event_map.lpi_map)) { + its_send_movi(dev, col, event); + } else { + its_send_mapti(dev, dev->event_map.lpi_base + event, + event); + its_foreign_config(dev, event, true); + } } *doorbell = its->get_msi_base(dev); return dev->event_map.lpi_base + event; @@ -3911,6 +3960,10 @@ static void its_foreign_release_on(struct its_node *its, void *owner) if (dev->foreign != owner) continue; + /* No move finds the device from here on, and none is under way */ + scoped_guard(raw_spinlock_irqsave, &its_foreign_lock) + dev->foreign = NULL; + for_each_set_bit(event, dev->event_map.lpi_map, dev->event_map.nr_lpis) { its_foreign_config(dev, event, false); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index b8db3ca26d8353..e563e3686e94e3 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -643,6 +643,8 @@ int gic_v3_send_sgi_to_mpidr(u64 mpidr, unsigned int sgi); int its_foreign_map(void *owner, struct irq_domain *domain, u32 dev_id, u32 event, u32 nvecs, unsigned int cpu, phys_addr_t *doorbell); +int its_foreign_move(void *owner, struct irq_domain *domain, u32 dev_id, + u32 event, unsigned int cpu); void its_foreign_release(void *owner); phys_addr_t its_foreign_doorbell(struct irq_domain *domain); int its_init(struct fwnode_handle *handle, struct rdists *rdists, From 643d8fd61341dfc06e25006a462bd75141c9634c Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 19:58:16 -0700 Subject: [PATCH 21/30] arm64/multikernel: Finish an MSI move before set_affinity returns An instance asked its host to move an MSI without waiting for the answer, because irq_set_affinity() runs with the descriptor locked. That is wrong in the case that matters most. A CPU going offline migrates its interrupts from inside stop_machine() and is turned off moments later, quite possibly before the host got to the request. LPIs aimed at a redistributor whose CPU firmware has put to sleep are not something the architecture promises to deliver later. A failure on the host's side was invisible too, and sending looked the host up under a mutex from atomic context. Give the move its own request, MK_IO_MSI_MOVE, and make it synchronous on both ends: - The instance sends it with mk_send_message_to() and polls the ring for the answer with mk_ipi_poll(), up to the second the ITS driver allows one of its own commands. Nobody else could take the doorbell: under stop_machine() every CPU has interrupts off. The host's verdict is what irq_set_affinity() returns. An interrupt stays where it is when the new mask allows that, which saves the round trip for most calls. - The host answers from its doorbell interrupt with its_foreign_move(). Neither the PCI device nor the instance can be looked up there, so MAP leaves a small route behind, the device's MSI domain and DeviceID and a reference to the instance, which MOVE finds under a raw spinlock and mk_arch_msi_release() drops. Requests now live on the requester's stack and echo what they asked for, so an answer finds its request whether it sleeps or polls. The host also sizes the device's ITT by the vectors the device has, MSI-X or MSI, instead of taking the number from the instance. The MSI core happens to pass the table size today, so nothing was broken, but how much the host allocates is not for an instance to decide. Tested on QEMU virt: with the NIC's interrupts pinned to its second CPU, a spawn takes that CPU offline and keeps its network, the interrupts continuing on CPU 0, booted from a device tree, under TF-A and from UEFI/ACPI. An NVMe disk, which frees and reallocates its vectors while probing, works in a spawn as well. Signed-off-by: Cong Wang --- arch/arm64/include/asm/multikernel.h | 12 +- arch/arm64/multikernel/its_proxy.c | 386 +++++++++++++++++++-------- drivers/irqchip/irq-gic-v3-its-mk.c | 33 ++- include/linux/multikernel.h | 1 + 4 files changed, 304 insertions(+), 128 deletions(-) diff --git a/arch/arm64/include/asm/multikernel.h b/arch/arm64/include/asm/multikernel.h index a499a7bdd5e837..382d37a7125827 100644 --- a/arch/arm64/include/asm/multikernel.h +++ b/arch/arm64/include/asm/multikernel.h @@ -64,13 +64,15 @@ static inline void mk_foreign_cpu_stop(void) #endif /* - * Ask the host to route a device's MSI to one of this kernel's CPUs, for - * the instance's MSI domain (irq-gic-v3-its-mk.c). Returns the interrupt - * number or -errno and sets @doorbell; a NULL @doorbell does not wait. + * Ask the host to route a device's MSI to one of this kernel's CPUs, and to + * move it to another later, for the instance's MSI domain + * (irq-gic-v3-its-mk.c). Mapping sleeps; moving works in any context and + * is done when it returns. */ #ifdef CONFIG_MULTIKERNEL_ITS_PROXY -int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, - u64 phys_cpu, phys_addr_t *doorbell); +int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u64 phys_cpu, + phys_addr_t *doorbell); +int mk_msi_proxy_move(u32 domain, u32 rid, u32 event, u64 phys_cpu); #endif /* diff --git a/arch/arm64/multikernel/its_proxy.c b/arch/arm64/multikernel/its_proxy.c index 80ff800d80aeea..eb279932ec33bc 100644 --- a/arch/arm64/multikernel/its_proxy.c +++ b/arch/arm64/multikernel/its_proxy.c @@ -5,10 +5,22 @@ * A GICv3 ITS has a single command queue, which belongs to the kernel * that booted the machine. An instance therefore cannot map the MSIs of * the devices it was given, and asks over the message ring instead: - * MK_IO_MSI_MAP names a device, an event and one of the instance's CPUs, - * the host programs its ITS and answers with the LPI that will arrive - * there. The instance writes address and data into the device itself, as - * it would with an ITS of its own. + * + * - MK_IO_MSI_MAP names a device, an event and one of the instance's + * CPUs. The host programs the ITS behind the device and answers with + * the LPI that will arrive there and the doorbell to write the event + * to. The instance asks from process context and sleeps; the host works + * in a work item, because the ITS driver sleeps too. + * - MK_IO_MSI_MOVE moves a mapped event to another CPU. It comes from an + * interrupt chip callback, with the descriptor locked and possibly from + * a dying CPU inside stop_machine(), and that CPU may be turned off the + * moment the callback returns. So the instance waits for the answer by + * polling the ring, and the host moves the event right in its doorbell + * interrupt, which takes a cache of what MAP resolved: neither the PCI + * device nor the instance can be looked up there. + * + * Both answers are MK_IO_MSI_ACK. There is no unmap: the host drops + * everything when the instance halts, and again before it is started. * * Both ends live here. The ITS side of the host is its_foreign_*() in the * ITS driver, the interrupt domain of the instance is irq-gic-v3-its-mk. @@ -17,6 +29,7 @@ #define pr_fmt(fmt) "mk_its: " fmt #include +#include #include #include #include @@ -25,51 +38,98 @@ #include #include -#define MK_MSI_TIMEOUT_MS 5000 +#define MK_MSI_MAP_TIMEOUT_MS 5000 +/* As long as the ITS driver gives one of its own commands */ +#define MK_MSI_MOVE_TIMEOUT_US USEC_PER_SEC +#define MK_MSI_MOVE_POLL_US 10 /* - * MSI routing request. Where MSIs go through a translation unit only one - * kernel can program (a GICv3 ITS), a spawn asks the kernel that owns it. - * @rid is the device as the spawn sees it on its PCI bus, @event the MSI - * data it will write. The answer names the interrupt that will arrive and - * the address to write @event to, which depends on the unit behind the - * device when the machine has several. Mapping an event again moves it to - * @phys_cpu, and everything is dropped by the host when the instance is - * started or torn down, so there is no unmap. + * @rid is the device as the instance sees it on its PCI bus, @event the + * MSI data it will write. */ struct mk_io_msi_payload { + u32 request; /* MK_IO_MSI_MAP or MK_IO_MSI_MOVE, echoed in the ACK */ u32 domain; /* PCI domain */ u32 rid; /* bus << 8 | devfn */ u32 event; - u32 nvecs; /* Events the device may use, fixed by the first request */ u64 phys_cpu; /* Target CPU, one of the sender's */ - u64 doorbell; /* ACK: address the device writes the event to */ - int result; /* ACK: interrupt number (LPI INTID), or -errno */ + u64 doorbell; /* ACK of a MAP: address the device writes the event to */ + int result; /* ACK: LPI INTID of a MAP, 0 of a MOVE, or -errno */ int sender_instance_id; }; +/* An answer belongs to the request it echoes */ +static bool mk_msi_same_request(const struct mk_io_msi_payload *a, + const struct mk_io_msi_payload *b) +{ + return a->request == b->request && a->domain == b->domain && + a->rid == b->rid && a->event == b->event; +} + +/* + * Host side + */ + struct mk_msi_work { struct work_struct work; struct mk_io_msi_payload req; }; -/* - * A request this kernel waits on. The answer is more than a result code, - * so it is kept here until the waiter picks it up. - */ -struct mk_msi_request { +/* What a MAP resolved about a device, for the MOVEs that follow */ +struct mk_msi_route { struct list_head list; - u64 key; - struct completion done; - struct mk_io_msi_payload answer; + struct mk_instance *instance; + u32 domain; + u32 rid; + struct irq_domain *msi_domain; + u32 dev_id; }; -static LIST_HEAD(mk_msi_requests); -static DEFINE_SPINLOCK(mk_msi_requests_lock); +static LIST_HEAD(mk_msi_routes); +static DEFINE_RAW_SPINLOCK(mk_msi_routes_lock); + +static struct mk_msi_route *mk_msi_find_route(int instance_id, u32 domain, u32 rid) +{ + struct mk_msi_route *route; + + list_for_each_entry(route, &mk_msi_routes, list) { + if (route->instance->id == instance_id && + route->domain == domain && route->rid == rid) + return route; + } + return NULL; +} -static u64 mk_msi_key(const struct mk_io_msi_payload *p) +static int mk_msi_remember_route(struct mk_instance *instance, + const struct mk_io_msi_payload *req, + struct irq_domain *msi_domain, u32 dev_id) { - return (u64)p->domain << 48 | (u64)p->rid << 32 | p->event; + struct mk_msi_route *route; + + scoped_guard(raw_spinlock_irqsave, &mk_msi_routes_lock) { + if (mk_msi_find_route(instance->id, req->domain, req->rid)) + return 0; + } + + route = kzalloc_obj(*route); + if (!route) + return -ENOMEM; + route->instance = mk_instance_get(instance); + route->domain = req->domain; + route->rid = req->rid; + route->msi_domain = msi_domain; + route->dev_id = dev_id; + + /* One work item at a time maps for a device, so nobody got in between */ + scoped_guard(raw_spinlock_irqsave, &mk_msi_routes_lock) + list_add(&route->list, &mk_msi_routes); + return 0; +} + +/* Every vector the device has: the instance's driver may use any of them */ +static u32 mk_msi_max_vectors(struct pci_dev *pdev) +{ + return max3(pci_msix_vec_count(pdev), pci_msi_vec_count(pdev), 1); } /* Only a device the instance was given, only a CPU it owns */ @@ -82,6 +142,7 @@ static int mk_msi_map_for(struct mk_instance *instance, struct pci_dev *pdev; bool owned = false; int cpu, ret; + u32 dev_id; list_for_each_entry(mine, &instance->pci_devices, list) { if (mine->domain == req->domain && mine->bus == req->rid >> 8 && @@ -104,16 +165,17 @@ static int mk_msi_map_for(struct mk_instance *instance, /* The device's MSI domain names the ITS behind it, of possibly several */ msi_domain = dev_get_msi_domain(&pdev->dev); - ret = its_foreign_map(instance, msi_domain, - pci_msi_domain_get_msi_rid(msi_domain, pdev), - req->event, req->nvecs, cpu, &doorbell); + dev_id = pci_msi_domain_get_msi_rid(msi_domain, pdev); + ret = its_foreign_map(instance, msi_domain, dev_id, req->event, + mk_msi_max_vectors(pdev), cpu, &doorbell); pci_dev_put(pdev); - if (ret >= 0) - req->doorbell = doorbell; - return ret; + if (ret < 0) + return ret; + + req->doorbell = doorbell; + return mk_msi_remember_route(instance, req, msi_domain, dev_id) ?: ret; } -/* The ITS driver sleeps, and messages arrive in interrupt context */ static void mk_msi_map_work(struct work_struct *work) { struct mk_msi_work *w = container_of(work, struct mk_msi_work, work); @@ -123,57 +185,156 @@ static void mk_msi_map_work(struct work_struct *work) instance = mk_instance_find(to); req->result = instance ? mk_msi_map_for(instance, req) : -ENOENT; - if (instance) - mk_instance_put(instance); - if (req->result < 0) pr_err("Instance %d: no MSI for %04x:%02x:%02x.%x event %u: %d\n", to, req->domain, req->rid >> 8, PCI_SLOT(req->rid), PCI_FUNC(req->rid), req->event, req->result); req->sender_instance_id = mk_self->id; - mk_send_message(to, MK_MSG_IO, MK_IO_MSI_ACK, req, sizeof(*req)); + if (instance) { + mk_send_message_to(instance, MK_MSG_IO, MK_IO_MSI_ACK, req, + sizeof(*req)); + mk_instance_put(instance); + } kfree(w); } +/* + * In the doorbell interrupt. The route keeps the instance alive, and the + * lock keeps the route: mk_arch_msi_release() takes it to unlink. Without + * a route there is nobody to answer: the instance never mapped the device, + * or has halted since, and its request times out. + */ +static void mk_msi_move(const struct mk_io_msi_payload *req) +{ + struct mk_io_msi_payload ack = *req; + struct mk_msi_route *route; + int cpu; + + guard(raw_spinlock_irqsave)(&mk_msi_routes_lock); + + route = mk_msi_find_route(req->sender_instance_id, req->domain, req->rid); + if (!route) + return; + + cpu = arch_cpu_from_physical_id(req->phys_cpu); + if (cpu < 0 || !mk_cpu_set_contains(route->instance->cpus, req->phys_cpu)) + ack.result = -EPERM; + else + ack.result = its_foreign_move(route->instance, route->msi_domain, + route->dev_id, req->event, cpu); + + ack.sender_instance_id = mk_self->id; + mk_send_message_to(route->instance, MK_MSG_IO, MK_IO_MSI_ACK, &ack, + sizeof(ack)); +} + +void mk_arch_msi_release(struct mk_instance *instance) +{ + struct mk_msi_route *route, *tmp; + LIST_HEAD(gone); + + scoped_guard(raw_spinlock_irqsave, &mk_msi_routes_lock) { + list_for_each_entry_safe(route, tmp, &mk_msi_routes, list) { + if (route->instance == instance) + list_move(&route->list, &gone); + } + } + + list_for_each_entry_safe(route, tmp, &gone, list) { + mk_instance_put(route->instance); + kfree(route); + } + + its_foreign_release(instance); +} + +phys_addr_t mk_arch_msi_doorbell(struct pci_dev *pdev) +{ + return its_foreign_doorbell(dev_get_msi_domain(&pdev->dev)); +} + +/* + * Instance side + */ + +/* On the requester's stack until the answer is in, or given up on */ +struct mk_msi_request { + struct list_head list; + struct completion done; + struct mk_io_msi_payload msg; /* the request, then its answer */ +}; + +static LIST_HEAD(mk_msi_requests); +static DEFINE_RAW_SPINLOCK(mk_msi_requests_lock); + static void mk_msi_answer(const struct mk_io_msi_payload *answer) { struct mk_msi_request *r; - unsigned long flags; - spin_lock_irqsave(&mk_msi_requests_lock, flags); + guard(raw_spinlock_irqsave)(&mk_msi_requests_lock); + list_for_each_entry(r, &mk_msi_requests, list) { - if (r->key == mk_msi_key(answer)) { - r->answer = *answer; + if (mk_msi_same_request(&r->msg, answer)) { + r->msg = *answer; complete(&r->done); - break; + return; } } - spin_unlock_irqrestore(&mk_msi_requests_lock, flags); } -static void mk_msi_msg_handler(u32 msg_type, u32 subtype, void *payload, - u32 payload_len, void *ctx) +static bool mk_msi_wait_sleeping(struct mk_msi_request *r) { - struct mk_io_msi_payload *p = payload; - struct mk_msi_work *w; - - if (payload_len < sizeof(*p)) - return; + return wait_for_completion_timeout(&r->done, + msecs_to_jiffies(MK_MSI_MAP_TIMEOUT_MS)); +} - switch (subtype) { - case MK_IO_MSI_MAP: - w = kzalloc(sizeof(*w), GFP_ATOMIC); - if (!w) - return; - w->req = *p; - INIT_WORK(&w->work, mk_msi_map_work); - schedule_work(&w->work); - break; - case MK_IO_MSI_ACK: - mk_msi_answer(p); - break; +/* + * Nobody may be there to take the doorbell: this CPU has interrupts off, + * and under stop_machine() so has every other. Drain the ring from here. + */ +static bool mk_msi_wait_polling(struct mk_msi_request *r) +{ + unsigned int waited; + + for (waited = 0; waited < MK_MSI_MOVE_TIMEOUT_US; + waited += MK_MSI_MOVE_POLL_US) { + mk_ipi_poll(); + if (try_wait_for_completion(&r->done)) + return true; + udelay(MK_MSI_MOVE_POLL_US); } + return false; +} + +static int mk_msi_request(struct mk_io_msi_payload *req, + bool (*wait)(struct mk_msi_request *r)) +{ + struct mk_msi_request r; + int ret; + + if (!host_instance) + return -ENODEV; + req->sender_instance_id = mk_self->id; + r.msg = *req; + init_completion(&r.done); + + scoped_guard(raw_spinlock_irqsave, &mk_msi_requests_lock) + list_add(&r.list, &mk_msi_requests); + + ret = mk_send_message_to(host_instance, MK_MSG_IO, req->request, req, + sizeof(*req)); + if (!ret && !wait(&r)) + ret = -ETIMEDOUT; + + /* Off the list before the stack frame goes: a late answer finds nothing */ + scoped_guard(raw_spinlock_irqsave, &mk_msi_requests_lock) + list_del(&r.list); + + if (ret) + return ret; + *req = r.msg; + return req->result; } /** @@ -181,69 +342,76 @@ static void mk_msi_msg_handler(u32 msg_type, u32 subtype, void *payload, * @domain: PCI domain of the device * @rid: bus << 8 | devfn * @event: MSI data the device will write - * @nvecs: events the device may use * @phys_cpu: target, one of this kernel's CPUs - * @doorbell: set to the address to write @event to. NULL moves an event - * that is mapped already without waiting for the answer, from a - * context that cannot sleep + * @doorbell: set to the address to write @event to * - * Returns the interrupt number (0 when not waiting) or a negative error. + * Sleeps. Returns the interrupt number or a negative error code. */ -int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u32 nvecs, - u64 phys_cpu, phys_addr_t *doorbell) +int mk_msi_proxy_map(u32 domain, u32 rid, u32 event, u64 phys_cpu, + phys_addr_t *doorbell) { struct mk_io_msi_payload req = { + .request = MK_IO_MSI_MAP, .domain = domain, .rid = rid, .event = event, - .nvecs = nvecs, .phys_cpu = phys_cpu, - .sender_instance_id = mk_self->id, }; - struct mk_msi_request *r; - unsigned long flags; - int ret; - - if (!doorbell) - return mk_send_message(0, MK_MSG_IO, MK_IO_MSI_MAP, &req, - sizeof(req)); - - r = kzalloc_obj(*r); - if (!r) - return -ENOMEM; - r->key = mk_msi_key(&req); - init_completion(&r->done); - - spin_lock_irqsave(&mk_msi_requests_lock, flags); - list_add(&r->list, &mk_msi_requests); - spin_unlock_irqrestore(&mk_msi_requests_lock, flags); + int ret = mk_msi_request(&req, mk_msi_wait_sleeping); - ret = mk_send_message(0, MK_MSG_IO, MK_IO_MSI_MAP, &req, sizeof(req)); - if (!ret && !wait_for_completion_timeout(&r->done, - msecs_to_jiffies(MK_MSI_TIMEOUT_MS))) - ret = -ETIMEDOUT; - - /* Off the list before the answer is read: a late one must not land in it */ - spin_lock_irqsave(&mk_msi_requests_lock, flags); - list_del(&r->list); - spin_unlock_irqrestore(&mk_msi_requests_lock, flags); - - if (!ret) { - ret = r->answer.result; - *doorbell = r->answer.doorbell; - } - kfree(r); + if (ret >= 0) + *doorbell = req.doorbell; return ret; } -void mk_arch_msi_release(struct mk_instance *instance) +/** + * mk_msi_proxy_move - Ask the host to move a routed MSI to another CPU + * @domain: PCI domain of the device + * @rid: bus << 8 | devfn + * @event: event that mk_msi_proxy_map() has routed + * @phys_cpu: new target, one of this kernel's CPUs + * + * Returns once the host has moved it, so the CPU it came from may go + * away. Any context. Returns 0 or a negative error code. + */ +int mk_msi_proxy_move(u32 domain, u32 rid, u32 event, u64 phys_cpu) { - its_foreign_release(instance); + struct mk_io_msi_payload req = { + .request = MK_IO_MSI_MOVE, + .domain = domain, + .rid = rid, + .event = event, + .phys_cpu = phys_cpu, + }; + + return mk_msi_request(&req, mk_msi_wait_polling); } -phys_addr_t mk_arch_msi_doorbell(struct pci_dev *pdev) +static void mk_msi_msg_handler(u32 msg_type, u32 subtype, void *payload, + u32 payload_len, void *ctx) { - return its_foreign_doorbell(dev_get_msi_domain(&pdev->dev)); + struct mk_io_msi_payload *p = payload; + struct mk_msi_work *w; + + if (payload_len < sizeof(*p)) + return; + + switch (subtype) { + case MK_IO_MSI_MAP: + w = kzalloc(sizeof(*w), GFP_ATOMIC); + if (!w) + return; + w->req = *p; + INIT_WORK(&w->work, mk_msi_map_work); + schedule_work(&w->work); + break; + case MK_IO_MSI_MOVE: + mk_msi_move(p); + break; + case MK_IO_MSI_ACK: + mk_msi_answer(p); + break; + } } static int __init mk_msi_proxy_init(void) diff --git a/drivers/irqchip/irq-gic-v3-its-mk.c b/drivers/irqchip/irq-gic-v3-its-mk.c index da57872ce3ae63..53a1e2ec1fcc36 100644 --- a/drivers/irqchip/irq-gic-v3-its-mk.c +++ b/drivers/irqchip/irq-gic-v3-its-mk.c @@ -56,27 +56,31 @@ static void mk_its_nop(struct irq_data *d) { } -/* @doorbell NULL: an event that is mapped already moves, and nobody waits */ -static int mk_its_route(struct mk_its_irq *irq, unsigned int cpu, - phys_addr_t *doorbell) -{ - struct mk_its_device *dev = irq->dev; - - return mk_msi_proxy_map(dev->domain, dev->rid, irq->event, dev->nvecs, - cpu_logical_map(cpu), doorbell); -} - -/* Called with the descriptor locked: the move is sent, not waited for */ +/* + * Called with the descriptor locked, and by a CPU on its way out that is + * turned off right after. Like the ITS driver's own, the move is complete + * when this returns: the host has sent the MOVI, which takes a pending + * interrupt along. + */ static int mk_its_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) { - unsigned int cpu = cpumask_first_and(mask, cpu_online_mask); + const struct cpumask *now = irq_data_get_effective_affinity_mask(d); + struct mk_its_irq *irq = irq_data_get_irq_chip_data(d); + unsigned int cpu; int ret; + /* Stay where it is if that is allowed: a move is a round trip to the host */ + cpu = cpumask_first_and_and(now, mask, cpu_online_mask); + if (cpu < nr_cpu_ids) + return IRQ_SET_MASK_OK_DONE; + + cpu = cpumask_first_and(mask, cpu_online_mask); if (cpu >= nr_cpu_ids) return -EINVAL; - ret = mk_its_route(irq_data_get_irq_chip_data(d), cpu, NULL); + ret = mk_msi_proxy_move(irq->dev->domain, irq->dev->rid, irq->event, + cpu_logical_map(cpu)); if (ret) return ret; @@ -159,7 +163,8 @@ static int mk_its_alloc_one(struct irq_domain *domain, unsigned int virq, irq->dev = mdev; irq->event = event; - lpi = mk_its_route(irq, cpu, &irq->doorbell); + lpi = mk_msi_proxy_map(mdev->domain, mdev->rid, event, + cpu_logical_map(cpu), &irq->doorbell); if (lpi < 0) { kfree(irq); return lpi; diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index e6ac4264d4e775..6403d7d56b1251 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -236,6 +236,7 @@ void mk_ipi_ring_drop_pending(void); /* MSI routing through the host, where the arch needs it: payload is the arch's */ #define MK_IO_MSI_MAP (MK_MSG_IO + 5) /* Route a device's MSI to one of my CPUs */ #define MK_IO_MSI_ACK (MK_MSG_IO + 6) /* Answer: the interrupt number, or an error */ +#define MK_IO_MSI_MOVE (MK_MSG_IO + 7) /* Move a routed MSI to another of my CPUs */ /* Resource management subtypes */ #define MK_RES_CPU_ADD (MK_MSG_RESOURCE + 1) From d39f32d97702b9d2e18b94bcdf32584c95a5138a Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 20:30:43 -0700 Subject: [PATCH 22/30] Documentation/multikernel: Pseudo-NMIs are required in an arm64 spawn The text offered irqchip.gicv3_pseudo_nmi=1 as an option. It is the only way force halt reaches a spawn CPU that has interrupts masked, and a CPU that cannot be stopped is lost to the host until reboot, so say that it is required, what it takes to build and boot with it, that kerf load adds the parameter, and where its reach ends. Signed-off-by: Cong Wang --- Documentation/multikernel/arm64.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Documentation/multikernel/arm64.rst b/Documentation/multikernel/arm64.rst index be8e05121cf1a3..434e5d40212192 100644 --- a/Documentation/multikernel/arm64.rst +++ b/Documentation/multikernel/arm64.rst @@ -40,12 +40,16 @@ off, instead of leaving them in a WFI loop inside an image the host is about to overwrite. The host learns that a CPU has left from ``AFFINITY_INFO``, and refuses to load a new image over one that has not. -**Force halt** sends the stop IPI to each CPU of the instance. A spawn -kernel booted with ``irqchip.gicv3_pseudo_nmi=1`` (and built with -``CONFIG_ARM64_PSEUDO_NMI``) takes it as an NMI, so it reaches a CPU that -spins with interrupts masked. Without pseudo-NMIs it is an ordinary -interrupt and such a CPU stays out of reach; the instance then cannot be -started again until the machine is rebooted. SDEI is not used yet. +**Force halt** sends the stop IPI to each CPU of the instance. PSCI has no +call to turn another CPU off, so the CPU has to take that interrupt and +turn itself off, and a CPU that spins with interrupts masked only takes it +as a pseudo-NMI. A spawn kernel therefore has to be built with +``CONFIG_ARM64_PSEUDO_NMI`` and booted with ``irqchip.gicv3_pseudo_nmi=1``; +``kerf load`` appends the parameter to every spawn's command line. Without +it such a CPU stays out of reach, and the instance cannot be started again +until the machine is rebooted: the host refuses to load an image over a +CPU that is still on. Pseudo-NMIs do not reach a CPU that has masked all +exceptions. SDEI is not used yet. The Boot Tree ============= From 93648f00c214868b2d6fcb14f26a475086bad34e Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Mon, 21 Sep 2026 09:46:22 -0700 Subject: [PATCH 23/30] multikernel: Keep a spawn's hands off the PCI bridges above its devices On a server every PCIe device sits behind a root port. A spawn has to see that port, or it could not reach the bus behind it, so the instance tree describes the bridges on the way down to a device. But the port still belongs to the kernel that spawned it, which has its port driver bound to it and may have other kernels' devices below it. A spawn given a NIC behind a root port on QEMU virt treated the port as its own: - pcieport bound to it a second time and asked the host for an MSI for a device the instance does not own; - the PCI core enabled the bridge on behalf of the NIC and rewrote its command register, which cleared the INTx disable bit the host had set, and it rewrites bridge windows and the port's side of ASPM the same way; - it cleared the port's AER status and enabled error reporting on it while it scanned the bus. Decide once, when a device is set up and has its node, whether the function is this kernel's: mk_pci_foreign() says it is not when the spawn's tree describes it without naming a device, which is how a bridge on the way down differs from a device that was given away. The answer is kept in struct pci_dev as "foreign", before the first config write, which is BAR sizing. The PCI core then - binds no driver to a foreign function, so none of the port services (AER, PME, hotplug, DPC) exist for it here; - drops config writes to it in pci_write_config_*(), next to the check that already drops them for a disconnected device, as if the function were read-only. That covers the core's own writes to parent bridges, the AER setup included, without chasing each of them: its owner configured it, and keeps it configured. A write from user space fails with -EPERM. Reads are untouched, so enumeration works as before. The spawn's host bridge keeps claiming the PCIe services as native: on a device the spawn owns that is right, its error reporting should be on so that the errors reach the port, where the host handles them. With this the root port's config space is byte for byte the same at boot, while a spawn uses the NIC behind it and after the spawn is killed. The NIC works in the spawn, behind an SMMUv3 too, the host binds and uses it in between, and a spawn CPU can go offline under its interrupts. Signed-off-by: Cong Wang --- drivers/pci/access.c | 9 +++++++++ drivers/pci/bus.c | 2 +- drivers/pci/probe.c | 2 ++ include/linux/multikernel.h | 5 +++++ include/linux/pci.h | 7 +++++++ kernel/multikernel/instance_dt.c | 21 +++++++++++++++++++++ 6 files changed, 45 insertions(+), 1 deletion(-) diff --git a/drivers/pci/access.c b/drivers/pci/access.c index b123da16b63ba3..828f80f78f944f 100644 --- a/drivers/pci/access.c +++ b/drivers/pci/access.c @@ -259,6 +259,9 @@ int pci_user_write_config_##size \ if (PCI_##size##_BAD) \ return -EINVAL; \ \ + if (dev->foreign) \ + return -EPERM; \ + \ raw_spin_lock_irq(&pci_lock); \ if (unlikely(dev->block_cfg_access)) \ pci_wait_cfg(dev); \ @@ -592,6 +595,8 @@ int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val) { if (pci_dev_is_disconnected(dev)) return PCIBIOS_DEVICE_NOT_FOUND; + if (dev->foreign) + return PCIBIOS_SUCCESSFUL; return pci_bus_write_config_byte(dev->bus, dev->devfn, where, val); } EXPORT_SYMBOL(pci_write_config_byte); @@ -600,6 +605,8 @@ int pci_write_config_word(const struct pci_dev *dev, int where, u16 val) { if (pci_dev_is_disconnected(dev)) return PCIBIOS_DEVICE_NOT_FOUND; + if (dev->foreign) + return PCIBIOS_SUCCESSFUL; return pci_bus_write_config_word(dev->bus, dev->devfn, where, val); } EXPORT_SYMBOL(pci_write_config_word); @@ -609,6 +616,8 @@ int pci_write_config_dword(const struct pci_dev *dev, int where, { if (pci_dev_is_disconnected(dev)) return PCIBIOS_DEVICE_NOT_FOUND; + if (dev->foreign) + return PCIBIOS_SUCCESSFUL; return pci_bus_write_config_dword(dev->bus, dev->devfn, where, val); } EXPORT_SYMBOL(pci_write_config_dword); diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c index 6c1ad1f542d9be..53b34c48d66ce6 100644 --- a/drivers/pci/bus.c +++ b/drivers/pci/bus.c @@ -368,7 +368,7 @@ void pci_bus_add_device(struct pci_dev *dev) */ pm_runtime_enable(&dev->dev); - if (!dn || of_device_is_available(dn)) + if ((!dn || of_device_is_available(dn)) && !dev->foreign) pci_dev_allow_binding(dev); device_initial_probe(&dev->dev); diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 7661517ebdf0df..d6ca9646e32f6a 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2036,6 +2036,8 @@ int pci_setup_device(struct pci_dev *dev) err = pci_set_of_node(dev); if (err) return err; + /* Known before the first write: sizing the BARs is one */ + dev->foreign = mk_pci_foreign(dev); pci_set_acpi_fwnode(dev); pci_dev_assign_slot(dev); diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 6403d7d56b1251..6ccdddc92ff6ce 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -1008,6 +1008,7 @@ void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size); /* Device probe filtering against the instance's allowlist */ bool mk_pci_should_probe(struct pci_bus *bus, int devfn); +bool mk_pci_foreign(const struct pci_dev *pdev); bool mk_platform_device_allowed(const char *name, const char *hid); /* Early CPU registration from the manifest (spawn kernels) */ @@ -1063,6 +1064,10 @@ static inline bool mk_pci_should_probe(struct pci_bus *bus, int devfn) { return true; } +static inline bool mk_pci_foreign(const struct pci_dev *pdev) +{ + return false; +} static inline bool mk_platform_device_allowed(const char *name, const char *hid) { return true; diff --git a/include/linux/pci.h b/include/linux/pci.h index 1c270f1d512301..79761d6895a440 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -490,6 +490,13 @@ struct pci_dev { * itself internal but devices downstream from it are external. */ unsigned int external_facing:1; + /* + * Seen by this kernel, owned by another: a bridge above the devices + * of a multikernel spawn, which the kernel that spawned it keeps + * driving. No driver binds to it and config writes are dropped, as + * if it were read-only. Its owner configured it. + */ + unsigned int foreign:1; unsigned int broken_intx_masking:1; /* INTx masking can't be used */ unsigned int io_window_1k:1; /* Intel bridge 1K I/O windows */ unsigned int irq_managed:1; diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 8503733372da18..3524179d043644 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -550,6 +550,27 @@ bool mk_pci_should_probe(struct pci_bus *bus, int devfn) } EXPORT_SYMBOL_GPL(mk_pci_should_probe); +/** + * mk_pci_foreign - Whether a PCI function this kernel sees is not its own + * @pdev: device being set up, its node attached + * + * A spawn kernel has to see the bridges above its devices, or it could not + * reach the buses behind them, but seeing is all. A port on the way down + * still belongs to the kernel that spawned this one: that kernel has a + * driver bound to it, and other kernels' devices may sit below it. In its + * tree a spawn tells the two apart by what the node says: one that names a + * device was given to it, a bridge on the way is described without one. + * + * Returns: true if the PCI core must leave the function alone + */ +bool mk_pci_foreign(const struct pci_dev *pdev) +{ + const struct device_node *np = pdev->dev.of_node; + + return mk_manifest_phys() && !(np && of_property_present(np, "vendor-id")); +} +EXPORT_SYMBOL_GPL(mk_pci_foreign); + /* * A netdev's alias is its interface name, the one the device had in the * kernel that gave it away, found from the node its device is bound to, From 360fd9818c6d4f1fd0bcf3ba0d5525f731dd483c Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 25 Sep 2026 09:52:26 -0700 Subject: [PATCH 24/30] arm64/multikernel: Keep the redistributor walk inside each mapping A server can describe its GIC redistributors one CPU at a time, with an address in each MADT GICC entry instead of a GICR range. The host's ACPI driver knows that each of those mappings holds one redistributor and stops there. A spawn boots from a device tree, so the host turns those addresses into separate reg entries in its GIC node. The DT driver did not keep that boundary. It walked from one redistributor to the next until GICR_TYPER.Last was set, even when only one redistributor had been mapped. If Last is clear, the next read is outside the mapping. This happens during interrupt setup, before the spawn's console or the stop IPI can help. QEMU's usual GICv3 redistributor ranges do not expose the lost boundary. Keep the size alongside each redistributor mapping, for both DT and ACPI, and stop the walk when it reaches that size. Last still ends a walk early, and ACPI's single-redistributor mappings still stop after one frame. Separate DT mappings no longer depend on the hardware marking each of their redistributors as the last one. The GICC conversion also assumed that every redistributor occupied 128 KiB. Read the distributor's architecture ID, as the native ACPI driver does, so a GICv4 mapping covers its full 256 KiB, including the virtual LPI frames. A simulated MMIO test of the driver's walk reads beyond an individual mapping before this change and stays inside it afterwards. Contiguous ranges, GICv4 frames, explicit strides and the existing stopping rules pass too. Both changed files compile for arm64. The server's first boot and force-kill/re-exec cycle still need hardware validation. Signed-off-by: Cong Wang --- arch/arm64/multikernel/dtb_acpi.c | 36 ++++++++++++++++++++++++++----- drivers/irqchip/irq-gic-v3.c | 14 +++++++++--- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/arch/arm64/multikernel/dtb_acpi.c b/arch/arm64/multikernel/dtb_acpi.c index b9ee0196321245..6e4c7ab28b95a0 100644 --- a/arch/arm64/multikernel/dtb_acpi.c +++ b/arch/arm64/multikernel/dtb_acpi.c @@ -11,14 +11,14 @@ #define pr_fmt(fmt) "mk_dtb: " fmt #include +#include #include +#include #include #include #include "internal.h" -/* A redistributor frame pair per CPU, where the MADT has no GICR entries */ -#define MK_GICR_PER_CPU_SIZE (2 * SZ_64K) #define MK_GICR_MAX_REGIONS 256 static int mk_dtb_add_psci(void *fdt) @@ -97,9 +97,28 @@ struct mk_gic_frames { u64 reg[2 * (1 + MK_GICR_MAX_REGIONS)]; /* distributor first */ int nr_gicr; int nr_gicc; + u32 gicr_size; bool have_gicd; }; +static int mk_gic_set_frame_size(struct mk_gic_frames *f) +{ + void __iomem *dist; + u32 arch; + + dist = ioremap(f->reg[0], SZ_64K); + if (!dist) + return -FDT_ERR_NOSPACE; + + arch = readl_relaxed(dist + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; + iounmap(dist); + if (arch != GIC_PIDR2_ARCH_GICv3 && arch != GIC_PIDR2_ARCH_GICv4) + return -FDT_ERR_BADVALUE; + + f->gicr_size = arch == GIC_PIDR2_ARCH_GICv4 ? 4 * SZ_64K : 2 * SZ_64K; + return 0; +} + /* * Redistributors are described either as GICR regions or, without any, * by each GICC's own frame address; the first kind wins, as in the @@ -146,7 +165,7 @@ static void mk_gic_collect(struct acpi_table_madt *madt, if (!gicc->gicr_base_address) continue; gicr[0] = gicc->gicr_base_address; - gicr[1] = MK_GICR_PER_CPU_SIZE; + gicr[1] = f->gicr_size; f->nr_gicr++; } } @@ -171,9 +190,16 @@ static int mk_dtb_add_gic(void *fdt) mk_gic_collect((void *)table, f, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR); mk_gic_collect((void *)table, f, ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR); - if (!f->nr_gicr) - mk_gic_collect((void *)table, f, ACPI_MADT_TYPE_GENERIC_INTERRUPT); + if (f->have_gicd && !f->nr_gicr) { + ret = mk_gic_set_frame_size(f); + if (!ret) + mk_gic_collect((void *)table, f, ACPI_MADT_TYPE_GENERIC_INTERRUPT); + } else { + ret = 0; + } acpi_put_table(table); + if (ret) + goto out; if (!f->have_gicd || !f->nr_gicr) { pr_err("The MADT describes no usable GICv3\n"); diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 31a322783bf9bd..a325405d05869d 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -50,6 +50,7 @@ static struct cpumask broken_rdists __read_mostly __maybe_unused; struct redist_region { void __iomem *redist_base; phys_addr_t phys_base; + resource_size_t size; bool single_redist; }; @@ -1080,6 +1081,10 @@ static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) if (typer & GICR_TYPER_VLPIS) ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ } + /* A DT region may describe one frame without its Last bit set. */ + if (ptr - gic_data.redist_regions[i].redist_base >= + gic_data.redist_regions[i].size) + break; } while (!(typer & GICR_TYPER_LAST)); } @@ -2333,6 +2338,7 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare goto out_unmap_rdist; } rdist_regs[i].phys_base = res.start; + rdist_regs[i].size = resource_size(&res); } if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) @@ -2378,12 +2384,14 @@ static struct } acpi_data __initdata; static void __init -gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) +gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base, + resource_size_t size) { static int count = 0; acpi_data.redist_regs[count].phys_base = phys_base; acpi_data.redist_regs[count].redist_base = redist_base; + acpi_data.redist_regs[count].size = size; acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; count++; } @@ -2408,7 +2416,7 @@ gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, gic_request_region(redist->base_address, redist->length, "GICR"); - gic_acpi_register_redist(redist->base_address, redist_base); + gic_acpi_register_redist(redist->base_address, redist_base, redist->length); return 0; } @@ -2450,7 +2458,7 @@ gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, (gicc->flags & ACPI_MADT_GICC_NON_COHERENT)) gic_data.rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; - gic_acpi_register_redist(gicc->gicr_base_address, redist_base); + gic_acpi_register_redist(gicc->gicr_base_address, redist_base, size); return 0; } From cca7aabb1fce18601c637d4ef46b687bc7339781 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Mon, 12 Oct 2026 21:08:05 -0700 Subject: [PATCH 25/30] firmware: arm_sdei: Let a caller signal event 0 to another CPU SDEI_EVENT_SIGNAL makes the software-signalled event 0 pending on any CPU, named by its MPIDR. Firmware delivers it as an EL3 interrupt, so the target's handler runs even while that CPU has every exception masked, which no interrupt the kernel can send itself achieves. Add the function ID and sdei_event_signal(). It is only a request: success means the event is pending, and firmware drops it on a CPU that has not registered event 0. Signed-off-by: Cong Wang --- drivers/firmware/arm_sdei.c | 10 ++++++++++ include/linux/arm_sdei.h | 7 +++++++ include/uapi/linux/arm_sdei.h | 1 + 3 files changed, 18 insertions(+) diff --git a/drivers/firmware/arm_sdei.c b/drivers/firmware/arm_sdei.c index f39ed7ba3a385e..7bcbaaa406c870 100644 --- a/drivers/firmware/arm_sdei.c +++ b/drivers/firmware/arm_sdei.c @@ -467,6 +467,16 @@ int sdei_event_disable(u32 event_num) return err; } +/* + * Success only means the event is now pending on @target_mpidr: firmware + * drops it if that CPU has not registered the event. + */ +int sdei_event_signal(u32 event_num, u64 target_mpidr) +{ + return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_SIGNAL, event_num, + target_mpidr, 0, 0, 0, NULL); +} + static int sdei_api_event_unregister(u32 event_num) { return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0, diff --git a/include/linux/arm_sdei.h b/include/linux/arm_sdei.h index f652a5028b5907..0810f44ef1ade9 100644 --- a/include/linux/arm_sdei.h +++ b/include/linux/arm_sdei.h @@ -48,11 +48,18 @@ int sdei_mask_local_cpu(void); int sdei_unmask_local_cpu(void); void __init acpi_sdei_init(void); void sdei_handler_abort(void); + +/* Only event 0, the software-signalled event, can be signalled */ +int sdei_event_signal(u32 event_num, u64 target_mpidr); #else static inline int sdei_mask_local_cpu(void) { return 0; } static inline int sdei_unmask_local_cpu(void) { return 0; } static inline void acpi_sdei_init(void) { } static inline void sdei_handler_abort(void) { } +static inline int sdei_event_signal(u32 event_num, u64 target_mpidr) +{ + return -EOPNOTSUPP; +} #endif /* CONFIG_ARM_SDE_INTERFACE */ diff --git a/include/uapi/linux/arm_sdei.h b/include/uapi/linux/arm_sdei.h index af0630ba5437d4..22eb61612673dc 100644 --- a/include/uapi/linux/arm_sdei.h +++ b/include/uapi/linux/arm_sdei.h @@ -22,6 +22,7 @@ #define SDEI_1_0_FN_SDEI_PE_UNMASK SDEI_1_0_FN(0x0C) #define SDEI_1_0_FN_SDEI_INTERRUPT_BIND SDEI_1_0_FN(0x0D) #define SDEI_1_0_FN_SDEI_INTERRUPT_RELEASE SDEI_1_0_FN(0x0E) +#define SDEI_1_0_FN_SDEI_EVENT_SIGNAL SDEI_1_0_FN(0x0F) #define SDEI_1_0_FN_SDEI_PRIVATE_RESET SDEI_1_0_FN(0x11) #define SDEI_1_0_FN_SDEI_SHARED_RESET SDEI_1_0_FN(0x12) From 627de43232ce240d8e953ca8799bd7f2fb35ff75 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Mon, 12 Oct 2026 21:08:19 -0700 Subject: [PATCH 26/30] firmware: arm_sdei: Leave shared events alone in a multikernel spawn The SDEI driver resets firmware's SDEI state at probe and again on reboot, panic and kexec: SDEI_PRIVATE_RESET on each of its CPUs, then SDEI_SHARED_RESET. Private events are per CPU, so a spawn kernel only resets its own. Shared events and interrupt bindings exist once for the machine, though, and the kernel that spawned this one may have them registered, for instance a GHES error source notified through SDEI. A spawn resetting them unregisters the host's handlers, or gets DENIED while one of them is in use and then disables SDEI for itself. Skip the shared reset in a spawn. Signed-off-by: Cong Wang --- drivers/firmware/arm_sdei.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/firmware/arm_sdei.c b/drivers/firmware/arm_sdei.c index 7bcbaaa406c870..87bb476a07a47d 100644 --- a/drivers/firmware/arm_sdei.c +++ b/drivers/firmware/arm_sdei.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -370,6 +371,14 @@ static int sdei_platform_reset(void) int err; on_each_cpu(&_ipi_private_reset, NULL, true); + + /* + * Shared events and interrupt bindings are machine-wide, and belong + * to the kernel that spawned this one. + */ + if (mk_spawned()) + return 0; + err = sdei_api_shared_reset(); if (err) { pr_err("Failed to reset platform: %d\n", err); From 29aeedf01a7132d15d46980f794bc80eee0b39b1 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Mon, 12 Oct 2026 21:09:35 -0700 Subject: [PATCH 27/30] multikernel: Tell a force halt which CPUs firmware can stop On arm64 the CPUs of another kernel can be stopped through firmware, with SDEI's software-signalled event, which reaches a CPU whatever it has masked. The request is not harmless, though: TF-A acknowledges the event at EL3 on a CPU that has SDEI masked and never completes it, which leaves that CPU unable to take any non-secure interrupt, the stop SGI included. Only the kernel on a CPU knows whether it can take the event at the moment. Give the shared area a byte per CPU, indexed like the parked table, that the kernel running on a CPU sets while firmware can stop it, and pass the instance to mk_force_stop_cpu() so the architecture can read it. x86 ignores both. Signed-off-by: Cong Wang --- arch/arm64/multikernel/spawn.c | 2 +- arch/x86/kernel/smp.c | 3 ++- include/linux/multikernel.h | 11 ++++++++++- kernel/multikernel/core.c | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/arch/arm64/multikernel/spawn.c b/arch/arm64/multikernel/spawn.c index f2ca53b96bb6bb..c000fdf9346ce4 100644 --- a/arch/arm64/multikernel/spawn.c +++ b/arch/arm64/multikernel/spawn.c @@ -113,7 +113,7 @@ core_initcall(mk_spawn_reboot_init); * is an ordinary SGI, which a CPU spinning with interrupts off never * sees. Without SDEI that is as far as force goes on arm64. */ -void mk_force_stop_cpu(mk_phys_cpu_t phys_cpu) +void mk_force_stop_cpu(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) { int ret = gic_v3_send_sgi_to_mpidr(phys_cpu, IPI_CPU_STOP_NMI); diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index a4bda8e87536c1..fba3b174227d21 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -446,6 +446,7 @@ device_initcall(mk_register_force_stop); /** * mk_force_stop_cpu - Send NMI to a specific CPU to force it to stop + * @instance: Instance the CPU belongs to * @phys_cpu: Physical CPU ID to stop * * The target belongs to a different kernel instance, so the NMI goes to @@ -454,7 +455,7 @@ device_initcall(mk_register_force_stop); * through it silently dropped the NMI for CPUs it missed. The ICR must * drain first, or back-to-back sends lose all but the first on xAPIC. */ -void mk_force_stop_cpu(u64 phys_cpu) +void mk_force_stop_cpu(struct mk_instance *instance, u64 phys_cpu) { safe_apic_wait_icr_idle(); apic_icr_write(APIC_DM_NMI | APIC_DEST_PHYSICAL, (u32)phys_cpu); diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 6ccdddc92ff6ce..8b6f599f362a2b 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -131,6 +131,15 @@ struct mk_shared_data { * the struct on re-exec. */ u8 parked[MK_PARKED_MAX]; + /* + * CPUs that firmware can stop out of band, indexed like parked[] + * and written by each CPU for itself: set once the architecture's + * firmware stop event can reach it, cleared before it stops being + * reachable. A force halt only asks firmware to stop the CPUs set + * here, because a request to any other CPU may leave it unable to + * take the stop interrupt as well. + */ + u8 fw_stop[MK_PARKED_MAX]; }; /* Function pointer type for IPI callbacks */ @@ -1198,7 +1207,7 @@ void __noreturn mk_enter_pool_state(void *info); * pseudo-NMI on arm64). Registration may fail where the architecture * has nothing suitable; graceful shutdown must keep working without it. */ -void mk_force_stop_cpu(mk_phys_cpu_t phys_cpu); +void mk_force_stop_cpu(struct mk_instance *instance, mk_phys_cpu_t phys_cpu); /* * Build the boot state for @instance from the loaded @image and start diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 4361c1e54ac29b..1486538d0bb973 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -1946,7 +1946,7 @@ int multikernel_force_halt_by_id(int mk_id) pr_err("Failed to arm force-halt marker: %d (sending NMI anyway)\n", ret); mk_cpu_set_for_each(i, phys_cpu, targets) { - mk_force_stop_cpu(phys_cpu); + mk_force_stop_cpu(instance, phys_cpu); cpu_count++; } From 8926823adc5af25f1a48af642c25b4d9a2679c78 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Mon, 12 Oct 2026 21:45:14 -0700 Subject: [PATCH 28/30] arm64/multikernel: Force halt through SDEI A force halt asks each CPU of an instance to turn itself off with the stop SGI. A CPU that spins with interrupts masked only takes it when its kernel runs with pseudo-NMIs, and none takes it once every exception is masked: such a CPU stays on, and the host refuses to load an image over it until the machine is rebooted. SDEI's software-signalled event 0 does reach it. Firmware takes it at EL3 and calls the handler the CPU's kernel registered, whatever that CPU has masked. So every multikernel kernel registers event 0 where firmware has SDEI, and the host signals it to each CPU of the instance before it sends the SGI. The handler checks the force-halt marker like the SGI path, completes the event, which TF-A needs before the CPU goes off, and turns the CPU off. A spawn finds SDEI in its boot tree, which gets a /firmware/sdei node with the host's conduit, from the host's own node or from its ACPI SDEI table. TF-A acknowledges event 0 on a CPU that has SDEI masked and never ends it, which leaves that CPU unable to take any non-secure interrupt, the stop SGI included. So the host only signals the CPUs that say, in the shared area, that they can take the event. A CPU says so once the SDEI driver has unmasked it and takes it back before anything masks it again: on the way off (local_cpu_stop() now leaves through the multikernel path before it masks SDEI), on hotplug and around power-down idle states. A CPU that masks SDEI between the host's check and its signal is still lost; that needs a firmware fix. On QEMU with TF-A, a spawn booted with irqchip.gicv3_pseudo_nmi=0 and a CPU wedged in LKDTM's HARDLOCKUP is now force halted, twice in a row, and runs again after each. Without SDEI the same spawn keeps that CPU and cannot be started again. Graceful halt, respawn and a force halt of an idle spawn still work. Signed-off-by: Cong Wang --- Documentation/multikernel/arm64.rst | 43 ++++-- arch/arm64/kernel/smp.c | 2 +- arch/arm64/multikernel/Makefile | 1 + arch/arm64/multikernel/dtb.c | 4 +- arch/arm64/multikernel/internal.h | 20 +++ arch/arm64/multikernel/sdei.c | 223 ++++++++++++++++++++++++++++ arch/arm64/multikernel/spawn.c | 16 +- 7 files changed, 292 insertions(+), 17 deletions(-) create mode 100644 arch/arm64/multikernel/sdei.c diff --git a/Documentation/multikernel/arm64.rst b/Documentation/multikernel/arm64.rst index 434e5d40212192..e14bfe33c69237 100644 --- a/Documentation/multikernel/arm64.rst +++ b/Documentation/multikernel/arm64.rst @@ -40,16 +40,31 @@ off, instead of leaving them in a WFI loop inside an image the host is about to overwrite. The host learns that a CPU has left from ``AFFINITY_INFO``, and refuses to load a new image over one that has not. -**Force halt** sends the stop IPI to each CPU of the instance. PSCI has no -call to turn another CPU off, so the CPU has to take that interrupt and -turn itself off, and a CPU that spins with interrupts masked only takes it -as a pseudo-NMI. A spawn kernel therefore has to be built with +**Force halt** has to make each CPU of the instance turn itself off, as +PSCI has no call to turn another CPU off. Where firmware implements SDEI, +every kernel registers SDEI's software-signalled event 0, and the host +first signals that event to each CPU with ``SDEI_EVENT_SIGNAL``. Firmware +takes it at EL3 and calls the spawn's handler whatever the CPU has +masked, all exceptions included. The handler completes the event and +turns the CPU off. Both kernels need ``CONFIG_ARM_SDE_INTERFACE``. + +TF-A (2.12, and its master branch at the time of writing) breaks a CPU +that receives event 0 while it has SDEI masked: it never ends the +interrupt, and the CPU takes no non-secure interrupt from then on. A +kernel masks SDEI on a CPU that is going off and around power-down idle +states, so each CPU publishes in the instance's shared area whether it +can take the event, and the host only signals the CPUs that say so. A +CPU that masks SDEI between the host's check and its signal still loses +its interrupts. + +The host then sends the stop IPI to every CPU of the instance, as it +does without SDEI. A CPU that spins with interrupts masked only takes it +as a pseudo-NMI, so a spawn kernel should also be built with ``CONFIG_ARM64_PSEUDO_NMI`` and booted with ``irqchip.gicv3_pseudo_nmi=1``; -``kerf load`` appends the parameter to every spawn's command line. Without -it such a CPU stays out of reach, and the instance cannot be started again -until the machine is rebooted: the host refuses to load an image over a -CPU that is still on. Pseudo-NMIs do not reach a CPU that has masked all -exceptions. SDEI is not used yet. +``kerf load`` appends the parameter to every spawn's command line. A CPU +that neither reaches stays out of reach, and the instance cannot be +started again until the machine is rebooted: the host refuses to load an +image over a CPU that is still on. The Boot Tree ============= @@ -73,6 +88,10 @@ instance's current resources, and adds to the instance tree: Copied from the host's tree, or written from the FADT, GTDT and MADT on an ACPI host. +``/firmware/sdei`` + Written when the host's tree has an SDEI node or its ACPI tables an + SDEI table, with the host's conduit. + ``/chosen`` ``bootargs``, the initrd and the seeds from the ``kexec_file_load()`` call, next to the multikernel properties, and @@ -147,3 +166,9 @@ firmware does so against real EL3 firmware and enters the kernels at EL2; ``iommu=smmuv3`` with ``iommu_platform=on,disable-legacy=on`` on a virtio PCI device puts that device behind an SMMU. A spawn needs no disk: an initramfs is enough. + +TF-A built with ``SDEI_SUPPORT=1 EL3_EXCEPTION_HANDLING=1`` provides SDEI +on QEMU, but neither QEMU nor TF-A adds an SDEI node to the tree. Dump +QEMU's tree with ``-machine ...,dumpdtb=virt.dtb``, add +``firmware { sdei { compatible = "arm,sdei-1.0"; method = "smc"; }; };`` +and boot the host with ``-dtb``. diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index fe0383e9e6bbfa..c4a7ee9f5ef62d 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -874,8 +874,8 @@ static void __noreturn local_cpu_stop(unsigned int cpu) set_cpu_online(cpu, false); local_daif_mask(); - sdei_mask_local_cpu(); mk_spawn_stop_this_cpu(); + sdei_mask_local_cpu(); cpu_park_loop(); } diff --git a/arch/arm64/multikernel/Makefile b/arch/arm64/multikernel/Makefile index 7928a5dfdea413..ca1f5398624a76 100644 --- a/arch/arm64/multikernel/Makefile +++ b/arch/arm64/multikernel/Makefile @@ -5,4 +5,5 @@ obj-y += spawn.o dtb.o dtb_of.o dtb_devices.o dtb_pci.o obj-$(CONFIG_ACPI) += dtb_acpi.o +obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_MULTIKERNEL_ITS_PROXY) += its_proxy.o diff --git a/arch/arm64/multikernel/dtb.c b/arch/arm64/multikernel/dtb.c index bb1720c2ed54ca..bfde8c9f33a648 100644 --- a/arch/arm64/multikernel/dtb.c +++ b/arch/arm64/multikernel/dtb.c @@ -6,7 +6,7 @@ * halves of what x86 splits between boot_params and the manifest: the * instance tree the generic code wrote, with its /resources and /chosen, * is the base, and the nodes the arm64 boot path itself reads go on top: - * /memory, /cpus, PSCI, the arch timer and the interrupt controller. + * /memory, /cpus, PSCI, the arch timer, the interrupt controller and SDEI. * * It is rebuilt on every spawn, into the device tree segment the loader * left room for: the memory grant and the CPUs can change between load @@ -208,6 +208,8 @@ int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance) if (!ret) ret = acpi_disabled ? mk_dtb_add_platform_of(fdt) : mk_dtb_add_platform_acpi(fdt); + if (!ret) + ret = mk_dtb_add_sdei(fdt); if (!ret) ret = mk_dtb_add_devices(fdt, instance); if (!ret) diff --git a/arch/arm64/multikernel/internal.h b/arch/arm64/multikernel/internal.h index 134dc6086913ec..1532d6e10cb9b4 100644 --- a/arch/arm64/multikernel/internal.h +++ b/arch/arm64/multikernel/internal.h @@ -45,4 +45,24 @@ static inline int mk_dtb_add_platform_acpi(void *fdt) } #endif +/* Force halt through SDEI, where firmware has it */ +#ifdef CONFIG_ARM_SDE_INTERFACE +int mk_dtb_add_sdei(void *fdt); +void mk_sdei_stop_cpu(struct mk_instance *instance, u64 mpidr); +void mk_sdei_unreachable(void); +#else +static inline int mk_dtb_add_sdei(void *fdt) +{ + return 0; +} + +static inline void mk_sdei_stop_cpu(struct mk_instance *instance, u64 mpidr) +{ +} + +static inline void mk_sdei_unreachable(void) +{ +} +#endif + #endif /* _ARM64_MULTIKERNEL_INTERNAL_H */ diff --git a/arch/arm64/multikernel/sdei.c b/arch/arm64/multikernel/sdei.c new file mode 100644 index 00000000000000..3a09a2b5ea186e --- /dev/null +++ b/arch/arm64/multikernel/sdei.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Force halt through SDEI. + * + * The stop SGI reaches a CPU that spins with interrupts masked only when + * its kernel runs with pseudo-NMIs, and never one that has masked every + * exception. SDEI's software-signalled event 0 is taken at EL3 and turned + * into a call of the handler its kernel registered, whatever that CPU has + * masked. So every kernel registers it, and a force halt signals it to the + * CPUs of an instance before it sends them the SGI. + * + * TF-A acknowledges event 0 on a CPU that has SDEI masked and never ends + * it, which leaves that CPU unable to take any non-secure interrupt. A CPU + * therefore publishes in fw_stop[] of its shared area whether it can take + * the event, and clears that before anything masks SDEI: going off, and + * around a power-down idle state. + */ + +#define pr_fmt(fmt) "mk_sdei: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* The software-signalled event, the only one SDEI_EVENT_SIGNAL raises */ +#define MK_SDEI_STOP_EVENT 0 + +static DEFINE_PER_CPU(u8 *, mk_fw_stop); + +#ifdef CONFIG_ACPI +static bool mk_acpi_has_sdei(void) +{ + struct acpi_table_header *table; + + if (acpi_disabled || ACPI_FAILURE(acpi_get_table(ACPI_SIG_SDEI, 0, &table))) + return false; + acpi_put_table(table); + return true; +} +#else +static bool mk_acpi_has_sdei(void) +{ + return false; +} +#endif + +/* The conduit to firmware's SDEI, as the SDEI driver finds it; NULL without */ +static const char *mk_sdei_method(void) +{ + struct device_node *np; + const char *method = NULL; + + np = of_find_compatible_node(NULL, NULL, "arm,sdei-1.0"); + if (np) { + of_property_read_string(np, "method", &method); + of_node_put(np); + } else if (mk_acpi_has_sdei()) { + method = acpi_psci_use_hvc() ? "hvc" : "smc"; + } + return method; +} + +int mk_dtb_add_sdei(void *fdt) +{ + const char *method = mk_sdei_method(); + int node, ret; + + if (!method) + return 0; + + node = fdt_subnode_offset(fdt, 0, "firmware"); + if (node == -FDT_ERR_NOTFOUND) + node = fdt_add_subnode(fdt, 0, "firmware"); + if (node >= 0) + node = fdt_add_subnode(fdt, node, "sdei"); + if (node < 0) + return node; + + ret = fdt_setprop_string(fdt, node, "compatible", "arm,sdei-1.0"); + if (!ret) + ret = fdt_setprop_string(fdt, node, "method", method); + return ret; +} + +static void mk_sdei_set_reachable(bool reachable) +{ + u8 *fw_stop = this_cpu_read(mk_fw_stop); + + if (fw_stop) + WRITE_ONCE(*fw_stop, reachable); +} + +void mk_sdei_unreachable(void) +{ + mk_sdei_set_reachable(false); +} + +void mk_sdei_stop_cpu(struct mk_instance *instance, u64 mpidr) +{ + struct mk_shared_data *sd = instance->ipi_data; + int rank = mk_cpu_rank(mpidr); + int ret; + + if (!sd || rank >= MK_PARKED_MAX || !READ_ONCE(sd->fw_stop[rank])) + return; + + ret = sdei_event_signal(MK_SDEI_STOP_EVENT, mpidr); + if (ret) + pr_err_ratelimited("No SDEI stop to CPU 0x%llx: %d\n", mpidr, ret); +} + +static int mk_sdei_stop(u32 event, struct pt_regs *regs, void *arg) +{ + if (!mk_has_pending_shutdown()) { + pr_warn_ratelimited("CPU%u: ignoring an SDEI stop from another kernel\n", + smp_processor_id()); + return 0; + } + + /* + * Complete the event before the CPU goes off. TF-A keeps the + * priority of a running handler active, and panics when the CPU + * takes the next event on top of it after its next CPU_ON. + */ + __sdei_handler_abort(); + __this_cpu_write(sdei_active_normal_event, NULL); + + mk_foreign_cpu_stop(); + return 0; +} + +/* + * The SDEI driver's own states and notifier mask and unmask the CPU. Its + * hotplug state comes first, so this one is set up after it unmasks and + * torn down before it masks; the PM notifiers sit on either side of its. + */ +static int mk_sdei_cpu_up(unsigned int cpu) +{ + struct mk_shared_data *sd = mk_self ? mk_self->ipi_data : NULL; + int rank = mk_cpu_rank(arch_cpu_physical_id(cpu)); + + if (sd && rank < MK_PARKED_MAX) + this_cpu_write(mk_fw_stop, &sd->fw_stop[rank]); + mk_sdei_set_reachable(true); + return 0; +} + +static int mk_sdei_cpu_down(unsigned int cpu) +{ + mk_sdei_set_reachable(false); + return 0; +} + +static int mk_sdei_pm_enter(struct notifier_block *nb, unsigned long action, + void *data) +{ + if (action == CPU_PM_ENTER) + mk_sdei_set_reachable(false); + else if (action == CPU_PM_ENTER_FAILED) + mk_sdei_set_reachable(true); + return NOTIFY_OK; +} + +static int mk_sdei_pm_exit(struct notifier_block *nb, unsigned long action, + void *data) +{ + if (action == CPU_PM_EXIT) + mk_sdei_set_reachable(true); + return NOTIFY_OK; +} + +static struct notifier_block mk_sdei_pm_enter_nb = { + .notifier_call = mk_sdei_pm_enter, + .priority = 1, +}; + +static struct notifier_block mk_sdei_pm_exit_nb = { + .notifier_call = mk_sdei_pm_exit, + .priority = -1, +}; + +static int __init mk_sdei_init(void) +{ + int ret; + + if (!mk_sdei_method()) + return 0; + + ret = sdei_event_register(MK_SDEI_STOP_EVENT, mk_sdei_stop, NULL); + if (ret) + goto out; + ret = sdei_event_enable(MK_SDEI_STOP_EVENT); + if (ret) + goto unregister; + + cpu_pm_register_notifier(&mk_sdei_pm_enter_nb); + cpu_pm_register_notifier(&mk_sdei_pm_exit_nb); + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "arm64/multikernel/sdei:online", + mk_sdei_cpu_up, mk_sdei_cpu_down); + if (ret >= 0) { + pr_info("Force halt stops this kernel's CPUs through SDEI\n"); + return 0; + } + + cpu_pm_unregister_notifier(&mk_sdei_pm_exit_nb); + cpu_pm_unregister_notifier(&mk_sdei_pm_enter_nb); + sdei_event_disable(MK_SDEI_STOP_EVENT); +unregister: + sdei_event_unregister(MK_SDEI_STOP_EVENT); +out: + pr_warn("No force halt through SDEI: %d\n", ret); + return 0; +} +late_initcall(mk_sdei_init); diff --git a/arch/arm64/multikernel/spawn.c b/arch/arm64/multikernel/spawn.c index c000fdf9346ce4..a3c8c0f63f09d1 100644 --- a/arch/arm64/multikernel/spawn.c +++ b/arch/arm64/multikernel/spawn.c @@ -71,6 +71,7 @@ void __noreturn mk_enter_pool_state(void *info) const struct cpu_operations *ops = get_cpu_ops(cpu); local_daif_mask(); + mk_sdei_unreachable(); sdei_mask_local_cpu(); if (ops && ops->cpu_die) @@ -107,16 +108,19 @@ core_initcall(mk_spawn_reboot_init); /* * PSCI cannot turn another CPU off, so a kernel that no longer answers - * is stopped in band, with the stop IPI its own smp_send_stop() would - * use. It is sent as IPI_CPU_STOP_NMI: a kernel running with pseudo-NMIs - * takes that one even with interrupts masked, and to any other kernel it - * is an ordinary SGI, which a CPU spinning with interrupts off never - * sees. Without SDEI that is as far as force goes on arm64. + * has to be made to turn its CPUs off itself. SDEI reaches a CPU whatever + * it has masked, but only one that said it can take the event. The stop + * IPI its own smp_send_stop() would use goes to every CPU regardless: as + * IPI_CPU_STOP_NMI a kernel running with pseudo-NMIs takes it even with + * interrupts masked, and to any other kernel it is an ordinary SGI. */ void mk_force_stop_cpu(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) { - int ret = gic_v3_send_sgi_to_mpidr(phys_cpu, IPI_CPU_STOP_NMI); + int ret; + + mk_sdei_stop_cpu(instance, phys_cpu); + ret = gic_v3_send_sgi_to_mpidr(phys_cpu, IPI_CPU_STOP_NMI); if (ret) pr_err("No stop SGI to CPU 0x%llx: %d\n", phys_cpu, ret); } From fc76f91ae5daa45f2c93817488081570bdf74868 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Tue, 13 Oct 2026 14:47:29 -0700 Subject: [PATCH 29/30] irqchip/gic-v3: Leave LPI setup to the host in tenant mode A tenant's CPUs run on redistributors whose LPIs the kernel that spawned it enabled, with that kernel's property and pending tables, and that kernel maps the tenant's MSIs on its ITSs. Yet the tenant still runs its_init() and its_cpu_init() like any other kernel. They only do nothing because the boot tree carries no ITS node: given one, its_cpu_init() would try to disable LPIs on each redistributor and take it over with tables of its own, while the host's ITSs keep delivering to it. Skip ITS and LPI setup in a tenant, at boot and when a CPU comes up. LPIs are still mapped in the GIC's domain, where the instance's MSI domain finds them. Signed-off-by: Cong Wang --- drivers/irqchip/irq-gic-v3.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index a325405d05869d..3cf751a5c9b7de 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1393,7 +1393,7 @@ static int gic_starting_cpu(unsigned int cpu) gic_cpu_sys_reg_enable(); gic_cpu_init(); - if (gic_dist_supports_lpis()) + if (gic_dist_supports_lpis() && !gic_data.tenant) its_cpu_init(); return 0; @@ -2152,6 +2152,13 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gic_smp_init(); gic_cpu_pm_init(); + /* + * A tenant's redistributors keep the LPI tables the host enabled + * them with, and the host programs the ITSs for it. + */ + if (gic_data.tenant) + return 0; + if (gic_dist_supports_lpis()) { its_init(handle, &gic_data.rdists, gic_data.domain, dist_prio_irq); its_cpu_init(); From 411bd1c489eae2543ba2cc1c6e01c9969c35b719 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Tue, 13 Oct 2026 15:35:15 -0700 Subject: [PATCH 30/30] arm64/multikernel: Give an instance no platform devices A platform device's interrupts are SPIs, and every SPI goes through the one GIC distributor, which stays with the host. Granting them took the boot tree a list of SPIs for the spawn's GIC driver to map, and still left both kernels writing the same distributor registers for them: the host could reroute a granted SPI or request it again, an instance that died while handling one left it active for the next kernel, and their trigger bits share words with SPIs of the host. Fixing all of that adds more shared state between the kernels than the devices are worth: they also need their clocks, resets and pin controllers, which stay with the host anyway. Give an arm64 instance PCI devices only, which interrupt with MSIs through the host's ITSs. Refuse to start an instance that has platform devices, stop copying their nodes into the boot tree, and drop "multikernel,spis": a tenant maps no SPIs at all. Signed-off-by: Cong Wang --- Documentation/multikernel/arm64.rst | 18 ++-- arch/arm64/multikernel/Makefile | 2 +- arch/arm64/multikernel/dtb.c | 25 ++++- arch/arm64/multikernel/dtb_devices.c | 133 --------------------------- arch/arm64/multikernel/dtb_of.c | 2 +- arch/arm64/multikernel/internal.h | 6 -- drivers/irqchip/irq-gic-v3.c | 50 +--------- 7 files changed, 37 insertions(+), 199 deletions(-) delete mode 100644 arch/arm64/multikernel/dtb_devices.c diff --git a/Documentation/multikernel/arm64.rst b/Documentation/multikernel/arm64.rst index e14bfe33c69237..53b96425422f56 100644 --- a/Documentation/multikernel/arm64.rst +++ b/Documentation/multikernel/arm64.rst @@ -112,11 +112,9 @@ a boot tree carries: ``multikernel,tenant`` The GICv3 driver leaves the distributor alone: it only checks that it is enabled with affinity routing, and initialises the redistributors - and CPU interfaces of its own CPUs. - -``multikernel,spis = ...`` - The SPIs the instance may map, as SPI numbers. Everything else fails - with ``-EPERM``. + and CPU interfaces of its own CPUs. It maps no SPIs, which fail with + ``-EPERM``, and sets up no LPIs of its own: the redistributors of its + CPUs keep the LPI tables the host enabled them with. The **doorbell** of the message ring is SGI 7. All eight non-secure SGIs are in use on arm64, so with ``CONFIG_MULTIKERNEL`` it takes the place of @@ -126,13 +124,9 @@ Devices ======= Platform devices - Granted by ``device-name``, the name under - ``/sys/bus/platform/devices``, for instance ``a003e00.virtio_mmio``. - The host copies the device's node into the boot tree and grants its - SPIs, so this takes a host that booted from a device tree, and a - device whose node refers to no clocks, resets, power domains or pin - controllers, which stay with the host. Unbind the host's driver first; - that also disables the SPIs on the host's side. + Not supported. Their interrupts are SPIs, and SPIs live in the one + distributor, which stays with the host. An instance that has platform + devices fails to start. PCI devices The root buses above an instance's devices appear as diff --git a/arch/arm64/multikernel/Makefile b/arch/arm64/multikernel/Makefile index ca1f5398624a76..ddf0495610e5ce 100644 --- a/arch/arm64/multikernel/Makefile +++ b/arch/arm64/multikernel/Makefile @@ -3,7 +3,7 @@ # Makefile for multikernel spawn support # -obj-y += spawn.o dtb.o dtb_of.o dtb_devices.o dtb_pci.o +obj-y += spawn.o dtb.o dtb_of.o dtb_pci.o obj-$(CONFIG_ACPI) += dtb_acpi.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_MULTIKERNEL_ITS_PROXY) += its_proxy.o diff --git a/arch/arm64/multikernel/dtb.c b/arch/arm64/multikernel/dtb.c index bfde8c9f33a648..7e2441ff188268 100644 --- a/arch/arm64/multikernel/dtb.c +++ b/arch/arm64/multikernel/dtb.c @@ -35,6 +35,29 @@ static const char * const mk_chosen_props[] = { "rng-seed", }; +/* + * Every wired interrupt goes through the one GIC distributor, which stays + * the host's, so an instance has no platform devices: its devices are PCI + * functions with MSIs. The spawn kernel's GIC driver leaves the + * distributor alone and maps no SPIs. + */ +static int mk_dtb_add_tenancy(void *fdt, struct mk_instance *instance) +{ + int node; + + if (instance->platform_devices_valid && + !list_empty(&instance->platform_devices)) { + pr_err("Instance %d: platform devices cannot be given to an instance on arm64\n", + instance->id); + return -FDT_ERR_BADSTATE; + } + + node = fdt_node_offset_by_compatible(fdt, -1, "arm,gic-v3"); + if (node < 0) + return node; + return fdt_setprop_empty(fdt, node, "multikernel,tenant"); +} + /* @pairs are base,size; the root of the instance tree has two cells for each */ int mk_dtb_set_reg(void *fdt, int node, const u64 *pairs, int nr) { @@ -211,7 +234,7 @@ int mk_build_boot_dtb(struct kimage *image, struct mk_instance *instance) if (!ret) ret = mk_dtb_add_sdei(fdt); if (!ret) - ret = mk_dtb_add_devices(fdt, instance); + ret = mk_dtb_add_tenancy(fdt, instance); if (!ret) ret = mk_dtb_add_pci(fdt); if (!ret) diff --git a/arch/arm64/multikernel/dtb_devices.c b/arch/arm64/multikernel/dtb_devices.c deleted file mode 100644 index c9d26c1a2db72e..00000000000000 --- a/arch/arm64/multikernel/dtb_devices.c +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Platform devices of an instance tree, and the instance's tenancy of - * the GIC. - * - * A spawn kernel probes what its tree describes, so granting a platform - * device is copying its node from the host tree. That takes a host that - * has one: on an ACPI host the device has no node to copy. Providers the - * node points at (clocks, resets, power domains, pinctrl) stay with the - * host and are not followed, which limits this to self-contained - * devices such as a virtio-mmio transport. - * - * The device's SPIs go with it. The spawn's GIC driver runs in tenant - * mode and maps nothing but the SPIs its node grants, and the host gives - * them up by unbinding its driver, which leaves them disabled at the - * distributor until a driver binds again after the instance is gone. - */ - -#define pr_fmt(fmt) "mk_dtb: " fmt - -#include -#include -#include -#include -#include - -#include "internal.h" - -#define MK_GIC_SPI 0 - -static int mk_dtb_gic_node(void *fdt) -{ - return fdt_node_offset_by_compatible(fdt, -1, "arm,gic-v3"); -} - -/* - * Write the device's interrupts as plain GIC specifiers and grant each - * SPI: whatever interrupt-parent or interrupt-map the host resolves them - * through is not part of the instance tree. - */ -static int mk_dtb_grant_irqs(void *fdt, const char *path, struct device_node *np) -{ - struct of_phandle_args irq; - int i, c, node, ret; - - node = fdt_path_offset(fdt, path); - if (node < 0) - return node; - fdt_delprop(fdt, node, "interrupts"); - fdt_delprop(fdt, node, "interrupts-extended"); - fdt_delprop(fdt, node, "interrupt-parent"); - - for (i = 0; !of_irq_parse_one(np, i, &irq); i++) { - bool spi = irq.args_count == 3 && irq.args[0] == MK_GIC_SPI && - of_device_is_compatible(irq.np, "arm,gic-v3"); - - of_node_put(irq.np); - if (!spi) { - pr_err("%pOF: interrupt %d is no GICv3 SPI\n", np, i); - return -FDT_ERR_BADVALUE; - } - - node = fdt_path_offset(fdt, path); - for (c = 0; c < 3 && node >= 0; c++) { - ret = fdt_appendprop_u32(fdt, node, "interrupts", irq.args[c]); - if (ret) - return ret; - } - - node = mk_dtb_gic_node(fdt); - if (node < 0) - return node; - ret = fdt_appendprop_u32(fdt, node, "multikernel,spis", irq.args[1]); - if (!ret) - ret = fdt_appendprop_u32(fdt, node, "multikernel,spis", 1); - if (ret) - return ret; - } - return 0; -} - -static int mk_dtb_add_device(void *fdt, const char *name) -{ - struct device *dev; - char path[64]; - int node, ret; - - dev = bus_find_device_by_name(&platform_bus_type, NULL, name); - if (!dev || !dev->of_node) { - pr_warn("Platform device %s has no node to describe it to an instance\n", - name); - put_device(dev); - return 0; - } - - if (dev->driver) { - pr_err("Platform device %s is still bound to %s here\n", - name, dev->driver->name); - put_device(dev); - return -FDT_ERR_BADSTATE; - } - - node = mk_dtb_copy_of_node(fdt, dev->of_node); - ret = node < 0 ? node : 0; - if (!ret) { - snprintf(path, sizeof(path), "/%pOFf", dev->of_node); - ret = mk_dtb_grant_irqs(fdt, path, dev->of_node); - } - put_device(dev); - return ret; -} - -int mk_dtb_add_devices(void *fdt, struct mk_instance *instance) -{ - struct mk_platform_device *plat; - int node, ret; - - node = mk_dtb_gic_node(fdt); - if (node < 0) - return node; - ret = fdt_setprop_empty(fdt, node, "multikernel,tenant"); - if (ret || !instance->platform_devices_valid) - return ret; - - list_for_each_entry(plat, &instance->platform_devices, list) { - if (!plat->name[0]) - continue; - ret = mk_dtb_add_device(fdt, plat->name); - if (ret) - return ret; - } - return 0; -} diff --git a/arch/arm64/multikernel/dtb_of.c b/arch/arm64/multikernel/dtb_of.c index 9f0d07cf565fdb..6da52cf463267c 100644 --- a/arch/arm64/multikernel/dtb_of.c +++ b/arch/arm64/multikernel/dtb_of.c @@ -37,7 +37,7 @@ static bool mk_prop_is_addressing(const char *name) * nodes keep working. Addressing is only valid below @np's parent bus, * so reg is written again, translated to CPU addresses. */ -int mk_dtb_copy_of_node(void *fdt, struct device_node *np) +static int mk_dtb_copy_of_node(void *fdt, struct device_node *np) { u64 reg[2 * MK_DTB_MAX_REG]; struct property *pp; diff --git a/arch/arm64/multikernel/internal.h b/arch/arm64/multikernel/internal.h index 1532d6e10cb9b4..cfe073b1919dcc 100644 --- a/arch/arm64/multikernel/internal.h +++ b/arch/arm64/multikernel/internal.h @@ -21,12 +21,6 @@ int mk_dtb_set_reg(void *fdt, int node, const u64 *pairs, int nr); /* reg entries a copied node may have; a GIC has one per redistributor region */ #define MK_DTB_MAX_REG 16 -struct device_node; -int mk_dtb_copy_of_node(void *fdt, struct device_node *np); - -/* Granted platform devices, and the SPIs the GIC node grants with them */ -int mk_dtb_add_devices(void *fdt, struct mk_instance *instance); - /* The driver and the MSI controller for the instance's PCI root buses */ int mk_dtb_add_pci(void *fdt); diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 3cf751a5c9b7de..f81446c59ad91f 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -69,8 +69,6 @@ struct gic_chip_data { struct partition_affinity *parts; unsigned int nr_parts; bool tenant; - u32 *tenant_spis; - unsigned int nr_tenant_spis; }; struct partition_affinity { @@ -932,26 +930,8 @@ static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) * those of its own CPUs like any other kernel. The distributor exists * once: initialising it would disable, regroup and reroute every SPI * of the machine, the host's included. A tenant leaves it as the host - * set it up and only ever touches the per-interrupt state of the SPIs - * it was granted ("multikernel,spis", pairs of SPI - * numbers), so everything else must not even map. + * set it up and maps no SPIs; its devices interrupt with MSIs. */ -static bool gic_tenant_owns(irq_hw_number_t hw) -{ - unsigned int i; - - if (__get_intid_range(hw) != SPI_RANGE) - return false; - - for (i = 0; i < gic_data.nr_tenant_spis; i += 2) { - u32 first = gic_data.tenant_spis[i] + 32; - - if (hw >= first && hw - first < gic_data.tenant_spis[i + 1]) - return true; - } - return false; -} - static int __init gic_tenant_check_dist(void) { u32 need = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A; @@ -963,30 +943,10 @@ static int __init gic_tenant_check_dist(void) return -ENXIO; } - pr_info("Tenant mode, %u granted SPI range(s)\n", - gic_data.nr_tenant_spis / 2); + pr_info("Tenant mode: the distributor is another kernel's\n"); return 0; } -static void __init gic_of_tenant_init(struct device_node *node) -{ - int n; - - gic_data.tenant = of_property_read_bool(node, "multikernel,tenant"); - if (!gic_data.tenant) - return; - - n = of_property_count_u32_elems(node, "multikernel,spis"); - if (n < 2 || n % 2) - return; - - gic_data.tenant_spis = kcalloc(n, sizeof(u32), GFP_KERNEL); - if (gic_data.tenant_spis && - !of_property_read_u32_array(node, "multikernel,spis", - gic_data.tenant_spis, n)) - gic_data.nr_tenant_spis = n; -} - static void __init gic_dist_init(void) { unsigned int i; @@ -1661,7 +1621,7 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, case SPI_RANGE: case ESPI_RANGE: - if (gic_data.tenant && !gic_tenant_owns(hw)) + if (gic_data.tenant) return -EPERM; irq_domain_set_info(d, irq, hw, chip, d->host_data, handle_fasteoi_irq, NULL, NULL); @@ -2127,7 +2087,7 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gic_data.has_rss = !!(typer & GICD_TYPER_RSS); - /* MBIs are SPIs used as MSIs, and a tenant was granted none for that */ + /* MBIs are SPIs used as MSIs, and a tenant maps no SPIs */ if ((typer & GICD_TYPER_MBIS) && !gic_data.tenant) { err = mbi_init(handle, gic_data.domain); if (err) @@ -2352,7 +2312,7 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare redist_stride = 0; gic_enable_of_quirks(node, gic_quirks, &gic_data); - gic_of_tenant_init(node); + gic_data.tenant = of_property_read_bool(node, "multikernel,tenant"); err = gic_init_bases(dist_phys_base, dist_base, rdist_regs, nr_redist_regions, redist_stride, &node->fwnode);