diff --git a/Documentation/ABI/testing/multikernel b/Documentation/ABI/testing/multikernel new file mode 100644 index 00000000000000..74378c0dfcf4ec --- /dev/null +++ b/Documentation/ABI/testing/multikernel @@ -0,0 +1,84 @@ +What: /sys/fs/multikernel/instances//stats +Date: August 2026 +Contact: Nikolay Nikolaev +Description: + Read-only reliability snapshot for one multikernel instance. The + format is one ``key value`` pair per line. The first line is always + ``stats_version 1``. Versioned formats are append-only; readers must + ignore unknown keys. + + The snapshot is observational, not atomic with concurrent producers. + Cumulative transport counters are unsigned 32-bit event counts and + wrap modulo 2^32. They reset when the shared buffer is initialized + for a new ``spawn_epoch`` and cannot be reset through this file. + Compare deltas only between samples with the same nonzero epoch. + Gauges are unsigned point-in-time counts and have no reset operation. + + Metadata keys (class; unit; meaning): + + ``stats_version`` (metadata; version; this file format, currently 1), + ``abi_version`` (metadata; version; exact shared transport ABI), + ``snapshot_atomic`` (metadata; boolean; always 0), + ``transport_counter_bits`` (metadata; bits; always 32), + ``transport_available`` (gauge; boolean; shared buffer is mapped), and + ``spawn_epoch`` (metadata; launch identifier; changes on each start). + + ``abi_version`` identifies the exact private kernel-to-kernel transport + contract used by the running instance. Exposing it here supports + diagnostics; it is not a stable userspace ABI or a compatibility promise + between transport versions. + + Ordered IPI keys (class; unit; increment or sampled condition; reset): + + The duplex transport serializes each direction with a local spinlock and + publishes a single shared READY bit. Consequently + ``ipi.producer_contention``, ``ipi.invalid_state``, + ``ipi.cancelled_writes``, ``ipi.gate_busy``, ``ipi.slot_writing``, + ``ipi.slot_consuming``, and ``ipi.slot_cancelled`` remain zero. They are + retained in version 1 for reader compatibility. + + ``ipi.producer_contention`` (compatibility counter; always zero), + ``ipi.full_failures`` (counter; sends; either direction's head slot + prevented publication; new epoch), + ``ipi.invalid_state`` and ``ipi.cancelled_writes`` (compatibility + counters; always zero), ``ipi.gate_busy`` and ``ipi.slot_writing`` + (compatibility gauges; always zero), ``ipi.slot_ready`` (gauge; READY + slots summed across both directions; sampled), and + ``ipi.slot_consuming`` and ``ipi.slot_cancelled`` (compatibility gauges; + always zero). + + Direct reply keys (class; unit; increment or sampled condition; reset): + + ``reply.late_replies`` (counter; replies; stale exact-token claim or + completion was rejected; new epoch), ``reply.cancelled_slots`` + (counter; slots; timeout + cancelled RESERVED/WRITING; new epoch), ``reply.atomic_timeouts`` + (counter; waits; bounded atomic wait expired in cancellable + RESERVED/WRITING or indeterminate EXECUTING state; new epoch), + ``reply.indeterminate_timeouts`` (counter; waits; subset where + EXECUTING won and changed to COMMITTED; new epoch), + ``reply.occupied_failures`` (counter; + reservations; no FREE slot; new epoch), ``reply.slot_busy`` (gauge; + slots; state is not FREE; sampled), ``reply.slot_reserved`` (gauge; + slots; RESERVED; sampled), ``reply.slot_writing`` (gauge; slots; + WRITING; sampled), ``reply.slot_executing`` (gauge; slots; EXECUTING; + sampled), ``reply.slot_committed`` (gauge; slots; COMMITTED; sampled), + ``reply.slot_ready`` (gauge; slots; READY; sampled), and + ``reply.slot_abandoned`` (gauge; slots; ABANDONED; sampled). + + Pending IRQ keys (class; unit; increment or sampled condition; reset): + + ``irq.recorded`` (counter; IRQs; pending count recorded; new epoch), + ``irq.coalesced`` (counter; IRQs; recorded onto an already-pending + slot; new epoch), ``irq.masked_deferred`` (counter; IRQs; delivery + deferred while logically masked; new epoch), ``irq.stale`` (counter; + observations; epoch/generation/route validation rejected work; new + epoch), ``irq.dispatch_failed`` (counter; IRQs; validated local dispatch + failed; new epoch), ``irq.saturated`` (counter; IRQs; pending count hit + its representable limit; new epoch), ``irq.slot_active`` (gauge; slots; + nonzero generation; sampled), ``irq.slot_pending`` (gauge; slots; + nonzero pending count; sampled), ``irq.slot_masked`` (gauge; slots; + MASKED set; sampled), ``irq.slot_consuming`` (gauge; slots; CONSUMING + set; sampled), and ``irq.pending_total`` (gauge; IRQs; sum of pending + counts across active slots; sampled). +Users: multikernel management and diagnostic tools diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index 8ed85d8ead7e18..955bea57f01f64 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -66,6 +66,20 @@ Workflow ``device-remove``, and destroy the instance with ``instance-remove`` once it is stopped. +SR-IOV Assignment Boundary +=========================== + +SR-IOV assignment is intended for cooperative spawned kernels. Filtering +configuration-space access and enumerating only assigned BDFs prevents +accidental access; it is not a security boundary against a privileged kernel +that deliberately issues configuration cycles or maps host physical windows. + +The host keeps exclusive leases for assigned VFs. Assignment fails unless the +device is an SR-IOV VF whose identity and ownership can be validated. Host +drivers remain bound while a VF is merely in the pool and are detached only +while committing a lease. Nested kernels cannot establish this host-owned +lifecycle and therefore cannot lease PCI devices to their children. + Instance States =============== @@ -75,6 +89,31 @@ Instance States - **active**: the kernel is running - **failed**: an error occurred; check ``dmesg`` +Transport Reliability Statistics +================================ + +Each instance exposes a read-only ``stats`` file. Version 1 is an +append-only key/value interface covering the duplex IPI transport, direct +reply slots, and pending IRQ mailbox. Readers must ignore unknown keys. + +The snapshot is observational rather than atomic. ``transport_available`` +reports whether the instance currently has shared transport state, and +``spawn_epoch`` identifies the launch whose counters are shown. Transport +counters are unsigned 32-bit values which may wrap and are reset when a new +epoch is initialized; compare them as modulo-32-bit deltas only between +samples with the same nonzero epoch. + +``ipi.*`` reports producer failures and current per-direction slot occupancy, +``reply.*`` reports timeout/recovery counters and reply-slot states, and +``irq.*`` reports recorded, coalesced, deferred, stale, failed, saturated, +active, pending, masked, and consuming mailbox state. Gauge values describe +only the instant at which each field was sampled. + +The duplex ready-bit protocol has no shared producer gate or intermediate +WRITING, CONSUMING, CANCELLED, or invalid states. The corresponding legacy +version-1 keys remain present and report zero; ``ipi.slot_ready`` and +``ipi.full_failures`` are summed across both link directions. + Restrictions ============ @@ -83,3 +122,8 @@ Restrictions - Instance files are read-only; an instance's resources change through overlays targeting ``/instances/``. - Rollback (``rmdir`` on a transaction) cannot destroy a running instance. +- Logical CPU 0 handles assigned-device MSI forwarding and must remain online + while those devices are active. +- Assigned MSI events are recorded in per-instance shared mailboxes before a + doorbell is sent. Masked events remain pending and lost doorbells are retried; + each spawn drains only its own parent-link mailbox. diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..6758247c93fd7e 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF_MK XLF_MULTIKERNEL_IPI +#else +# define XLF_MK 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF_MK cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 3e53e9cbb82c56..0faabfeee9d4ca 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -11,6 +11,7 @@ #ifndef __ASSEMBLY__ #include +#include #include #include #include @@ -127,7 +128,7 @@ struct mk_spawn_context { u32 target_apic_id; /* Target CPU's APIC ID */ u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ - u32 reserved; /* Padding for alignment */ + u32 abi_magic; /* Host/spawn generation marker */ /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ /* Optional boot data belongs after boot_params, in the zeroed tail. */ @@ -195,7 +196,8 @@ void mk_set_spawn_context(struct mk_spawn_context *ctx, int mk_spawn_cpu(struct mk_instance *instance, int cpu, struct mk_spawn_context *ctx); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); @@ -218,6 +220,11 @@ int multikernel_wakeup_secondary_cpu_64(u32 apicid, unsigned long start_eip, int multikernel_restore_ap(unsigned int cpu, unsigned long cr3, unsigned long gs_base, unsigned long stack, unsigned long entry); +#if defined(CONFIG_MULTIKERNEL) && defined(CONFIG_PCI) +void __init x86_multikernel_pci_platform_init(void); +#else +static inline void x86_multikernel_pci_platform_init(void) { } +#endif /* NMI on an offline pool CPU: honor a pending force halt */ #ifdef CONFIG_MULTIKERNEL diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..7099f7cd167dce 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,7 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI 0x0100 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 1002fbcf2f5d0b..1341501952d81b 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -554,6 +554,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; @@ -747,10 +752,29 @@ static void *bzImage64_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { +#ifdef CONFIG_MULTIKERNEL image->arch.mk_boot_params = bootparam_load_addr; + + /* + * setup_boot_parameters() copies the host subarchitecture. A + * spawn kernel must take the multikernel platform path instead. + */ + params->hdr.hardware_subarch = X86_SUBARCH_MULTIKERNEL; + + /* + * The spawn trampoline enters the compressed kernel directly, + * bypassing purgatory. The x86 boot protocol's 64-bit entry is + * 0x200 bytes from the start of the protected-mode payload. + */ + image->arch.mk_kernel_entry = kernel_load_addr + 0x200; + ret = mk_e820_fill(image->mk_instance, params); if (ret) goto out_free_params; +#else + ret = -EOPNOTSUPP; + goto out_free_params; +#endif } /* Allocate loader specific data */ diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index a7d31a722f33b7..942dce301108a2 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -62,12 +62,17 @@ struct elf_kernel_info { /* * Find multikernel entry point from PT_NOTE section. - * Looks for note with name "Linux" and type 0x4d4b ('MK'). + * The note type carries the generation; the descriptor remains one u64. */ -static unsigned long find_multikernel_entry_note(const void *buf, size_t len, - const Elf64_Ehdr *ehdr) +#define MK_VMLINUX_LEGACY_NOTE_TYPE 0x4d4b +#define MK_VMLINUX_PREVIOUS_NOTE_TYPE 0x4d4b0002 + +static int find_multikernel_entry_note(const void *buf, size_t len, + const Elf64_Ehdr *ehdr, + unsigned long *entry) { const Elf64_Phdr *phdrs = buf + ehdr->e_phoff; + bool legacy = false; int i; for (i = 0; i < ehdr->e_phnum; i++) { @@ -91,25 +96,54 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (ptr + note_size > end) break; - if (nhdr->n_type == 0x4d4b && - nhdr->n_namesz == 6 && + if (nhdr->n_namesz == 6 && nhdr->n_descsz == sizeof(u64) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const u64 *note_entry; + + note_entry = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + if (nhdr->n_type == MK_VMLINUX_NOTE_TYPE) { + *entry = *note_entry; + return 0; + } + if (nhdr->n_type == MK_VMLINUX_LEGACY_NOTE_TYPE || + nhdr->n_type == MK_VMLINUX_PREVIOUS_NOTE_TYPE) + legacy = true; } ptr += note_size; } } - return 0; + return legacy ? -EPROTONOSUPPORT : -ENOENT; +} + +static bool multikernel_entry_is_file_backed(const Elf64_Ehdr *ehdr, + unsigned long entry) +{ + const Elf64_Phdr *phdr = (const void *)ehdr + ehdr->e_phoff; + int i; + + if (!entry) + return false; + + for (i = 0; i < ehdr->e_phnum; i++) { + if (phdr[i].p_type != PT_LOAD || !phdr[i].p_filesz) + continue; + if (entry < phdr[i].p_paddr) + continue; + if (entry - phdr[i].p_paddr < phdr[i].p_filesz) + return true; + } + + return false; } /* * Parse ELF kernel and extract key information */ -static int kexec_parse_elf_kernel(const void *kernel_buf, unsigned long kernel_len, +static int kexec_parse_elf_kernel(const void *kernel_buf, + unsigned long kernel_len, + bool multikernel, struct elf_kernel_info *info) { const Elf64_Ehdr *ehdr; @@ -159,13 +193,31 @@ static int kexec_parse_elf_kernel(const void *kernel_buf, unsigned long kernel_l * PT_NOTE contains physical offset from load base, not virtual address. * This is the canonical way and survives symbol stripping. */ - info->multikernel_entry = find_multikernel_entry_note(kernel_buf, kernel_len, ehdr); - if (!info->multikernel_entry) { - pr_err("multikernel_startup_64 entry offset not found in PT_NOTE\n"); - return -ENOEXEC; - } + info->multikernel_entry = 0; + if (multikernel) { + int ret; + + ret = find_multikernel_entry_note(kernel_buf, kernel_len, ehdr, + &info->multikernel_entry); + if (ret == -EPROTONOSUPPORT) + pr_err("legacy vmlinux note type 0x%x is incompatible; expected 0x%x\n", + MK_VMLINUX_LEGACY_NOTE_TYPE, MK_VMLINUX_NOTE_TYPE); + else if (ret) + pr_err("multikernel ABI note type 0x%x not found\n", + MK_VMLINUX_NOTE_TYPE); + if (ret) + return ret == -ENOENT ? -ENOEXEC : ret; + + if (!multikernel_entry_is_file_backed(ehdr, + info->multikernel_entry)) { + pr_err("multikernel entry 0x%lx is outside file-backed PT_LOAD segments\n", + info->multikernel_entry); + return -ENOEXEC; + } - pr_info("Multikernel entry offset: 0x%lx\n", info->multikernel_entry); + pr_info("Multikernel entry offset: 0x%lx\n", + info->multikernel_entry); + } /* Find lowest load address and calculate total memory needed */ phdr = (const Elf64_Phdr *)(kernel_buf + ehdr->e_phoff); @@ -326,12 +378,13 @@ static void *vmlinux_load(struct kimage *image, char *kernel, .top_down = true }; struct kexec_buf pbuf = { .image = image, .buf_min = MIN_PURGATORY_ADDR, .buf_max = ULONG_MAX, .top_down = true }; + bool multikernel = image->type == KEXEC_TYPE_MULTIKERNEL; int ret; pr_info("Loading ELF vmlinux (type=%d)\n", image->type); /* Parse ELF headers */ - ret = kexec_parse_elf_kernel(kernel, kernel_len, &elf_info); + ret = kexec_parse_elf_kernel(kernel, kernel_len, multikernel, &elf_info); if (ret) { pr_err("Failed to parse ELF kernel: %d\n", ret); return ERR_PTR(ret); @@ -531,12 +584,17 @@ static void *vmlinux_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { +#ifdef CONFIG_MULTIKERNEL ret = mk_e820_fill(image->mk_instance, params); if (ret) { kvfree(ldata->kernel_buf); kfree(ldata); goto out_free_params; } +#else + ret = -EOPNOTSUPP; + goto out_free_params; +#endif } ldata->bootparams_buf = params; diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 5d4705d407b4c4..ca677416a1d805 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -47,9 +46,9 @@ static void __init multikernel_setup_calibration(void) { phys_addr_t ctx_phys = orig_boot_params - offsetof(struct mk_spawn_context, bp); - struct mk_spawn_context *ctx = __va(ctx_phys); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); - if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + if (!ctx || !ctx->boot_tsc_khz) return; multikernel_tsc_khz = ctx->boot_tsc_khz; @@ -171,6 +170,7 @@ void __init x86_early_init_platform_quirks(void) break; case X86_SUBARCH_MULTIKERNEL: multikernel_setup_calibration(); + x86_multikernel_pci_platform_init(); x86_platform.legacy.devices.pnpbios = 0; x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; x86_platform.legacy.rtc = 0; @@ -194,9 +194,6 @@ void __init x86_early_init_platform_quirks(void) x86_init.paging.pagetable_init = multikernel_pagetable_init; #endif x86_init.mpparse.parse_smp_cfg = multikernel_parse_smp_config; -#ifdef CONFIG_PCI - x86_init.pci.init = pci_multikernel_init; -#endif #endif x86_init.mpparse.early_parse_smp_cfg = x86_init_noop; /* diff --git a/arch/x86/multikernel/Makefile b/arch/x86/multikernel/Makefile index 5989cbbb691989..7238cf5d90f337 100644 --- a/arch/x86/multikernel/Makefile +++ b/arch/x86/multikernel/Makefile @@ -4,3 +4,4 @@ # obj-y += spawn.o direct_boot.o head_64.o e820.o +obj-$(CONFIG_PCI) += pci.o diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..54890d936e615e 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -269,7 +270,7 @@ SYM_CODE_END(multikernel_secondary_startup) .balign 4 .long 2f - 1f .long 4f - 3f - .long 0x4d4b + .long MK_VMLINUX_NOTE_TYPE 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c new file mode 100644 index 00000000000000..4fc89c56af1b6a --- /dev/null +++ b/arch/x86/multikernel/pci.c @@ -0,0 +1,977 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * x86 PCI support for multikernel spawn kernels. + * + * Spawn kernels discover only assigned BDFs and proxy all configuration + * accesses to the host kernel. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static bool mk_pci_roots_ready; +static atomic64_t mk_pci_request_id = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_count = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_total_ns = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_max_ns = ATOMIC64_INIT(0); +static DEFINE_RATELIMIT_STATE(mk_pci_cfg_error_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); +#define MK_PCI_RESET_TIMEOUT_MS 70000 +#ifdef CONFIG_PCI_MSI +static void mk_pci_forward_irq_set_mask(struct irq_data *data, bool masked); +static void mk_pci_irq_mailbox_requeue(struct mk_irq_mailbox *mailbox, + unsigned int slot); + +static void mk_pci_forward_irq_noop(struct irq_data *data) +{ +} + +static void mk_pci_forward_irq_mask(struct irq_data *data) +{ + mk_pci_forward_irq_set_mask(data, true); +} + +static void mk_pci_forward_irq_unmask(struct irq_data *data) +{ + mk_pci_forward_irq_set_mask(data, false); +} + +static void mk_pci_forward_irq_write_msg(struct irq_data *data, + struct msi_msg *msg) +{ +} + +static struct irq_chip mk_pci_forward_irq_chip = { + .name = "multikernel-pci-forward", + .irq_ack = mk_pci_forward_irq_noop, + /* Host process-context lifecycle owns physical mask state. */ + .irq_mask = mk_pci_forward_irq_mask, + .irq_unmask = mk_pci_forward_irq_unmask, + .irq_write_msi_msg = mk_pci_forward_irq_write_msg, +}; + +static void mk_pci_bind_local_irqs(unsigned int irq, unsigned int count) +{ + unsigned int i; + + for (i = 0; i < count; i++) + irq_set_chip_and_handler(irq + i, &mk_pci_forward_irq_chip, + handle_edge_irq); +} + +static bool mk_pci_forward_irq_matches(u32 irq_number, u32 vector, + u32 device_id, + struct irq_data **irq_data) +{ + struct irq_data *data = irq_get_irq_data(irq_number); + struct msi_desc *desc; + struct pci_dev *dev; + unsigned int offset; + + if (!data) + return false; + desc = irq_data_get_msi_desc(data); + if (!desc || vector < desc->msi_index) + return false; + + dev = msi_desc_to_pci_dev(desc); + offset = vector - desc->msi_index; + if (offset >= desc->nvec_used || desc->irq + offset != irq_number || + pci_domain_nr(dev->bus) != MK_PCI_IRQ_ID_DOMAIN(device_id) || + dev->bus->number != MK_PCI_IRQ_ID_BUS(device_id) || + dev->devfn != MK_PCI_IRQ_ID_DEVFN(device_id)) + return false; + + *irq_data = data; + return true; +} + +static void mk_pci_forward_irq_set_mask(struct irq_data *data, bool masked) +{ + struct msi_desc *desc = irq_data_get_msi_desc(data); + struct mk_irq_mailbox *mailbox; + struct pci_dev *dev; + u32 device_id; + u32 generation; + u32 vector; + unsigned int slot; + u64 epoch; + + if (!desc || !mk_self || !mk_self->ipi_data) + return; + dev = msi_desc_to_pci_dev(desc); + generation = READ_ONCE(dev->multikernel_msi_generation); + epoch = READ_ONCE(mk_self->ipi_data->spawn_epoch); + device_id = MK_PCI_IRQ_ID(pci_domain_nr(dev->bus), dev->bus->number, + dev->devfn); + vector = desc->msi_index + data->irq - desc->irq; + mailbox = &mk_self->ipi_data->irq_mailbox; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry = &mailbox->entries[slot]; + u64 old, new; + u32 slot_generation; + + old = atomic64_read_acquire(&entry->pending_generation); + if (!mk_irq_mailbox_generation(old) || + READ_ONCE(entry->lifecycle_epoch) != epoch || + READ_ONCE(entry->lifecycle_generation) != generation || + READ_ONCE(entry->device_id) != device_id || + READ_ONCE(entry->local_irq) != data->irq || + READ_ONCE(entry->vector) != vector) + continue; + slot_generation = mk_irq_mailbox_generation(old); + for (;;) { + new = masked ? old | MK_IRQ_MAILBOX_MASKED : + old & ~MK_IRQ_MAILBOX_MASKED; + if (new == old || + atomic64_cmpxchg(&entry->pending_generation, + old, new) == old) + break; + old = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(old) != slot_generation) + return; + } + if (!masked && mk_irq_mailbox_pending(new)) { + mk_phys_cpu_t target; + + mk_pci_irq_mailbox_requeue(mailbox, slot); + target = arch_cpu_physical_id(smp_processor_id()); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + return; + } +} + +static void mk_pci_irq_mailbox_requeue(struct mk_irq_mailbox *mailbox, + unsigned int slot) +{ + atomic64_or(BIT_ULL(slot & 63), + &mailbox->pending_bitmap[slot / 64]); +} + +static void mk_pci_irq_mailbox_drain_slot(struct mk_shared_data *shared, + unsigned int slot) +{ + struct mk_irq_mailbox *mailbox = &shared->irq_mailbox; + struct mk_irq_mailbox_entry *entry = &mailbox->entries[slot]; + struct irq_data *irq_data; + struct pci_dev *dev; + u64 lifecycle_epoch; + u64 token, base, claim; + u32 lifecycle_generation; + u32 pending; + u32 device_id; + u32 local_irq; + u16 vector; + + token = atomic64_read_acquire(&entry->pending_generation); + pending = mk_irq_mailbox_pending(token); + if (!mk_irq_mailbox_generation(token) || !pending) + return; + if (mk_irq_mailbox_consuming(token)) { + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + lifecycle_epoch = READ_ONCE(entry->lifecycle_epoch); + lifecycle_generation = READ_ONCE(entry->lifecycle_generation); + device_id = READ_ONCE(entry->device_id); + local_irq = READ_ONCE(entry->local_irq); + vector = READ_ONCE(entry->vector); + base = token & ~MK_IRQ_MAILBOX_PENDING_MASK; + + if (!local_irq || !mk_self || !mk_self->ipi_data || + lifecycle_epoch != READ_ONCE(shared->spawn_epoch)) + goto stale; + if (mk_irq_mailbox_masked(token)) { + atomic_inc(&mailbox->masked_deferred); + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + claim = base | MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_acquire(&entry->pending_generation, + token, claim) != token) { + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim)) + goto claimed_stale; + if (mk_irq_mailbox_masked(token)) { + atomic_inc(&mailbox->masked_deferred); + goto claimed_defer; + } + if (!mk_pci_forward_irq_matches(local_irq, vector, device_id, + &irq_data)) + goto claimed_stale; + dev = msi_desc_to_pci_dev(irq_data_get_msi_desc(irq_data)); + if (lifecycle_epoch != READ_ONCE(entry->lifecycle_epoch) || + lifecycle_generation != + READ_ONCE(entry->lifecycle_generation) || + device_id != READ_ONCE(entry->device_id) || + local_irq != READ_ONCE(entry->local_irq) || + vector != READ_ONCE(entry->vector) || + READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE || + lifecycle_generation != + READ_ONCE(dev->multikernel_msi_generation) || + irq_data_get_irq_chip(irq_data) != &mk_pci_forward_irq_chip) + goto claimed_stale; + if (pending > 1) + atomic_add(pending - 1, &mailbox->coalesced); + if (generic_handle_irq_safe(local_irq)) + atomic_inc(&mailbox->dispatch_failed); + + for (;;) { + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new = token & ~MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + if (mk_irq_mailbox_pending(new)) + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + +claimed_defer: + for (;;) { + u32 new_pending; + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new_pending = mk_irq_mailbox_pending(token); + if (new_pending < MK_IRQ_MAILBOX_PENDING_MASK) + new_pending++; + new = mk_irq_mailbox_token(mk_irq_mailbox_generation(token), + new_pending) | + (token & MK_IRQ_MAILBOX_MASKED); + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + mk_pci_irq_mailbox_requeue(mailbox, slot); + if (!mk_irq_mailbox_masked(new)) { + mk_phys_cpu_t target = + arch_cpu_physical_id(smp_processor_id()); + + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + return; + } + +claimed_stale: + atomic_inc(&mailbox->stale); + for (;;) { + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new = token & ~MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + if (mk_irq_mailbox_pending(new)) + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + return; + +stale: + if (atomic64_cmpxchg_acquire(&entry->pending_generation, + token, base) != token) + mk_pci_irq_mailbox_requeue(mailbox, slot); + atomic_inc(&mailbox->stale); +} + +void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) +{ + unsigned int word; + + lockdep_assert_in_irq(); + if (!shared) + return; + for (word = 0; word < MK_IRQ_MAILBOX_WORDS; word++) { + atomic64_t *pending_bitmap; + unsigned long bits; + + pending_bitmap = &shared->irq_mailbox.pending_bitmap[word]; + bits = atomic64_xchg_acquire(pending_bitmap, 0); + + while (bits) { + unsigned int bit = __ffs(bits); + + bits &= bits - 1; + mk_pci_irq_mailbox_drain_slot(shared, word * 64 + bit); + } + } +} + +static int mk_pci_send_irq_request(struct mk_pci_irq_request *request) +{ + struct mk_reply_handle reply; + s32 status; + int ret; + + if (WARN_ON_ONCE(irqs_disabled() || !in_task())) + return -EWOULDBLOCK; + might_sleep(); + if (!request->lifecycle_generation || !mk_self || + !mk_self->ipi_data || !host_instance) + return -EINVAL; + request->lifecycle_epoch = + READ_ONCE(mk_self->ipi_data->spawn_epoch); + if (!request->lifecycle_epoch) + return -EPROTO; + + request->request_id = atomic64_inc_return(&mk_pci_request_id); + request->sender_instance_id = mk_self ? mk_self->id : -1; + ret = mk_reply_reserve(mk_self->ipi_data, MK_REPLY_PCI_IRQ, + request->request_id, &reply); + if (ret) + return ret; + request->reply_slot = reply.slot; + request->reply_generation = reply.generation; + + ret = mk_send_message_to_instance(host_instance, MK_MSG_PCI, + MK_PCI_IRQ_REQUEST, request, + sizeof(*request)); + if (ret) { + mk_reply_release(mk_self->ipi_data, &reply); + return ret; + } + + ret = mk_reply_wait(mk_self->ipi_data, &reply, 1000, + &status, NULL); + return ret ? ret : status; +} + +bool mk_pci_msi_controlled(struct pci_dev *dev) +{ + return mk_pci_controlled(dev); +} + +static int mk_pci_msi_teardown_generation(struct pci_dev *dev, + u32 generation); + +int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) +{ + u32 generation; + u8 state; + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_SETUP, + .nr_vectors = nvec, + .msix = type == PCI_CAP_ID_MSIX, + }; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + state = READ_ONCE(dev->multikernel_msi_state); + if (state != MK_PCI_MSI_IDLE && state != MK_PCI_MSI_FAILED) + return -EBUSY; + generation = READ_ONCE(dev->multikernel_msi_generation) + 1; + if (!generation) + generation = 1; + WRITE_ONCE(dev->multikernel_msi_generation, generation); + request.lifecycle_generation = generation; + ret = mk_pci_send_irq_request(&request); + if (ret) { + if (ret == -EINPROGRESS) { + int cleanup_ret; + + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : + MK_PCI_MSI_IDLE); + } + return ret; + } + WRITE_ONCE(dev->multikernel_msi_nvec, nvec); + WRITE_ONCE(dev->multikernel_msi_msix, type == PCI_CAP_ID_MSIX); + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_PREPARED); + return 0; +} + +static int mk_pci_msi_bind(struct pci_dev *dev, unsigned int index, + unsigned int irq, unsigned int nvec, bool msix, + u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_BIND, + .vector = index, + .nr_vectors = nvec, + .msix = msix, + .local_irq = irq, + .lifecycle_generation = generation, + }; + unsigned int count = msix ? 1 : nvec; + unsigned int i; + int ret; + + /* The local descriptor must be dispatchable before the host unmasks. */ + mk_pci_bind_local_irqs(irq, count); + ret = mk_pci_send_irq_request(&request); + if (ret) + return ret; + /* Publish the irqdesc's initial logical mask state into the token. */ + for (i = 0; i < count; i++) { + struct irq_data *data = irq_get_irq_data(irq + i); + + if (!data) + return -EINVAL; + mk_pci_forward_irq_set_mask(data, + irqd_irq_disabled(data) || + irqd_irq_masked(data)); + } + return 0; +} + +static int mk_pci_msi_teardown_generation(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_TEARDOWN, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_commit(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_COMMIT, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_host_activate(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_ACTIVATE, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_restore_begin(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_RESTORE_BEGIN, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_bind_all(struct pci_dev *dev, u32 generation) +{ + struct msi_desc *desc; + unsigned int expected = READ_ONCE(dev->multikernel_msi_nvec); + unsigned int next = 0; + bool msix = READ_ONCE(dev->multikernel_msi_msix); + int ret; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = msix ? 1 : desc->nvec_used; + + if (next >= expected || desc->msi_index != next || !count || + count > expected - next) + return -EINVAL; + ret = mk_pci_msi_bind(dev, desc->msi_index, desc->irq, + desc->nvec_used, msix, generation); + if (ret) + return ret; + next += count; + } + if (next != expected) + return -EINVAL; + return mk_pci_msi_commit(dev, generation); +} + +static int mk_pci_msi_mask_mailbox(struct pci_dev *dev) +{ + struct msi_desc *desc; + unsigned long deadline; + u64 epoch; + u32 device_id; + u32 generation; + unsigned int slot; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = desc->pci.msi_attrib.is_msix ? + 1 : desc->nvec_used; + unsigned int i; + + for (i = 0; i < count; i++) { + struct irq_data *data = irq_get_irq_data(desc->irq + i); + + if (data) + mk_pci_forward_irq_set_mask(data, true); + } + } + + epoch = READ_ONCE(mk_self->ipi_data->spawn_epoch); + generation = READ_ONCE(dev->multikernel_msi_generation); + device_id = MK_PCI_IRQ_ID(pci_domain_nr(dev->bus), dev->bus->number, + dev->devfn); + deadline = jiffies + msecs_to_jiffies(1000); + for (;;) { + bool consuming = false; + + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry = + &mk_self->ipi_data->irq_mailbox.entries[slot]; + u64 token; + + token = atomic64_read_acquire(&entry->pending_generation); + + if (mk_irq_mailbox_generation(token) && + mk_irq_mailbox_consuming(token) && + READ_ONCE(entry->lifecycle_epoch) == epoch && + READ_ONCE(entry->lifecycle_generation) == generation && + READ_ONCE(entry->device_id) == device_id) { + consuming = true; + break; + } + } + if (!consuming) + break; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(50, 100); + } + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = desc->pci.msi_attrib.is_msix ? + 1 : desc->nvec_used; + unsigned int i; + + for (i = 0; i < count; i++) + synchronize_irq(desc->irq + i); + } + return 0; +} + +int mk_pci_msi_activate(struct pci_dev *dev) +{ + u32 generation; + int cleanup_ret; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_PREPARED) + return -EIO; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_bind_all(dev, generation); + if (ret) { + pr_err("Failed to activate host-owned MSI vectors for %s: %d\n", + pci_name(dev), ret); + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; + } + + /* The guest must be able to consume the first edge before host unmask. */ + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_ACTIVE); + ret = mk_pci_msi_host_activate(dev, generation); + if (ret) { + cleanup_ret = mk_pci_msi_mask_mailbox(dev); + if (!cleanup_ret) + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; + } + return 0; +} + +int mk_pci_msi_restore(struct pci_dev *dev) +{ + u32 generation; + int cleanup_ret; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE) + return -EIO; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_mask_mailbox(dev); + if (ret) { + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_FAILED); + return ret; + } + ret = mk_pci_msi_restore_begin(dev, generation); + if (!ret) + ret = mk_pci_msi_bind_all(dev, generation); + if (!ret) + ret = mk_pci_msi_host_activate(dev, generation); + if (ret) { + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_FAILED); + if (cleanup_ret) + pr_err("Failed to quiesce host-owned MSI after restore failure for %s: %d\n", + pci_name(dev), cleanup_ret); + return ret; + } + return 0; +} + +int mk_pci_msi_teardown(struct pci_dev *dev) +{ + u32 generation; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) == MK_PCI_MSI_IDLE) + return 0; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_mask_mailbox(dev); + if (!ret) + ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; +} +#endif /* CONFIG_PCI_MSI */ + +bool mk_pci_controlled(struct pci_dev *dev) +{ + return mk_self && mk_self->id != 0 && + mk_pci_get_assigned_identity_bdf(pci_domain_nr(dev->bus), + dev->bus->number, dev->devfn, + NULL, NULL); +} + +int mk_pci_reset_flr(struct pci_dev *dev) +{ + struct mk_pci_reset_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + }; + struct mk_reply_handle reply; + u32 generation; + s32 status; + int ret; + + if (WARN_ON_ONCE(irqs_disabled() || !in_task())) + return -EWOULDBLOCK; + might_sleep(); + if (!mk_self || !mk_self->ipi_data || !host_instance) + return -ENODEV; + request.lifecycle_epoch = + READ_ONCE(mk_self->ipi_data->spawn_epoch); + if (!request.lifecycle_epoch) + return -EPROTO; + + generation = READ_ONCE(dev->multikernel_reset_generation) + 1; + if (!generation) + generation = 1; + WRITE_ONCE(dev->multikernel_reset_generation, generation); + request.reset_generation = generation; + request.request_id = atomic64_inc_return(&mk_pci_request_id); + request.sender_instance_id = mk_self->id; + ret = mk_reply_reserve(mk_self->ipi_data, MK_REPLY_PCI_RESET, + request.request_id, &reply); + if (ret) + return ret; + request.reply_slot = reply.slot; + request.reply_generation = reply.generation; + + ret = mk_send_message_to_instance(host_instance, MK_MSG_PCI, + MK_PCI_RESET_REQUEST, &request, + sizeof(request)); + if (ret) { + mk_reply_release(mk_self->ipi_data, &reply); + return ret; + } + + ret = mk_reply_wait(mk_self->ipi_data, &reply, + MK_PCI_RESET_TIMEOUT_MS, + &status, NULL); + return ret ? ret : status; +} + +static void mk_pci_record_latency(u64 start) +{ + u64 elapsed = ktime_get_mono_fast_ns() - start; + u64 old_max = atomic64_read(&mk_pci_cfg_max_ns); + + atomic64_inc(&mk_pci_cfg_count); + atomic64_add(elapsed, &mk_pci_cfg_total_ns); + while (elapsed > old_max) { + u64 previous = atomic64_cmpxchg(&mk_pci_cfg_max_ns, old_max, + elapsed); + + if (previous == old_max) + break; + old_max = previous; + } +} + +static int mk_pci_remote_config(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + bool write, u32 *value) +{ + struct mk_pci_cfg_request request = { + .request_id = atomic64_inc_return(&mk_pci_request_id), + .lifecycle_epoch = mk_self && mk_self->ipi_data ? + READ_ONCE(mk_self->ipi_data->spawn_epoch) : 0, + .sender_instance_id = mk_self ? mk_self->id : -1, + .domain = domain, + .bus = bus, + .devfn = devfn, + .reg = where, + .len = size, + .write = write, + .value = *value, + }; + struct mk_reply_handle reply; + s32 status; + u32 response_value; + u64 start = ktime_get_mono_fast_ns(); + int ret; + + if (!mk_self || !mk_self->ipi_data || !host_instance) { + ret = -ENODEV; + goto out_error; + } + ret = mk_reply_reserve(mk_self->ipi_data, MK_REPLY_PCI_CFG, + request.request_id, &reply); + if (ret) + goto out_error; + request.reply_slot = reply.slot; + request.reply_generation = reply.generation; + + ret = mk_send_message_to_instance(host_instance, MK_MSG_PCI, + MK_PCI_CFG_REQUEST, &request, + sizeof(request)); + if (ret) { + mk_reply_release(mk_self->ipi_data, &reply); + goto out_error; + } + ret = mk_reply_wait_atomic(mk_self->ipi_data, &reply, 20000, + &status, &response_value); + if (ret) + goto out_error; + if (status < 0) { + ret = status; + goto out_error; + } + if (!write) + *value = response_value; + mk_pci_record_latency(start); + return status; + +out_error: + if (ret < 0 && __ratelimit(&mk_pci_cfg_error_rs)) + printk_deferred(KERN_ERR + "Multikernel PCI config request timed out or failed to send: %d\n", + ret); + return PCIBIOS_SET_FAILED; +} + +static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, + u32 *value) +{ + u32 identity; + u32 mask; + + if (where < PCI_VENDOR_ID || where + size > PCI_COMMAND) + return false; + + identity = vendor | (u32)device << 16; + mask = size == sizeof(identity) ? ~0U : (1U << (size * 8)) - 1; + *value = (identity >> (where * 8)) & mask; + return true; +} + +static int mk_pci_raw_read(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + u32 *value) +{ + u16 vendor, device; + + if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, &vendor, + &device)) { + *value = ~0U; + return PCIBIOS_DEVICE_NOT_FOUND; + } + if (mk_pci_identity_read(vendor, device, where, size, value)) + return PCIBIOS_SUCCESSFUL; + + return mk_pci_remote_config(domain, bus, devfn, where, size, false, + value); +} + +static int mk_pci_raw_write(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + u32 value) +{ + if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, NULL, NULL)) + return PCIBIOS_DEVICE_NOT_FOUND; + + return mk_pci_remote_config(domain, bus, devfn, where, size, true, + &value); +} + +static const struct pci_raw_ops mk_pci_filtered_raw_ops = { + .read = mk_pci_raw_read, + .write = mk_pci_raw_write, +}; + +static int __init x86_multikernel_pci_arch_init(void) +{ + if (!mk_self || !host_instance || !mk_self->pci_devices_valid) + return 0; + + raw_pci_ops = &mk_pci_filtered_raw_ops; + raw_pci_ext_ops = &mk_pci_filtered_raw_ops; + mk_pci_roots_ready = true; + pr_notice("Multikernel selected host-mediated PCI config access\n"); + + return 0; +} + +static struct pci_bus * __init mk_pci_get_root(u16 domain, u8 bus_number) +{ + struct resource_entry *window; + struct pci_sysdata *sd; + struct pci_bus *bus; + bool has_busn_res = false; + LIST_HEAD(resources); + + if (domain && !pci_domains_supported) { + pr_err("Multikernel cannot scan PCI root %04x:%02x without domain support\n", + domain, bus_number); + return ERR_PTR(-EOPNOTSUPP); + } + bus = pci_find_bus(domain, bus_number); + if (bus) + return bus; + + sd = kzalloc_obj(*sd, GFP_KERNEL); + if (!sd) + return ERR_PTR(-ENOMEM); + sd->domain = domain; + sd->node = x86_pci_root_bus_node(bus_number); + x86_pci_root_bus_resources(bus_number, &resources); + resource_list_for_each_entry(window, &resources) { + if (window->res->flags & IORESOURCE_BUS) { + has_busn_res = true; + break; + } + } + bus = pci_create_root_bus(NULL, bus_number, &pci_root_ops, sd, + &resources); + if (!bus) { + pci_free_resource_list(&resources); + kfree(sd); + return ERR_PTR(-ENOMEM); + } + if (!has_busn_res) { + if (!pci_bus_insert_busn_res(bus, bus_number, bus_number)) { + pci_remove_root_bus(bus); + kfree(sd); + return ERR_PTR(-EBUSY); + } + } + pr_notice("Multikernel created synthetic PCI root %04x:%02x\n", + domain, bus_number); + return bus; +} + +static int __init x86_multikernel_pci_init(void) +{ + const struct mk_pci_device *device; + struct pci_bus *bus; + struct pci_dev *pdev; + + if (!mk_self) + panic("Multikernel lost restored instance metadata"); + if (!mk_self->pci_device_count) + return 0; + if (!mk_self->pci_devices_valid || !mk_pci_roots_ready) + panic("Multikernel assigned PCI inventory is unavailable"); + + list_for_each_entry(device, &mk_self->pci_devices, list) { + bus = mk_pci_get_root(device->domain, device->bus); + if (IS_ERR(bus)) + panic("Multikernel failed to create synthetic PCI root %04x:%02x: %ld", + device->domain, device->bus, PTR_ERR(bus)); + pdev = pci_scan_single_device(bus, + PCI_DEVFN(device->slot, device->func)); + if (!pdev) + panic("Multikernel failed to enumerate assigned PCI device %04x:%02x:%02x.%x", + device->domain, device->bus, device->slot, + device->func); + pci_bus_add_devices(bus); + } + if (atomic64_read(&mk_pci_cfg_count)) { + u64 count = atomic64_read(&mk_pci_cfg_count); + + pr_notice("Multikernel PCI control plane: %llu config requests, average %llu ns, max %llu ns\n", + count, atomic64_read(&mk_pci_cfg_total_ns) / count, + atomic64_read(&mk_pci_cfg_max_ns)); + } + + /* Suppress legacy bus 0 probing after every assigned root is present. */ + return 0; +} +void __init x86_multikernel_pci_platform_init(void) +{ + pci_probe = PCI_PROBE_NOEARLY; + x86_init.pci.arch_init = x86_multikernel_pci_arch_init; + x86_init.pci.init = x86_multikernel_pci_init; +} diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 2cdafcd0586903..61660d7d73b519 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -72,6 +72,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * This kernel's own context: where its CPUs park and wake. A spawn's @@ -86,6 +87,19 @@ static struct mk_spawn_context *mk_own_context(void) return mk_pool ? mk_pool->arch.slot : NULL; } +static void *spawn_trampoline_va; +static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} + extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; extern char mk_secondary_trampoline[]; @@ -455,6 +469,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, instance->arch.spawn_ctx->boot_tsc_khz = tsc_khz; instance->arch.spawn_ctx->boot_apic_hz = (unsigned long)lapic_timer_period * HZ; + instance->arch.spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->arch.spawn_ctx); } @@ -677,18 +692,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u32 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; + } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -705,14 +740,28 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; /* * A spawn kernel cannot calibrate against legacy timers because they * belong to the host. Reuse the selected physical CPU's delay and local @@ -748,19 +797,27 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. The copy matters: * the page arrives holding the HOST kernel's trampoline code, and when * the two binaries differ its offsets are wrong for this kernel. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_own_context(); unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -768,32 +825,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - memcpy(__va(ctx->trampoline_phys), multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)__va(ctx->trampoline_phys) & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. @@ -1627,7 +1700,7 @@ bool mk_pool_park_uses(phys_addr_t start, size_t size) } /** - * mk_pool_park_teardown() - Return the pool park area to the pool + * mk_arch_pool_park_teardown() - Return the pool park area to the pool * * The area is pool memory, so the chunk holding it cannot be removed * while it exists. Every parked CPU executes the park page and runs on @@ -1635,10 +1708,12 @@ bool mk_pool_park_uses(phys_addr_t start, size_t size) * context, so the area may only go back while no CPU is in the pool at * all. The next baseline builds it again, possibly elsewhere. * - * Returns 0 if there was nothing to tear down, -EBUSY if the pool still - * has CPUs. + * The generic caller holds the instance and CPU transaction locks and has + * verified that every pool CPU is back before entering this function. + * + * Returns 0 after releasing the park area or if none exists. */ -int mk_pool_park_teardown(void) +int mk_arch_pool_park_teardown(void) { if (!mk_pool) return 0; @@ -1648,9 +1723,6 @@ int mk_pool_park_teardown(void) if (!mk_pool->arch.slot) return 0; - if (!mk_pool_cpus_returned()) - return -EBUSY; - multikernel_free(mk_pool->arch.slot_phys, ALIGN(sizeof(struct mk_spawn_context), PAGE_SIZE)); mk_pool->arch.slot = NULL; diff --git a/drivers/pci/msi/api.c b/drivers/pci/msi/api.c index 818d55fbad0dfa..ca7195c58ef8e0 100644 --- a/drivers/pci/msi/api.c +++ b/drivers/pci/msi/api.c @@ -10,6 +10,7 @@ #include #include +#include #include "msi.h" @@ -123,6 +124,8 @@ bool pci_msix_can_alloc_dyn(struct pci_dev *dev) { if (!dev->msix_cap) return false; + if (mk_pci_msi_controlled(dev)) + return false; return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY); } @@ -388,6 +391,12 @@ EXPORT_SYMBOL(pci_free_irq_vectors); */ void pci_restore_msi_state(struct pci_dev *dev) { + if (mk_pci_msi_controlled(dev)) { + if (mk_pci_msi_restore(dev)) + pr_err("Failed to restore host-owned MSI vectors for %s\n", + pci_name(dev)); + return; + } __pci_restore_msi_state(dev); __pci_restore_msix_state(dev); } diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index 6e65f0f44112e6..e79bf05f07a325 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -4,6 +4,7 @@ */ #include #include +#include #include #include "msi.h" @@ -11,18 +12,38 @@ int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) { struct irq_domain *domain; + bool controlled = mk_pci_msi_controlled(dev); + int ret; + + if (controlled) { + ret = mk_pci_msi_prepare(dev, nvec, type); + if (ret) + return ret; + } domain = dev_get_msi_domain(&dev->dev); if (domain && irq_domain_is_hierarchy(domain)) - return msi_domain_alloc_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN, nvec); - - return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); + ret = msi_domain_alloc_irqs_all_locked(&dev->dev, + MSI_DEFAULT_DOMAIN, nvec); + else + ret = pci_msi_legacy_setup_msi_irqs(dev, nvec, type); + if (!ret && controlled) + ret = mk_pci_msi_activate(dev); + + if (ret && controlled && mk_pci_msi_teardown(dev)) + pr_err("Failed to roll back host-owned MSI setup for %s\n", + pci_name(dev)); + return ret; } void pci_msi_teardown_msi_irqs(struct pci_dev *dev) { struct irq_domain *domain; + if (mk_pci_msi_controlled(dev) && mk_pci_msi_teardown(dev)) + pr_err("Failed to tear down host-owned MSI vectors for %s\n", + pci_name(dev)); + domain = dev_get_msi_domain(&dev->dev); if (domain && irq_domain_is_hierarchy(domain)) { msi_domain_free_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN); diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index e2412175d7af20..6694a75e8ac92b 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "../pci.h" #include "msi.h" @@ -240,6 +241,14 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { struct pci_dev *dev = msi_desc_to_pci_dev(entry); + if (mk_pci_msi_controlled(dev)) { + /* Host programming is committed later from the setup context. */ + entry->msg = *msg; + if (entry->write_msi_msg) + entry->write_msi_msg(entry, entry->write_msi_msg_data); + return; + } + if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) { /* Don't touch the hardware now */ } else if (entry->pci.msi_attrib.is_msix) { @@ -714,6 +723,7 @@ static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, int nvec, struct irq_affinity *affd) { + bool controlled = mk_pci_msi_controlled(dev); int ret, tsize; u16 control; @@ -736,6 +746,15 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, ret = -ENOMEM; goto out_disable; } + if (controlled && + !pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { + /* + * The controlled setup path activates host-owned vectors from + * msix_setup_interrupts(). Clear stale table entries while the + * function-wide mask is still set, before host activation. + */ + msix_mask_all(dev->msix_base, tsize); + } ret = msix_setup_interrupts(dev, entries, nvec, affd); if (ret) @@ -744,7 +763,8 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, /* Disable INTX */ pci_intx_for_msi(dev, 0); - if (!pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { + if (!controlled && + !pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { /* * Ensure that all table entries are masked to prevent * stale entries from firing in a crash kernel. diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 8479c2e1f74f10..7a9f7a327971ef 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -4380,6 +4381,8 @@ int pcie_reset_flr(struct pci_dev *dev, bool probe) if (probe) return 0; + if (mk_pci_controlled(dev)) + return mk_pci_reset_flr(dev); return pcie_flr(dev); } diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 7661517ebdf0df..bccc7a4bdd7943 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -24,7 +24,6 @@ #include #include #include -#include #include "pci.h" static struct resource busn_resource = { @@ -2601,14 +2600,6 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) struct pci_dev *dev; u32 l; - /* - * For multikernel spawns, check if we should even probe this location - * BEFORE any config space access. This prevents hardware conflicts - * when the host kernel is also using PCI devices. - */ - if (IS_ENABLED(CONFIG_MULTIKERNEL) && !mk_pci_should_probe(bus, devfn)) - return NULL; - if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000)) return NULL; diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index f544bf342a09d0..59a6c998ecec94 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -27,6 +27,7 @@ #include #include #include +#include #define MKTTY_IPI_TYPE 0x4D4B5459U /* "MKTY" */ #define MKTTY_MAX_DATA (MK_MAX_DATA_SIZE - 16) @@ -54,16 +55,25 @@ struct mktty_host_conn { int instance_id; /* -1 = not connected */ struct list_head list; wait_queue_head_t wait; - spinlock_t rx_lock; + struct work_struct wake_work; + raw_spinlock_t rx_lock; char *rx_buf; int rx_head; int rx_tail; }; static LIST_HEAD(mktty_host_conns); -static DEFINE_SPINLOCK(mktty_host_conns_lock); +static DEFINE_RAW_SPINLOCK(mktty_host_conns_lock); static struct mk_ipi_handler *mktty_host_handler; +static void mktty_host_wake_workfn(struct work_struct *work) +{ + struct mktty_host_conn *conn = + container_of(work, struct mktty_host_conn, wake_work); + + wake_up_interruptible(&conn->wait); +} + static struct mktty_host_conn *mktty_find_conn(int instance_id) { struct mktty_host_conn *conn; @@ -78,6 +88,7 @@ static struct mktty_host_conn *mktty_find_conn(int instance_id) static int mktty_host_open(struct inode *inode, struct file *filp) { struct mktty_host_conn *conn; + unsigned long flags; conn = kzalloc(sizeof(*conn), GFP_KERNEL); if (!conn) @@ -91,13 +102,14 @@ static int mktty_host_open(struct inode *inode, struct file *filp) conn->instance_id = -1; init_waitqueue_head(&conn->wait); - spin_lock_init(&conn->rx_lock); + INIT_WORK(&conn->wake_work, mktty_host_wake_workfn); + raw_spin_lock_init(&conn->rx_lock); conn->rx_head = 0; conn->rx_tail = 0; - spin_lock(&mktty_host_conns_lock); + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); list_add(&conn->list, &mktty_host_conns); - spin_unlock(&mktty_host_conns_lock); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); filp->private_data = conn; return 0; @@ -106,11 +118,13 @@ static int mktty_host_open(struct inode *inode, struct file *filp) static int mktty_host_release(struct inode *inode, struct file *filp) { struct mktty_host_conn *conn = filp->private_data; + unsigned long flags; - spin_lock(&mktty_host_conns_lock); + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); list_del(&conn->list); - spin_unlock(&mktty_host_conns_lock); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); + cancel_work_sync(&conn->wake_work); kfree(conn->rx_buf); kfree(conn); return 0; @@ -128,22 +142,22 @@ static ssize_t mktty_host_read(struct file *filp, char __user *buf, return -ENOTCONN; while (copied == 0) { - spin_lock_irqsave(&conn->rx_lock, flags); + raw_spin_lock_irqsave(&conn->rx_lock, flags); while (copied < count && conn->rx_head != conn->rx_tail) { char c = conn->rx_buf[conn->rx_tail]; conn->rx_tail = (conn->rx_tail + 1) % MKTTY_RX_BUF_SIZE; - spin_unlock_irqrestore(&conn->rx_lock, flags); + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); if (put_user(c, buf + copied)) { return copied > 0 ? copied : -EFAULT; } copied++; - spin_lock_irqsave(&conn->rx_lock, flags); + raw_spin_lock_irqsave(&conn->rx_lock, flags); } - spin_unlock_irqrestore(&conn->rx_lock, flags); + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); if (copied > 0) break; @@ -240,10 +254,10 @@ static __poll_t mktty_host_poll(struct file *filp, poll_table *wait) poll_wait(filp, &conn->wait, wait); - spin_lock_irqsave(&conn->rx_lock, flags); + raw_spin_lock_irqsave(&conn->rx_lock, flags); if (conn->rx_head != conn->rx_tail) mask |= EPOLLIN | EPOLLRDNORM; - spin_unlock_irqrestore(&conn->rx_lock, flags); + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); mask |= EPOLLOUT | EPOLLWRNORM; return mask; @@ -278,10 +292,10 @@ static void mktty_host_ipi_handler(struct mk_ipi_data *data, void *ctx) if (msg->len > MKTTY_MAX_DATA) return; - spin_lock_irqsave(&mktty_host_conns_lock, flags); + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); conn = mktty_find_conn(msg->console_id); if (conn) { - spin_lock(&conn->rx_lock); + raw_spin_lock(&conn->rx_lock); for (i = 0; i < msg->len; i++) { space = (conn->rx_tail - conn->rx_head - 1 + MKTTY_RX_BUF_SIZE) % MKTTY_RX_BUF_SIZE; if (space == 0) @@ -289,10 +303,10 @@ static void mktty_host_ipi_handler(struct mk_ipi_data *data, void *ctx) conn->rx_buf[conn->rx_head] = msg->data[i]; conn->rx_head = (conn->rx_head + 1) % MKTTY_RX_BUF_SIZE; } - spin_unlock(&conn->rx_lock); - wake_up_interruptible(&conn->wait); + raw_spin_unlock(&conn->rx_lock); + schedule_work(&conn->wake_work); } - spin_unlock_irqrestore(&mktty_host_conns_lock, flags); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); } static int mktty_host_init(void) @@ -337,6 +351,17 @@ static struct tty_driver *mktty_spawn_driver; static struct mktty_spawn_state mktty_spawn; static struct mk_ipi_handler *mktty_spawn_handler; static struct console mktty_spawn_console; + +static int mktty_spawn_parent_id(void) +{ + if (!mk_self) + return -ENODEV; + if (mk_self->id == 0) + return 0; + if (!host_instance) + return -ENODEV; + return READ_ONCE(host_instance->id); +} static bool mktty_console_registered; static int mktty_spawn_activate(struct tty_port *port, struct tty_struct *tty) @@ -376,11 +401,14 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, size_t count) { struct mktty_message *msg; - size_t sent = 0, chunk; - int ret; + size_t sent = 0, chunk, msg_size; + int ret, parent_id; if (tty->index != 0) return -ENODEV; + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + return parent_id; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) @@ -393,10 +421,10 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, msg->len = chunk; msg->reserved = 0; memcpy(msg->data, buf + sent, chunk); + msg_size = sizeof(*msg) - MKTTY_MAX_DATA + chunk; - ret = multikernel_send_ipi_data(0, msg, - sizeof(*msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + ret = multikernel_send_ipi_data(parent_id, msg, msg_size, + MKTTY_IPI_TYPE); if (ret < 0) { kfree(msg); return sent > 0 ? sent : ret; @@ -445,12 +473,16 @@ static struct mktty_message mktty_console_msg; static DEFINE_SPINLOCK(mktty_console_lock); static void mktty_spawn_console_write(struct console *con, const char *s, - unsigned int count) + unsigned int count) { unsigned long flags; - size_t chunk; + size_t chunk, msg_size; + int parent_id; spin_lock_irqsave(&mktty_console_lock, flags); + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + goto out; while (count > 0) { chunk = min_t(size_t, count, MKTTY_MAX_DATA); mktty_console_msg.type = MKTTY_MSG_OUTPUT; @@ -458,13 +490,14 @@ static void mktty_spawn_console_write(struct console *con, const char *s, mktty_console_msg.len = chunk; mktty_console_msg.reserved = 0; memcpy(mktty_console_msg.data, s, chunk); + msg_size = sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk; - multikernel_send_ipi_data(0, &mktty_console_msg, - sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + (void)multikernel_send_ipi_data_to_host(&mktty_console_msg, + msg_size, MKTTY_IPI_TYPE); s += chunk; count -= chunk; } +out: spin_unlock_irqrestore(&mktty_console_lock, flags); } diff --git a/include/linux/kexec.h b/include/linux/kexec.h index cef2afc6d9eead..ac9a587eee8924 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -455,6 +455,7 @@ extern int kernel_kexec(void); #ifdef CONFIG_MULTIKERNEL extern int multikernel_kexec_by_id(int mk_id); extern struct kimage *kimage_find_by_id(int mk_id); +int kimage_prepare_multikernel_unload(struct kimage *image); #else static inline int multikernel_kexec_by_id(int mk_id) { @@ -464,6 +465,11 @@ static inline struct kimage *kimage_find_by_id(int mk_id) { return NULL; } + +static inline int kimage_prepare_multikernel_unload(struct kimage *image) +{ + return -EOPNOTSUPP; +} #endif extern struct page *kimage_alloc_control_pages(struct kimage *image, unsigned int order); diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index b470f10b4890d3..cb2abee1095ddf 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -10,12 +10,19 @@ #include #include #include +#include #include #include #include -#include #include #include +#include +#include +#include +#include + +struct pci_bus; +struct mk_instance; #ifdef CONFIG_MULTIKERNEL #include @@ -40,6 +47,7 @@ typedef u64 mk_phys_cpu_t; #define MK_PHYS_CPU_INVALID (~(mk_phys_cpu_t)0) struct mk_cpu_set { + raw_spinlock_t lock; unsigned int nr; /* Entries in use */ unsigned int cap; /* Allocated capacity */ mk_phys_cpu_t *ids; @@ -54,26 +62,18 @@ bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id); bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id); int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src); int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set); +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set); +bool mk_cpu_set_empty(const struct mk_cpu_set *set); +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set); +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id); -static inline unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) -{ - return set ? set->nr : 0; -} - -static inline bool mk_cpu_set_empty(const struct mk_cpu_set *set) -{ - return mk_cpu_set_count(set) == 0; -} - -static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) -{ - return mk_cpu_set_empty(set) ? MK_PHYS_CPU_INVALID : set->ids[0]; -} +void mk_cpu_transaction_lock(void); +void mk_cpu_transaction_unlock(void); #define mk_cpu_set_for_each(i, id, set) \ for ((i) = 0; \ - (set) && (i) < (set)->nr && \ - (((id) = (set)->ids[(i)]), true); \ + mk_cpu_set_get((set), (i), &(id)); \ (i)++) /** @@ -86,21 +86,120 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* IPI ring buffer size - must be power of 2 for efficient modulo */ #define MK_IPI_RING_SIZE 64 +#define MK_REPLY_SLOTS 16 +#define MK_REPLY_STATE_BITS 3 +#define MK_IRQ_MAILBOX_SLOTS 256 +#define MK_IRQ_MAILBOX_WORDS (MK_IRQ_MAILBOX_SLOTS / 64) +#define MK_IRQ_MAILBOX_SLOT_INVALID ((u32)~0U) +#define MK_IRQ_MAILBOX_PENDING_MASK 0x3fffffffULL +#define MK_IRQ_MAILBOX_CONSUMING 0x40000000ULL +#define MK_IRQ_MAILBOX_MASKED 0x80000000ULL +#define MK_IRQ_MAILBOX_GENERATION_SHIFT 32 + +enum mk_reply_state { + MK_REPLY_FREE = 0, + MK_REPLY_RESERVED, + MK_REPLY_WRITING, + MK_REPLY_EXECUTING, + MK_REPLY_COMMITTED, + MK_REPLY_READY, + MK_REPLY_ABANDONED, +}; + +enum mk_reply_kind { + MK_REPLY_PCI_CFG = 1, + MK_REPLY_PCI_IRQ, + MK_REPLY_PCI_RESET, +}; + +struct mk_reply_slot { + atomic64_t state_generation; + u64 request_id; + u32 kind; + s32 status; + u32 value; + u32 reserved; +}; + +struct mk_reply_table { + struct mk_reply_slot slots[MK_REPLY_SLOTS]; + atomic_t late_replies; + atomic_t cancelled_slots; + atomic_t atomic_timeouts; + atomic_t indeterminate_timeouts; + atomic_t occupied_failures; +}; + +struct mk_reply_handle { + u32 slot; + u32 kind; + u64 request_id; + u64 generation; +}; + /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + u32 ready; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ - unsigned int type; /* User-defined type identifier */ + s32 sender_instance_id; /* Receiver-authenticated duplex peer */ + unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ char buffer[MK_MAX_DATA_SIZE]; /* Actual data buffer */ }; /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ + atomic_t full_failures; +}; + +struct mk_irq_mailbox_entry { + atomic64_t pending_generation; + u64 lifecycle_epoch; + u32 lifecycle_generation; + u32 device_id; + u32 local_irq; + u16 vector; + u16 reserved; }; +struct mk_irq_mailbox { + atomic64_t pending_bitmap[MK_IRQ_MAILBOX_WORDS]; + struct mk_irq_mailbox_entry entries[MK_IRQ_MAILBOX_SLOTS]; + atomic_t next_generation; + atomic_t recorded; + atomic_t coalesced; + atomic_t masked_deferred; + atomic_t stale; + atomic_t dispatch_failed; + atomic_t saturated; +}; + +static inline u64 mk_irq_mailbox_token(u32 generation, u32 pending) +{ + return (u64)generation << MK_IRQ_MAILBOX_GENERATION_SHIFT | pending; +} + +static inline u32 mk_irq_mailbox_generation(u64 token) +{ + return token >> MK_IRQ_MAILBOX_GENERATION_SHIFT; +} + +static inline u32 mk_irq_mailbox_pending(u64 token) +{ + return token & MK_IRQ_MAILBOX_PENDING_MASK; +} + +static inline bool mk_irq_mailbox_masked(u64 token) +{ + return token & MK_IRQ_MAILBOX_MASKED; +} + +static inline bool mk_irq_mailbox_consuming(u64 token) +{ + return token & MK_IRQ_MAILBOX_CONSUMING; +} + /* The spawn boot tree's fixed budget, shared by allocator and writer */ #define MK_MANIFEST_SIZE SZ_256K /* Room the manifest leaves for a user-provided host tree */ @@ -109,9 +208,10 @@ struct mk_ipi_ring { /* Presence table capacity, in CPUs (see mk_cpu_rank) */ #define MK_PARKED_MAX 512 -/* Shared memory structures - per-instance design */ +/* One duplex link per parent/child pair. Each ring has one kernel producer. */ struct mk_shared_data { - struct mk_ipi_ring ring; /* IPI message ring buffer */ + struct mk_ipi_ring to_child; + struct mk_ipi_ring to_parent; /* * Force-halt marker, host-owned. Armed before the host NMIs the * instance's CPUs and cleared with the rest of this struct when @@ -131,6 +231,95 @@ struct mk_shared_data { * the struct on re-exec. */ u8 parked[MK_PARKED_MAX]; + s32 parent_id; + s32 child_id; + u32 reserved; + u64 parent_doorbell_cpu; + u64 child_doorbell_cpu; + /* Changes on every launch; zero means the link has not been launched. */ + u64 spawn_epoch; + /* Generation-tagged synchronous replies, independent of ring progress. */ + struct mk_reply_table replies; + struct mk_irq_mailbox irq_mailbox; +}; + +static inline void mk_reply_table_reset(struct mk_reply_table *table) +{ + unsigned int i; + + for (i = 0; i < MK_REPLY_SLOTS; i++) { + atomic64_set(&table->slots[i].state_generation, MK_REPLY_FREE); + WRITE_ONCE(table->slots[i].request_id, 0); + WRITE_ONCE(table->slots[i].kind, 0); + WRITE_ONCE(table->slots[i].status, 0); + WRITE_ONCE(table->slots[i].value, 0); + } + atomic_set(&table->late_replies, 0); + atomic_set(&table->cancelled_slots, 0); + atomic_set(&table->atomic_timeouts, 0); + atomic_set(&table->indeterminate_timeouts, 0); + atomic_set(&table->occupied_failures, 0); +} + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + unsigned int i; + + for (i = 0; i < MK_IPI_RING_SIZE; i++) + WRITE_ONCE(ring->entries[i].ready, 0); + atomic_set(&ring->full_failures, 0); +} + +static inline void mk_irq_mailbox_reset(struct mk_irq_mailbox *mailbox) +{ + unsigned int i; + + for (i = 0; i < MK_IRQ_MAILBOX_WORDS; i++) + atomic64_set(&mailbox->pending_bitmap[i], 0); + for (i = 0; i < MK_IRQ_MAILBOX_SLOTS; i++) { + struct mk_irq_mailbox_entry *entry = &mailbox->entries[i]; + + atomic64_set(&entry->pending_generation, 0); + WRITE_ONCE(entry->lifecycle_epoch, 0); + WRITE_ONCE(entry->lifecycle_generation, 0); + WRITE_ONCE(entry->device_id, 0); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, 0); + WRITE_ONCE(entry->reserved, 0); + } + atomic_set(&mailbox->next_generation, 0); + atomic_set(&mailbox->recorded, 0); + atomic_set(&mailbox->coalesced, 0); + atomic_set(&mailbox->masked_deferred, 0); + atomic_set(&mailbox->stale, 0); + atomic_set(&mailbox->dispatch_failed, 0); + atomic_set(&mailbox->saturated, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->to_child); + mk_ipi_ring_reset(&shared->to_parent); + WRITE_ONCE(shared->force_halt, 0); + memset(shared->parked, 0, sizeof(shared->parked)); + mk_reply_table_reset(&shared->replies); + mk_irq_mailbox_reset(&shared->irq_mailbox); +} + +struct mk_ipi_endpoint { + struct mk_instance *peer; + struct mk_ipi_ring *tx; + struct mk_ipi_ring *rx; + raw_spinlock_t tx_lock; + raw_spinlock_t rx_lock; + u32 tx_head; + u32 rx_tail; + u64 tx_epoch; + bool tx_enabled; + bool rx_dispatching; + bool parent_side; + bool registered; + struct list_head rx_node; }; /* Function pointer type for IPI callbacks */ @@ -156,6 +345,9 @@ struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, /** * multikernel_unregister_handler - Unregister a multikernel IPI callback * @handler: Handler pointer returned from multikernel_register_handler + * + * Must be called from process context and not from any multikernel IPI + * callback. */ void multikernel_unregister_handler(struct mk_ipi_handler *handler); @@ -166,17 +358,42 @@ void multikernel_unregister_handler(struct mk_ipi_handler *handler); * @data_size: Size of data * @type: User-defined type identifier * - * This function copies the data to per-CPU storage and sends an IPI - * to the target CPU. + * This process-context API looks up the instance by ID, copies the data to + * its endpoint, and sends an IPI to the target CPU. Atomic callers with a + * lifetime-stable instance must use multikernel_send_ipi_data_to_host(). * * Returns 0 on success, negative error code on failure */ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type); +int multikernel_send_ipi_data_to_host(void *data, size_t data_size, + unsigned long type); void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side); +void mk_ipi_endpoint_unregister(struct mk_instance *instance); +void mk_ipi_endpoint_close(struct mk_instance *instance); +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu); +void mk_ipi_handlers_enable(void); +int mk_reply_reserve(struct mk_shared_data *shared, u32 kind, u64 request_id, + struct mk_reply_handle *reply); +int mk_reply_claim(struct mk_instance *instance, + const struct mk_reply_handle *reply); +int mk_reply_begin_execute(struct mk_instance *instance, + const struct mk_reply_handle *reply); +int mk_reply_publish(struct mk_instance *instance, + const struct mk_reply_handle *reply, s32 status, u32 value); +int mk_reply_wait_atomic(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_us, + s32 *status, u32 *value); +int mk_reply_wait(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_ms, + s32 *status, u32 *value); +void mk_reply_release(struct mk_shared_data *shared, + struct mk_reply_handle *reply); +void mk_reply_scan(struct mk_shared_data *shared); /* * Multikernel Messaging System @@ -192,6 +409,7 @@ void mk_ipi_ring_drop_pending(void); #define MK_MSG_SYSTEM 0x3000 #define MK_MSG_USER 0x4000 #define MK_MSG_NETWORK 0x5000 +#define MK_MSG_PCI 0x6000 /* I/O interrupt forwarding subtypes */ #define MK_IO_IRQ_FORWARD (MK_MSG_IO + 1) @@ -219,6 +437,14 @@ void mk_ipi_ring_drop_pending(void); #define MK_NET_VSOCK_PKT (MK_MSG_NETWORK + 1) /* vsock packet */ #define MK_NET_DATA_READY (MK_MSG_NETWORK + 2) /* Data available notification */ +/* Host-mediated PCI control-plane subtypes */ +#define MK_PCI_CFG_REQUEST (MK_MSG_PCI + 1) +#define MK_PCI_CFG_RESPONSE (MK_MSG_PCI + 2) +#define MK_PCI_IRQ_REQUEST (MK_MSG_PCI + 3) +#define MK_PCI_IRQ_RESPONSE (MK_MSG_PCI + 4) +#define MK_PCI_RESET_REQUEST (MK_MSG_PCI + 5) +#define MK_PCI_RESET_RESPONSE (MK_MSG_PCI + 6) + /** * Core message structure */ @@ -240,6 +466,85 @@ struct mk_io_irq_payload { u32 vector; /* Interrupt vector */ u32 device_id; /* Device identifier (optional) */ u32 flags; /* Control flags (priority, etc.) */ + u32 lifecycle_generation; + u32 reserved; + u64 lifecycle_epoch; +}; + +#define MK_PCI_IRQ_ID(domain, bus, devfn) \ + (((u32)(domain) << 16) | ((u32)(bus) << 8) | (u32)(devfn)) +#define MK_PCI_IRQ_ID_DOMAIN(id) ((u16)((id) >> 16)) +#define MK_PCI_IRQ_ID_BUS(id) ((u8)((id) >> 8)) +#define MK_PCI_IRQ_ID_DEVFN(id) ((u8)(id)) + +struct mk_pci_cfg_request { + u64 request_id; + u64 lifecycle_epoch; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u16 reg; + u8 len; + u8 write; + u32 value; + u32 reply_slot; + u32 reply_reserved; + u64 reply_generation; +}; + +struct mk_pci_cfg_response { + u64 request_id; + s32 status; + u32 value; +}; + +enum mk_pci_irq_operation { + MK_PCI_IRQ_SETUP = 1, + MK_PCI_IRQ_RESTORE_BEGIN, + MK_PCI_IRQ_BIND, + MK_PCI_IRQ_COMMIT, + MK_PCI_IRQ_ACTIVATE, + MK_PCI_IRQ_TEARDOWN, +}; + +enum mk_pci_msi_lifecycle { + MK_PCI_MSI_IDLE = 0, + MK_PCI_MSI_PREPARED, + MK_PCI_MSI_COMMITTED, + MK_PCI_MSI_ACTIVE, + MK_PCI_MSI_FAILED, +}; + +struct mk_pci_irq_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u16 operation; + u16 vector; + u16 nr_vectors; + u8 msix; + u8 reserved; + u32 local_irq; + u32 reply_slot; + u32 lifecycle_generation; + u64 reply_generation; + u64 lifecycle_epoch; +}; + +struct mk_pci_reset_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u32 reset_generation; + u32 reply_slot; + u32 reserved; + u64 reply_generation; + u64 lifecycle_epoch; }; /* IRQ control flags */ @@ -309,7 +614,8 @@ struct mk_shutdown_payload { * Message handler callback type */ typedef void (*mk_msg_handler_t)(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx); + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx); /* Opaque type for pending message tracking */ struct mk_pending_msg; @@ -319,17 +625,23 @@ struct mk_pending_msg; */ /** - * mk_send_message - Send a message to another CPU + * mk_send_message - Send a message to another CPU from process context * @instance_id: Target multikernel instance ID * @msg_type: Message type identifier * @subtype: Message subtype * @payload: Pointer to payload data (can be NULL) * @payload_len: Length of payload data * + * This ID-based API may sleep while looking up the target instance. Atomic + * callers with a lifetime-stable instance must use + * mk_send_message_to_instance(). + * * Returns 0 on success, negative error code on failure */ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, void *payload, u32 payload_len); +int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, + u32 subtype, void *payload, u32 payload_len); /** * mk_register_msg_handler - Register handler for specific message type @@ -445,7 +757,7 @@ size_t mk_pool_total_bytes(void); size_t mk_pool_avail_bytes(void); bool mk_pool_empty(void); int mk_pool_for_each_chunk(int (*fn)(struct mk_pool_chunk *, void *), void *data); -bool mk_pool_cpus_returned(void); +int mk_pool_park_teardown(void); /** * struct mk_pool_chunk_range - a pool chunk copied out of the chunk list @@ -471,7 +783,7 @@ int mk_pool_snapshot_chunks(struct mk_pool_chunk_range *out, int max); int mk_pool_park_setup(void); int mk_arch_pool_chunk_added(phys_addr_t start, size_t size); bool mk_pool_park_uses(phys_addr_t start, size_t size); -int mk_pool_park_teardown(void); +int mk_arch_pool_park_teardown(void); /* The pool park set (park page, slot, page tables) as base,size pairs */ int mk_pool_park_regions(u64 *pairs, int max); /* The slot pool CPUs watch, 0 when there is none */ @@ -492,7 +804,7 @@ static inline bool mk_pool_park_uses(phys_addr_t start, size_t size) return false; } -static inline int mk_pool_park_teardown(void) +static inline int mk_arch_pool_park_teardown(void) { return 0; } @@ -562,6 +874,13 @@ struct mk_memory_region { * Format: vendor:device@domain:bus:slot.func */ #define MK_PCI_ALIAS_LEN 32 /* Devicetree spec: alias names are at most 31 chars */ +#define MK_PCI_RESOURCE_COUNT 6 + +struct mk_pci_resource { + u64 start; + u64 end; + u64 flags; +}; /* * A PCI host bridge as the instance device tree describes it: the @@ -579,6 +898,8 @@ struct mk_pci_device { u8 bus; /* PCI bus number */ u8 slot; /* PCI slot number */ u8 func; /* PCI function number */ + struct mk_pci_resource resources[MK_PCI_RESOURCE_COUNT]; + bool resources_valid; struct list_head list; /* Link to device list */ }; @@ -644,6 +965,8 @@ struct mk_instance { int id; /* Kernel-assigned instance ID */ char *name; /* User-provided instance name */ enum mk_instance_state state; /* Current state */ + /* Serializes memory topology changes with PCI assignment leases. */ + struct mutex resource_mutex; /* Resource management - list of reserved memory regions */ struct list_head memory_regions; /* List of struct mk_memory_region */ @@ -654,11 +977,18 @@ struct mk_instance { /* CPU resources */ struct mk_cpu_set *cpus; /* Set of assigned physical CPU IDs */ + /* Pins the CPU selected for control messages and forwarded IRQs. */ + struct rw_semaphore control_route_sem; + raw_spinlock_t control_route_lock; + mk_phys_cpu_t irq_route_cpu; + struct delayed_work irq_retry_work; /* PCI device resources */ struct list_head pci_devices; /* List of struct mk_pci_device */ int pci_device_count; /* Number of PCI devices */ bool pci_devices_valid; /* Whether PCI device list is valid */ + /* Host-only live PCI assignment leases (private elements). */ + struct list_head pci_assignments; /* Platform device resources */ struct list_head platform_devices; /* List of struct mk_platform_device */ @@ -669,6 +999,11 @@ struct mk_instance { struct mk_shared_data *ipi_data; /* IPI shared memory buffer (virtual address) */ phys_addr_t ipi_phys; /* IPI buffer physical address */ u32 ipi_pages; /* IPI buffer size in pages */ + /* + * Separate host self area used when a spawn fences its parent. + * NULL for ordinary child records, whose halt area is @ipi_data. + */ + struct mk_shared_data *halt_data; /* * On a spawn's record of its host: the physical address of the * host's pool wake slot, where the host's CPUs park. A backup @@ -683,6 +1018,7 @@ struct mk_instance { * explicitly so routing never masquerades as ownership in @cpus. */ mk_phys_cpu_t ipi_target; + struct mk_ipi_endpoint ipi_endpoint; /* Kexec integration */ struct kimage *kimage; /* Associated kimage object */ @@ -733,6 +1069,24 @@ struct mk_instance { struct kref refcount; /* Reference count for cleanup */ }; +static inline mk_phys_cpu_t +mk_instance_irq_route_load(const struct mk_instance *instance) +{ + /* Pair with the release store that publishes route changes. */ + return smp_load_acquire(&instance->irq_route_cpu); +} + +static inline void mk_instance_irq_route_store(struct mk_instance *instance, + mk_phys_cpu_t target) +{ + unsigned long flags; + + /* Publish the route after its associated control state. */ + raw_spin_lock_irqsave(&instance->control_route_lock, flags); + smp_store_release(&instance->irq_route_cpu, target); + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); +} + /** * Device Tree Parsing Functions */ @@ -856,14 +1210,14 @@ struct mk_instance *mk_instance_get(struct mk_instance *instance); void __noreturn mk_halt_to_pool(void); /** - * mk_instance_reserve_resources() - Reserve CPU and memory resources for instance - * @instance: Instance to reserve resources for - * @config: Device tree configuration with memory size and CPU assignment + * mk_instance_reserve_resources() - Atomically reserve instance resources + * @instance: Empty instance to populate + * @config: Parsed memory, CPU, PCI, and platform resource configuration * - * Allocates the specified memory size from the multikernel pool, creates - * memory regions, and copies CPU assignment. + * Reserves every configured resource class or returns all resources acquired + * by the attempt. A failure never leaves a partially populated instance. * - * Returns 0 on success, negative error code on failure. + * Returns: 0 on success, negative error code on failure */ int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config); @@ -900,21 +1254,6 @@ void mk_instance_mem_free(struct mk_instance *instance, void *virt_addr, size_t const char *mk_state_to_string(enum mk_instance_state state); enum mk_instance_state mk_string_to_state(const char *str); -/** - * Kexec Integration Functions - * - * These functions bridge the gap between the sysfs instance management - * and the kexec multikernel system. - */ - -/** - * mk_instance_set_kexec_active() - Mark instance as active for kexec - * @mk_id: Multikernel ID from kexec system - * - * Returns 0 on success, negative error code on failure. - */ -int mk_instance_set_kexec_active(int mk_id); - /* * The declarations below are referenced from always-built code (kexec, * PCI and platform device probing, SMP setup), so they carry stubs for @@ -936,13 +1275,40 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +void mk_instance_mark_failed(struct mk_instance *instance); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size); -/* Device probe filtering against the instance's allowlist */ +/* Device filtering against the current kernel's boot tree */ bool mk_pci_should_probe(struct pci_bus *bus, int devfn); +bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, + u16 *device); +#if defined(CONFIG_X86) && defined(CONFIG_PCI) +bool mk_pci_controlled(struct pci_dev *dev); +int mk_pci_reset_flr(struct pci_dev *dev); +#else +static inline bool mk_pci_controlled(struct pci_dev *dev) { return false; } +static inline int mk_pci_reset_flr(struct pci_dev *dev) { return -EOPNOTSUPP; } +#endif +#if defined(CONFIG_X86) && defined(CONFIG_PCI) && defined(CONFIG_PCI_MSI) +bool mk_pci_msi_controlled(struct pci_dev *dev); +int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type); +int mk_pci_msi_activate(struct pci_dev *dev); +int mk_pci_msi_restore(struct pci_dev *dev); +int mk_pci_msi_teardown(struct pci_dev *dev); +void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared); +#else +static inline bool mk_pci_msi_controlled(struct pci_dev *dev) { return false; } +static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) { return 0; } +static inline int mk_pci_msi_activate(struct pci_dev *dev) { return 0; } +static inline int mk_pci_msi_restore(struct pci_dev *dev) { return 0; } +static inline int mk_pci_msi_teardown(struct pci_dev *dev) { return 0; } +static inline void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) { } +#endif bool mk_platform_device_allowed(const char *name, const char *hid); /* Early CPU registration from the manifest (spawn kernels) */ @@ -950,9 +1316,19 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ -int mk_manifest_finalize(struct kimage *image); +int mk_manifest_finalize(struct kimage *image, + mk_phys_cpu_t parent_doorbell_cpu); +#ifdef CONFIG_PCI +int mk_pci_prepare_instance_start(struct mk_instance *instance); +#else +static inline int mk_pci_prepare_instance_start(struct mk_instance *instance) +{ + return 0; +} +#endif #else static inline bool multikernel_allow_emergency_restart(void) { @@ -967,6 +1343,15 @@ static inline bool mk_crash_notes_wanted(void) return false; } +static inline bool mk_pci_controlled(struct pci_dev *dev) { return false; } +static inline int mk_pci_reset_flr(struct pci_dev *dev) { return -EOPNOTSUPP; } +static inline bool mk_pci_msi_controlled(struct pci_dev *dev) { return false; } +static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) { return 0; } +static inline int mk_pci_msi_activate(struct pci_dev *dev) { return 0; } +static inline int mk_pci_msi_restore(struct pci_dev *dev) { return 0; } +static inline int mk_pci_msi_teardown(struct pci_dev *dev) { return 0; } +static inline void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) { } + static inline int multikernel_force_halt_by_id(int mk_id) { return -ENODEV; @@ -995,6 +1380,14 @@ static inline bool mk_pci_should_probe(struct pci_bus *bus, int devfn) { return true; } + +static inline bool +mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, u16 *device) +{ + return false; +} + static inline bool mk_platform_device_allowed(const char *name, const char *hid) { return true; @@ -1005,6 +1398,11 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -1012,6 +1410,7 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 +/* Bumped whenever the shared-memory layout or message semantics change. */ #define MK_FDT_COMPATIBLE "multikernel-v1" /** @@ -1029,8 +1428,6 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) * the host side, and restore an instance from it on the spawn side. */ - - /** * mk_instance_restore_from_manifest() - Restore this instance from the manifest * @@ -1048,7 +1445,7 @@ int __init mk_instance_restore_from_manifest(void); */ /** - * PCI Device Enforcement Functions + * PCI Device Filtering Functions */ /** @@ -1056,13 +1453,24 @@ int __init mk_instance_restore_from_manifest(void); * @bus: PCI bus * @devfn: PCI device/function number * - * Called BEFORE any PCI config space reads to determine if probing - * should proceed. This prevents config space accesses to devices - * that are not in the whitelist, avoiding hardware conflicts on bare metal. + * Returns: true if probing should proceed, false to skip entirely. + */ + +/** + * mk_pci_get_assigned_identity_bdf() - Identify an assigned PCI function + * @domain: PCI domain number + * @bus: PCI bus number + * @devfn: PCI device/function number + * @vendor: optional assigned vendor ID output + * @device: optional assigned device ID output * - * Returns: true if probing should proceed, false to skip entirely + * The raw x86 PCI configuration wrappers use this BDF-only lookup before + * reaching their hardware backend. Spawn kernels accept exact functions + * recorded in their boot tree; a host kernel without a parent keeps normal + * PCI discovery. A privileged spawn kernel can bypass these wrappers, so the + * check prevents accidental access rather than isolating a hostile kernel. * - * Declared above with the CONFIG_MULTIKERNEL stubs. + * Returns: true for an exact assignment match, false otherwise. */ /** @@ -1113,6 +1521,8 @@ void mk_arch_register_cpu(mk_phys_cpu_t phys_id); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86, SDEI or diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..9b5f5978ebe53d --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* One generation marker shared by the spawn context and image capabilities. */ +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b0003 +#define MK_VMLINUX_NOTE_TYPE 0x4d4b0003 + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 1c270f1d512301..591b35215bdd08 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -503,6 +503,9 @@ struct pci_dev { unsigned int non_mappable_bars:1; /* BARs can't be mapped to user-space */ pci_dev_flags_t dev_flags; atomic_t enable_cnt; /* pci_enable_device has been called */ +#ifdef CONFIG_MULTIKERNEL + u32 multikernel_reset_generation; +#endif spinlock_t pcie_cap_lock; /* Protects RMW ops in capability accessors */ u32 saved_config_space[16]; /* Config space saved at suspend time */ @@ -524,6 +527,12 @@ struct pci_dev { #ifdef CONFIG_PCI_MSI void __iomem *msix_base; raw_spinlock_t msi_lock; +#ifdef CONFIG_MULTIKERNEL + u32 multikernel_msi_generation; + u16 multikernel_msi_nvec; + u8 multikernel_msi_state; + u8 multikernel_msi_msix; +#endif #endif struct pci_vpd vpd; #ifdef CONFIG_PCIE_DPC diff --git a/kernel/kexec.c b/kernel/kexec.c index fff660fef63330..7cc0f3150aac6c 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -127,6 +127,14 @@ static int do_kexec_load(unsigned long entry, unsigned long nr_segments, int count, i; count = kimage_get_all_by_type(KEXEC_TYPE_MULTIKERNEL, images, 10); + for (i = 0; i < count; i++) { + ret = kimage_prepare_multikernel_unload(images[i]); + if (ret) { + pr_err("Multikernel image %d is still in use: %d\n", + images[i]->mk_id, ret); + goto out_unlock; + } + } for (i = 0; i < count; i++) { kimage_remove_from_list(images[i]); kimage_free(images[i]); diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index e1d0d55b1a0f46..31ef02d5db4438 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -57,6 +57,7 @@ #include #include "kexec_internal.h" +#include "multikernel/internal.h" atomic_t __kexec_lock = ATOMIC_INIT(0); @@ -595,6 +596,7 @@ void kimage_free(struct kimage *image) { kimage_entry_t *ptr, entry; kimage_entry_t ind = 0; + struct mk_instance *route_instance = NULL; if (!image) return; @@ -607,8 +609,26 @@ void kimage_free(struct kimage *image) kimage_update_compat_pointers(NULL, KEXEC_TYPE_CRASH); if (image->type == KEXEC_TYPE_MULTIKERNEL) { + unsigned long route_flags; unsigned long i; + route_instance = image->mk_instance; + if (route_instance) { + /* + * The retry worker takes the raw route lock. Drain it before + * taking the sleepable route lock to keep this ordering acyclic, + * and keep it disabled until the shared pages are gone. + */ + mk_pci_irq_retry_disable_sync(route_instance); + down_write(&route_instance->control_route_sem); + } + + /* Stop delivery before image-owned shared pages are returned. */ +#ifdef CONFIG_MULTIKERNEL + if (image->mk_instance) + mk_ipi_endpoint_unregister(image->mk_instance); +#endif + for (i = 0; i < image->nr_segments; i++) { void *virt_addr = phys_to_virt(image->segment[i].mem); @@ -625,11 +645,14 @@ void kimage_free(struct kimage *image) * instance lands in pages the allocator has already * handed to someone else. */ - image->mk_instance->ipi_data = NULL; - image->mk_instance->ipi_phys = 0; + raw_spin_lock_irqsave(&route_instance->control_route_lock, + route_flags); + route_instance->ipi_data = NULL; + route_instance->ipi_phys = 0; + raw_spin_unlock_irqrestore(&route_instance->control_route_lock, + route_flags); image->mk_instance->kimage = NULL; mk_instance_set_state(image->mk_instance, MK_STATE_READY); - mk_instance_put(image->mk_instance); image->mk_instance = NULL; } @@ -645,6 +668,11 @@ void kimage_free(struct kimage *image) __free_pages(phys_to_page(image->mk_ipi), order); image->mk_ipi = 0; } + if (route_instance) { + up_write(&route_instance->control_route_sem); + mk_pci_irq_retry_enable(route_instance); + mk_instance_put(route_instance); + } } #ifdef CONFIG_CRASH_DUMP if (image->vmcoreinfo_data_copy) { @@ -1683,12 +1711,46 @@ struct kimage *kimage_find_by_id(int mk_id) return image; } +int kimage_prepare_multikernel_unload(struct kimage *image) +{ + struct mk_instance *instance; + int state; + int ret; + + if (!image || image->type != KEXEC_TYPE_MULTIKERNEL) + return -EINVAL; + instance = image->mk_instance; + if (!instance) + return -EINVAL; + + /* ACTIVE always means CPUs may still execute from image memory. */ + state = READ_ONCE(instance->state); + if (state == MK_STATE_ACTIVE) + return -EBUSY; + + /* + * FAILED can describe a partially aborted launch, while LOADED may + * precede the final park acknowledgment. In either case the parked + * proof, rather than state alone, permits the image to be released. + */ + ret = mk_instance_confirm_parked(instance); + if (ret) + return ret == -ENOMEM ? ret : -EBUSY; + + return 0; +} + int multikernel_kexec_by_id(int mk_id) { struct kimage *mk_image; struct mk_instance *instance; + unsigned long route_flags; + bool instance_locked = false; + bool transaction_locked = false; + bool route_locked = false; int cpu = -1; int i, rc; + mk_phys_cpu_t parent_cpu; if (!kexec_trylock()) return -EBUSY; @@ -1701,9 +1763,24 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } + mutex_lock(&mk_instance_mutex); + instance_locked = true; + mk_cpu_transaction_lock(); + transaction_locked = true; + down_write(&instance->control_route_sem); + route_locked = true; if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); + if (!mk_cpu_set_contains(instance->cpus, + mk_instance_irq_route_load(instance))) + mk_instance_irq_route_store(instance, phys_cpu); cpu = arch_cpu_from_physical_id(phys_cpu); if (cpu < 0) { pr_err("Physical CPU %llu not found in logical CPU map\n", phys_cpu); @@ -1732,6 +1809,18 @@ int multikernel_kexec_by_id(int mk_id) goto unlock; } + /* + * Stop and reset every leased VF before rewriting instance memory. A + * force-halted kernel may have left bus mastering enabled and DMA in + * flight into the image that is about to be reused. + */ + rc = mk_pci_prepare_instance_start(instance); + if (rc) { + pr_err("Failed to prepare PCI assignments for instance %d restart: %d\n", + mk_id, rc); + goto unlock; + } + /* * Booting consumes the image: the spawn kernel writes its .data and * patches its own text, so the copy in instance memory is spent once @@ -1754,11 +1843,20 @@ int multikernel_kexec_by_id(int mk_id) } } - rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + /* Logical CPU 0 stays with this parent and receives child doorbells. */ + parent_cpu = arch_cpu_physical_id(0); + if (parent_cpu == MK_PHYS_CPU_INVALID) { + rc = -ENODEV; + goto unlock; + } + rc = mk_manifest_finalize(mk_image, parent_cpu); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + mutex_unlock(&mk_instance_mutex); + instance_locked = false; + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1769,39 +1867,46 @@ int multikernel_kexec_by_id(int mk_id) * each other's messages. */ if (mk_image->mk_ipi) { + raw_spin_lock_irqsave(&instance->control_route_lock, + route_flags); instance->ipi_phys = mk_image->mk_ipi; instance->ipi_data = phys_to_virt(mk_image->mk_ipi); instance->ipi_pages = PAGE_ALIGN(sizeof(struct mk_shared_data)) >> PAGE_SHIFT; + raw_spin_unlock_irqrestore(&instance->control_route_lock, + route_flags); } - /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. - */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); + /* Reset only this parent/child link, after the old child is parked. */ + if (instance->ipi_data) { + mk_ipi_link_reset(instance, mk_self->id, mk_id, + parent_cpu, mk_cpu_set_first(instance->cpus)); + } + rc = mk_arch_spawn_instance(mk_image, instance, cpu); + if (rc) { + mk_ipi_endpoint_close(instance); + goto unlock; + } /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * Publish the running state before dropping the global kexec lock so a + * second exec cannot race this boot while its CPUs are leaving the park + * loop. */ - mk_ipi_ring_drop_pending(); + mk_instance_set_state(instance, MK_STATE_ACTIVE); - rc = mk_arch_spawn_instance(mk_image, instance, cpu); - if (rc == 0) { - rc = mk_instance_set_kexec_active(mk_image->mk_id); - if (rc) - pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); - } + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + kexec_unlock(); + return 0; unlock: + if (route_locked) + up_write(&instance->control_route_sem); + if (transaction_locked) + mk_cpu_transaction_unlock(); + if (instance_locked) + mutex_unlock(&mk_instance_mutex); kexec_unlock(); return rc; } diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 10e670a479d79b..269e553e3f8abc 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -369,16 +369,13 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, image->mk_instance = instance; image->mk_id = mk_id; - instance->kimage = image; - - pr_info("Associated kimage with multikernel instance %d\n", mk_id); fdt_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(MK_MANIFEST_SIZE)); if (!fdt_page) { pr_err("Failed to allocate FDT pages for multikernel kimage\n"); ret = -ENOMEM; - goto out_free_image; + goto out_free_fdt; } image->mk_manifest = page_to_phys(fdt_page); @@ -406,7 +403,7 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, ret = kimage_file_prepare_segments(image, kernel_fd, initrd_fd, cmdline_ptr, cmdline_len, flags); if (ret) - goto out_free_image; + goto out_free_fdt; ret = sanity_check_segment_list(image); if (ret) @@ -428,6 +425,11 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, } } + if (image->type == KEXEC_TYPE_MULTIKERNEL) { + image->mk_instance->kimage = image; + pr_info("Associated kimage with multikernel instance %d\n", + image->mk_id); + } *rimage = image; return 0; out_free_control_pages: @@ -436,8 +438,18 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, kimage_file_post_load_cleanup(image); out_free_fdt: if (image->type == KEXEC_TYPE_MULTIKERNEL) { + unsigned long i; + + for (i = 0; i < image->nr_segments; i++) { + void *virt_addr = phys_to_virt(image->segment[i].mem); + + mk_kimage_free(image, virt_addr, + image->segment[i].memsz); + } + image->nr_segments = 0; if (image->mk_ipi) { unsigned int order = get_order(sizeof(struct mk_shared_data)); + __free_pages(phys_to_page(image->mk_ipi), order); image->mk_ipi = 0; } @@ -446,13 +458,13 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, get_order(MK_MANIFEST_SIZE)); image->mk_manifest = 0; } - if (image->mk_instance) { - image->mk_instance->kimage = NULL; - mk_instance_put(image->mk_instance); - image->mk_instance = NULL; - } + /* + * The caller retains the lookup reference unless allocation + * succeeds and the image is published above. + */ + image->mk_instance = NULL; } -out_free_image: + kimage_file_free_cmdline(image); kfree(image); return ret; } @@ -499,9 +511,16 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, goto out; } + ret = kimage_prepare_multikernel_unload(mk_image); + if (ret) { + pr_err("Multikernel instance %d is still using its image: %d\n", + mk_id, ret); + goto out; + } + pr_info("Unloading kernel from multikernel instance %d\n", mk_id); - kimage_remove_from_list(mk_image); - image = mk_image; + /* Keep lookup, detach, and free serialized with exec. */ + kimage_free(mk_image); ret = 0; goto out; } else { @@ -648,6 +667,15 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, arch_kexec_protect_crashkres(); #endif + /* + * A partially prepared multikernel image is already visible through + * its instance. Keep failure cleanup serialized with load, exec, and + * unload until that pointer and all image-owned resources are gone. + */ + if (image && image->type == KEXEC_TYPE_MULTIKERNEL) { + kimage_free(image); + image = NULL; + } kexec_unlock(); kimage_free(image); return ret; diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index cec1ea32d83361..5b90f67d722b57 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -17,6 +17,7 @@ config MULTIKERNEL depends on KEXEC_CORE depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE + depends on OF select LIBFDT select OF_DYNAMIC if OF help @@ -32,4 +33,7 @@ config MULTIKERNEL - A manifest handed to each spawn kernel on multikernel's own boot channel, describing its resources and message rings + SR-IOV VF assignment additionally requires PCI_IOV and an active + hardware IOMMU with isolated interrupt delivery. Assignment fails + closed when those isolation facilities are unavailable. If unsure, say N. diff --git a/kernel/multikernel/Makefile b/kernel/multikernel/Makefile index cf2e1391038ec7..2ff0a209b57c0c 100644 --- a/kernel/multikernel/Makefile +++ b/kernel/multikernel/Makefile @@ -5,6 +5,7 @@ obj-y += core.o cpuset.o mem.o contig.o kernfs.o dts.o instance_dt.o manifest.o ipi.o messaging.o overlay.o hotplug.o baseline.o obj-$(CONFIG_OF) += of_devices.o +obj-$(CONFIG_PCI) += pci.o # DMA-BUF heap for multikernel memory allocation obj-$(CONFIG_DMABUF_HEAPS) += dma_heap.o diff --git a/kernel/multikernel/baseline.c b/kernel/multikernel/baseline.c index 7e7d73f76ff8ed..a9ec32bf9a54b1 100644 --- a/kernel/multikernel/baseline.c +++ b/kernel/multikernel/baseline.c @@ -238,10 +238,14 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, } if (strcmp(device_type, "pci") == 0) { + struct mk_pci_device *existing; struct mk_pci_device *pci_dev; const char *pci_id_str; const fdt32_t *vendor_prop, *device_prop; - unsigned int domain, bus, slot, func; + u32 vendor, device; + u16 domain; + u8 bus, slot, func; + int ret; pci_id_str = fdt_getprop(fdt, dev_node, "pci-id", &len); if (!pci_id_str) { @@ -250,10 +254,12 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s' for device '%s'\n", - pci_id_str, dev_name); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s' for device '%s'\n", + len, pci_id_str, dev_name); + return ret; } vendor_prop = fdt_getprop(fdt, dev_node, "vendor-id", &len); @@ -269,6 +275,22 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, dev_name); return -EINVAL; } + vendor = fdt32_to_cpu(*vendor_prop); + device = fdt32_to_cpu(*device_prop); + if (vendor > U16_MAX || device > U16_MAX) { + pr_err("Out-of-range vendor-id or device-id for device '%s'\n", + dev_name); + return -ERANGE; + } + list_for_each_entry(existing, pci_list, list) { + if (existing->domain == domain && + existing->bus == bus && existing->slot == slot && + existing->func == func) { + pr_err("Duplicate baseline PCI BDF %04x:%02x:%02x.%x\n", + domain, bus, slot, func); + return -EEXIST; + } + } pci_dev = kzalloc_obj(*pci_dev, GFP_KERNEL); if (!pci_dev) { @@ -279,12 +301,12 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, alias = mk_dt_node_alias(fdt, dev_node); if (alias) strscpy(pci_dev->alias, alias, sizeof(pci_dev->alias)); - pci_dev->vendor = (u16)fdt32_to_cpu(*vendor_prop); - pci_dev->device = (u16)fdt32_to_cpu(*device_prop); - pci_dev->domain = (u16)domain; - pci_dev->bus = (u8)bus; - pci_dev->slot = (u8)slot; - pci_dev->func = (u8)func; + pci_dev->vendor = (u16)vendor; + pci_dev->device = (u16)device; + pci_dev->domain = domain; + pci_dev->bus = bus; + pci_dev->slot = slot; + pci_dev->func = func; list_add_tail(&pci_dev->list, pci_list); pci_count++; @@ -384,7 +406,7 @@ static int mk_baseline_initialize_cpus(const struct mk_cpu_set *requested) static int mk_baseline_initialize_devices(struct list_head *pci_list) { struct mk_pci_device *pci_dev; - int ret, failed = 0, moved = 0; + int ret, failed = 0, available = 0; if (list_empty(pci_list)) { pr_debug("No PCI devices in baseline to move into the pool\n"); @@ -392,24 +414,25 @@ static int mk_baseline_initialize_devices(struct list_head *pci_list) } list_for_each_entry(pci_dev, pci_list, list) { - ret = mk_pool_device_add(pci_dev->domain, pci_dev->bus, - PCI_DEVFN(pci_dev->slot, pci_dev->func), - pci_dev->alias); + ret = mk_root_add_pci_device(pci_dev->domain, pci_dev->bus, + PCI_DEVFN(pci_dev->slot, pci_dev->func), + pci_dev->alias); if (ret) { - pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x not moved into the pool: %d\n", + pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x unavailable for the pool: %d\n", pci_dev->vendor, pci_dev->device, pci_dev->domain, pci_dev->bus, pci_dev->slot, pci_dev->func, ret); failed++; continue; } - moved++; + available++; } if (failed > 0) - pr_warn("Failed to move %d PCI devices into the pool\n", failed); + pr_warn("Failed to inventory %d PCI devices for the pool\n", failed); - pr_info("Moved %d PCI devices into the multikernel pool\n", moved); + pr_info("Inventoried %d PCI devices; host drivers remain bound until assignment\n", + available); return 0; } diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 9de50f787d2737..eff8380ec835c0 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -13,30 +14,114 @@ #include #include #include +#include #include +#include #include "internal.h" +/* Lock order: transaction -> route write -> ownership -> resources. */ +static DEFINE_MUTEX(mk_cpu_transaction_mutex); +static DEFINE_MUTEX(mk_cpu_ownership_mutex); + +void mk_cpu_transaction_lock(void) +{ + mutex_lock(&mk_cpu_transaction_mutex); +} + +void mk_cpu_transaction_unlock(void) +{ + mutex_unlock(&mk_cpu_transaction_mutex); +} + +void mk_cpu_ownership_lock(void) +{ + mutex_lock(&mk_cpu_ownership_mutex); +} + +void mk_cpu_ownership_unlock(void) +{ + mutex_unlock(&mk_cpu_ownership_mutex); +} + +void mk_cpu_ownership_assert_held(void) +{ + lockdep_assert_held(&mk_cpu_ownership_mutex); +} + +static int __mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing) +{ + mk_phys_cpu_t replacement = MK_PHYS_CPU_INVALID; + mk_phys_cpu_t route_cpu; + mk_phys_cpu_t phys_cpu; + unsigned int i; + + lockdep_assert_held_write(&instance->control_route_sem); + route_cpu = mk_instance_irq_route_load(instance); + if (route_cpu == MK_PHYS_CPU_INVALID || + !mk_cpu_set_contains(removing, route_cpu)) + return 0; + + mk_cpu_ownership_lock(); + mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { + if (!mk_cpu_set_contains(removing, phys_cpu)) { + replacement = phys_cpu; + break; + } + } + mk_cpu_ownership_unlock(); + if (replacement == MK_PHYS_CPU_INVALID && + READ_ONCE(instance->state) == MK_STATE_ACTIVE) + return -EBUSY; + + mutex_lock(&instance->resource_mutex); + if (replacement != MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, replacement); + mk_pci_sync_instance_irq_route(instance); + if (replacement == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); + mutex_unlock(&instance->resource_mutex); + return 0; +} + +int mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing) +{ + int ret; + + if (!instance || !removing) + return -EINVAL; + down_write(&instance->control_route_sem); + ret = __mk_instance_migrate_irq_route(instance, removing); + up_write(&instance->control_route_sem); + return ret; +} + static void mk_instance_return_all_cpus(struct mk_instance *instance) { if (!instance || mk_cpu_set_empty(instance->cpus)) return; - if (instance == mk_self || instance->id == 0) + if (!instance || instance == mk_self || instance->id == 0) return; mk_instance_return_cpus(instance, instance->cpus); } -static void mk_instance_return_pci_devices(struct mk_instance *instance) +static int mk_instance_return_pci_devices(struct mk_instance *instance) { struct mk_pci_device *pci_dev, *pci_tmp; int returned_count = 0; + int ret; - if (!instance || !instance->pci_devices_valid) - return; + if (!instance || instance == mk_self || instance->id == 0) + return 0; - if (instance == mk_self || instance->id == 0) - return; + ret = mk_pci_release_assignments(instance); + if (ret) + return ret; + if (!instance || !instance->pci_devices_valid) + return 0; if (!mk_self) { pr_warn("Cannot return PCI devices from instance %d (%s): no self instance\n", @@ -88,17 +173,18 @@ static void mk_instance_return_pci_devices(struct mk_instance *instance) } instance->pci_device_count = 0; instance->pci_devices_valid = false; + return 0; } static void mk_instance_return_platform_devices(struct mk_instance *instance) { - struct mk_platform_device *plat_dev, *plat_tmp; - int returned_count = 0; + struct mk_platform_device *device, *tmp; + int returned = 0; - if (!instance || !instance->platform_devices_valid) + if (!instance || instance == mk_self || instance->id == 0) return; - - if (instance == mk_self || instance->id == 0) + if (!instance->platform_devices_valid && + list_empty(&instance->platform_devices)) return; if (!mk_self) { @@ -107,35 +193,21 @@ static void mk_instance_return_platform_devices(struct mk_instance *instance) goto cleanup; } - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - struct mk_platform_device *self_dev; - - self_dev = kzalloc_obj(*self_dev, GFP_KERNEL); - if (!self_dev) - continue; - - *self_dev = *plat_dev; - INIT_LIST_HEAD(&self_dev->list); - - list_add_tail(&self_dev->list, &mk_self->platform_devices); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, list) { + list_move_tail(&device->list, &mk_self->platform_devices); mk_self->platform_device_count++; mk_self->platform_devices_valid = true; - - pr_debug("Returned platform device '%s' from instance %d to root\n", - self_dev->name, instance->id); - - returned_count++; + returned++; } - if (returned_count > 0) { + if (returned) pr_info("Returned %d platform devices from instance %d (%s) to self instance\n", - returned_count, instance->id, instance->name); - } + returned, instance->id, instance->name); cleanup: - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - list_del(&plat_dev->list); - kfree(plat_dev); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, list) { + list_del(&device->list); + kfree(device); } instance->platform_device_count = 0; instance->platform_devices_valid = false; @@ -165,19 +237,39 @@ static void mk_instance_track_dump(struct mk_instance *instance, atomic_dec(&mk_dump_backups); } +int mk_instance_release_resources(struct mk_instance *instance) +{ + int ret; + + if (!instance || instance == mk_self || instance->id == 0) + return 0; + ret = mk_instance_return_pci_devices(instance); + if (ret) + return ret; + mk_instance_return_platform_devices(instance); + mk_instance_return_all_cpus(instance); + mk_instance_free_memory(instance); + return 0; +} + static void mk_instance_release(struct kref *kref) { struct mk_instance *instance = container_of(kref, struct mk_instance, refcount); + int ret; pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); - - mk_instance_return_all_cpus(instance); - mk_instance_return_pci_devices(instance); - mk_instance_return_platform_devices(instance); - mk_instance_free_memory(instance); + ret = mk_instance_release_resources(instance); + if (WARN_ON_ONCE(ret)) { + pr_crit("Retaining multikernel instance %d (%s) after resource release failed: %d\n", + instance->id, instance->name, ret); + return; + } + mk_ipi_endpoint_unregister(instance); mk_instance_track_dump(instance, instance->state, MK_STATE_READY); + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -230,11 +322,20 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) goto err_free_name; instance->state = MK_STATE_READY; + init_rwsem(&instance->control_route_sem); + raw_spin_lock_init(&instance->control_route_lock); + instance->irq_route_cpu = MK_PHYS_CPU_INVALID; instance->ipi_target = MK_PHYS_CPU_INVALID; + raw_spin_lock_init(&instance->ipi_endpoint.tx_lock); + raw_spin_lock_init(&instance->ipi_endpoint.rx_lock); + INIT_LIST_HEAD(&instance->ipi_endpoint.rx_node); INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); + mutex_init(&instance->resource_mutex); + INIT_LIST_HEAD(&instance->pci_assignments); INIT_LIST_HEAD(&instance->platform_devices); + mk_pci_lease_instance_init(instance); kref_init(&instance->refcount); return instance; @@ -297,6 +398,8 @@ void mk_instance_free(struct mk_instance *instance) list_del(&plat_dev->list); kfree(plat_dev); } + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -327,6 +430,19 @@ void mk_instance_set_state(struct mk_instance *instance, */ } +void mk_instance_mark_failed(struct mk_instance *instance) +{ + if (!instance) + return; + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mk_instance_set_state(instance, MK_STATE_FAILED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); +} + struct mk_instance *mk_instance_find_by_name(const char *name) { struct mk_instance *instance; @@ -357,23 +473,6 @@ struct mk_instance *mk_instance_find(int mk_id) return instance; } -int mk_instance_set_kexec_active(int mk_id) -{ - struct mk_instance *instance; - - instance = mk_instance_find(mk_id); - if (!instance) { - pr_err("No sysfs instance found for multikernel ID %d\n", mk_id); - return -ENOENT; - } - - mk_instance_set_state(instance, MK_STATE_ACTIVE); - mk_instance_put(instance); - pr_info("Multikernel instance %d is now active\n", mk_id); - - return 0; -} - bool multikernel_allow_emergency_restart(void) { struct mk_instance *instance; @@ -423,12 +522,25 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; /* Empty until the instance first ran, so nothing of it is executing */ - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + if (!instance->cpus_on_slot) + return 0; + + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } + + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -436,6 +548,7 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -453,6 +566,7 @@ int mk_instance_confirm_parked(struct mk_instance *instance) int mk_instance_transfer_cpus(struct mk_instance *instance, const struct mk_cpu_set *cpus) { + struct mk_cpu_set *requested; unsigned int i, requested_count; mk_phys_cpu_t phys_cpu; int unavailable = 0; @@ -464,14 +578,26 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, return -EINVAL; } - requested_count = mk_cpu_set_count(cpus); + requested = mk_cpu_set_alloc(); + if (!requested) + return -ENOMEM; + + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(requested, cpus); + if (ret) + goto out_unlock; + + requested_count = mk_cpu_set_count(requested); if (requested_count == 0) { pr_info("No CPUs requested for instance %d (%s)\n", instance->id, instance->name); - return 0; + ret = 0; + goto out_unlock; } - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, requested) { if (!mk_cpu_set_contains(mk_pool->cpus, phys_cpu)) { pr_err("CPU %llu not available in the pool\n", phys_cpu); @@ -485,18 +611,22 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, unavailable++; } } + if (mk_instance_irq_route_load(instance) == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, + mk_cpu_set_first(instance->cpus)); if (unavailable > 0) { pr_err("Instance %d (%s): %d CPUs are not available\n", - instance->id, instance->name, unavailable); - return -EBUSY; + instance->id, instance->name, unavailable); + ret = -EBUSY; + goto out_unlock; } ret = mk_cpu_set_reserve(instance->cpus, requested_count); if (ret) - return ret; + goto out_unlock; - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, requested) { mk_cpu_set_del(mk_pool->cpus, phys_cpu); mk_cpu_set_add(instance->cpus, phys_cpu); } @@ -505,7 +635,13 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, pr_info("Transferred %u CPUs from pool to instance %d (%s): %s\n", requested_count, instance->id, instance->name, buf); - return 0; + ret = 0; +out_unlock: + mk_cpu_ownership_unlock(); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(requested); + return ret; } /** @@ -521,6 +657,7 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, int mk_instance_return_cpus(struct mk_instance *instance, const struct mk_cpu_set *cpus) { + struct mk_cpu_set *requested; unsigned int i, requested_count; mk_phys_cpu_t phys_cpu; int not_found = 0; @@ -532,15 +669,27 @@ int mk_instance_return_cpus(struct mk_instance *instance, return -EINVAL; } - requested_count = mk_cpu_set_count(cpus); + requested = mk_cpu_set_alloc(); + if (!requested) + return -ENOMEM; + + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(requested, cpus); + if (ret) + goto out_unlock; + + requested_count = mk_cpu_set_count(requested); if (requested_count == 0) { pr_info("No CPUs requested to return from instance %d (%s)\n", instance->id, instance->name); - return 0; + ret = 0; + goto out_unlock; } /* Validate all CPUs are assigned to this instance */ - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, requested) { if (!mk_cpu_set_contains(instance->cpus, phys_cpu)) { pr_err("CPU %llu not assigned to instance %d (%s)\n", phys_cpu, instance->id, instance->name); @@ -550,23 +699,23 @@ int mk_instance_return_cpus(struct mk_instance *instance, if (not_found > 0) { pr_err("Instance %d (%s): %d CPUs are not assigned to this instance\n", - instance->id, instance->name, not_found); - return -EINVAL; + instance->id, instance->name, not_found); + ret = -EINVAL; + goto out_unlock; } ret = mk_cpu_set_reserve(mk_pool->cpus, requested_count); if (ret) - return ret; + goto out_unlock; - mk_cpu_set_format(buf, sizeof(buf), cpus); + mk_cpu_set_format(buf, sizeof(buf), requested); + mk_cpu_ownership_unlock(); + ret = __mk_instance_migrate_irq_route(instance, requested); + if (ret) + goto out_route; + mk_cpu_ownership_lock(); - /* - * @cpus may alias instance->cpus (returning everything on - * teardown), so walk it back-to-front: a deletion then never - * shifts entries the walk has yet to visit. - */ - for (i = requested_count; i-- > 0; ) { - phys_cpu = cpus->ids[i]; + mk_cpu_set_for_each(i, phys_cpu, requested) { mk_cpu_set_add(mk_pool->cpus, phys_cpu); mk_cpu_set_del(instance->cpus, phys_cpu); } @@ -574,11 +723,18 @@ int mk_instance_return_cpus(struct mk_instance *instance, pr_info("Returned %u CPUs from instance %d (%s) to the pool: %s\n", requested_count, instance->id, instance->name, buf); - return 0; + ret = 0; +out_unlock: + mk_cpu_ownership_unlock(); +out_route: + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(requested); + return ret; } /** - * mk_pool_cpus_returned() - Is every pool CPU back in this kernel? + * mk_pool_cpus_returned_locked() - Is every pool CPU back in this kernel? * * True when the pool holds no free CPU and no instance owns one, so * nothing can be sitting in a park loop. Pool memory that parked CPUs @@ -587,15 +743,17 @@ int mk_instance_return_cpus(struct mk_instance *instance, * instance's set before it parks, which is why the move paths reserve * room in the destination set up front. */ -bool mk_pool_cpus_returned(void) +static bool mk_pool_cpus_returned_locked(void) { struct mk_instance *instance; bool returned = true; + lockdep_assert_held(&mk_instance_mutex); + lockdep_assert_held(&mk_cpu_transaction_mutex); + if (mk_pool && !mk_cpu_set_empty(mk_pool->cpus)) return false; - mutex_lock(&mk_instance_mutex); list_for_each_entry(instance, &mk_instance_list, list) { if (instance == mk_self) continue; @@ -606,18 +764,42 @@ bool mk_pool_cpus_returned(void) break; } } - mutex_unlock(&mk_instance_mutex); return returned; } +/** + * mk_pool_park_teardown() - Safely return the architecture park area + * + * Freeze instance membership and CPU ownership while checking that every + * CPU is home, and keep both frozen until the architecture has returned the + * executable park area to ordinary pool memory. + * + * Returns 0 on success, or -EBUSY while a pool CPU remains assigned or parked. + */ +int mk_pool_park_teardown(void) +{ + int ret; + + mutex_lock(&mk_instance_mutex); + mk_cpu_transaction_lock(); + if (!mk_pool_cpus_returned_locked()) + ret = -EBUSY; + else + ret = mk_arch_pool_park_teardown(); + mk_cpu_transaction_unlock(); + mutex_unlock(&mk_instance_mutex); + + return ret; +} + static int mk_instance_reserve_cpus(struct mk_instance *instance, const struct mk_dt_config *config) { if (!config->cpus) { - pr_warn("No CPU configuration for instance %d (%s)\n", - instance->id, instance->name); - return 0; + pr_err("No CPU configuration for instance %d (%s)\n", + instance->id, instance->name); + return -EINVAL; } return mk_instance_transfer_cpus(instance, config->cpus); @@ -627,11 +809,6 @@ static int mk_instance_transfer_pci_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count) { - struct mk_pci_device *req_dev, *self_dev, *tmp; - int transferred = 0; - int not_found = 0; - bool found; - if (!mk_self || !mk_self->pci_devices_valid) { pr_err("No self instance or PCI devices not initialized\n"); return -EINVAL; @@ -644,102 +821,34 @@ static int mk_instance_transfer_pci_devices(struct mk_instance *instance, return 0; } - /* A spawn lends out of its tree */ - if (mk_manifest_phys()) { - list_for_each_entry(req_dev, requested_devices, list) { - u8 devfn = PCI_DEVFN(req_dev->slot, req_dev->func); - - if (!mk_of_pci_available(req_dev->domain, req_dev->bus, devfn)) { - pr_err("PCI device %04x:%02x:%02x.%x is not this kernel's to lend\n", - req_dev->domain, req_dev->bus, req_dev->slot, - req_dev->func); - not_found++; - } - } - if (not_found) - return -ENOENT; - list_for_each_entry(req_dev, requested_devices, list) { - int ret = mk_of_pci_lend(instance, req_dev->domain, req_dev->bus, - PCI_DEVFN(req_dev->slot, req_dev->func)); - - if (ret) - return ret; - transferred++; - } - instance->pci_devices_valid = true; - pr_info("Lent %d PCI devices to instance %d (%s)\n", - transferred, instance->id, instance->name); - return 0; - } - - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(self_dev, &mk_self->pci_devices, list) { - if (self_dev->vendor == req_dev->vendor && - self_dev->device == req_dev->device && - self_dev->domain == req_dev->domain && - self_dev->bus == req_dev->bus && - self_dev->slot == req_dev->slot && - self_dev->func == req_dev->func) { - found = true; - break; - } - } - if (!found) { - pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x not available in root pool\n", - req_dev->vendor, req_dev->device, req_dev->domain, - req_dev->bus, req_dev->slot, req_dev->func); - not_found++; - } - } - - if (not_found > 0) { - pr_err("Instance %d (%s): %d PCI devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; - } - - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(self_dev, tmp, &mk_self->pci_devices, list) { - if (self_dev->vendor == req_dev->vendor && - self_dev->device == req_dev->device && - self_dev->domain == req_dev->domain && - self_dev->bus == req_dev->bus && - self_dev->slot == req_dev->slot && - self_dev->func == req_dev->func) { - - list_del(&self_dev->list); - list_add_tail(&self_dev->list, &instance->pci_devices); - mk_self->pci_device_count--; - instance->pci_device_count++; - transferred++; - - pr_debug("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - self_dev->vendor, self_dev->device, self_dev->domain, - self_dev->bus, self_dev->slot, self_dev->func, - instance->id); - break; - } - } - } - - instance->pci_devices_valid = true; - pr_info("Transferred %d PCI devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, mk_self->pci_device_count); + /* Nested kernels cannot establish the host-owned VF lifecycle. */ + if (mk_manifest_phys()) + return -EOPNOTSUPP; - return 0; + return mk_pci_assign_devices(instance, requested_devices, + requested_count); } static int mk_instance_reserve_pci_devices(struct mk_instance *instance, const struct mk_dt_config *config) { - if (!config->pci_devices_valid || config->pci_device_count == 0) { + if (!config->pci_devices_valid) { + if (config->pci_device_count || !list_empty(&config->pci_devices)) + return -EINVAL; + instance->pci_devices_valid = true; + return 0; + } + if (!config->pci_device_count) { + if (!list_empty(&config->pci_devices)) + return -EINVAL; instance->pci_devices_valid = true; instance->pci_device_count = 0; pr_debug("No PCI devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->pci_devices)) + return -EINVAL; return mk_instance_transfer_pci_devices(instance, &config->pci_devices, @@ -750,77 +859,87 @@ static int mk_instance_transfer_platform_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count) { - struct mk_platform_device *req_dev, *self_dev, *tmp; + struct mk_platform_device *requested, *other, *self_device; + int actual_count = 0; int transferred = 0; - int not_found = 0; - bool found; if (!mk_self || !mk_self->platform_devices_valid) { pr_err("No self instance or platform devices not initialized\n"); return -EINVAL; } - if (requested_count == 0 || list_empty(requested_devices)) { - pr_info("No platform devices requested for instance %d (%s)\n", - instance->id, instance->name); - instance->platform_devices_valid = true; - return 0; - } + if (requested_count <= 0 || list_empty(requested_devices)) + return -EINVAL; - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(self_dev, &mk_self->platform_devices, list) { - if (strcmp(self_dev->name, req_dev->name) == 0) { - found = true; + list_for_each_entry(requested, requested_devices, list) { + actual_count++; + list_for_each_entry(other, requested_devices, list) { + if (other == requested) break; - } + if (!strcmp(other->name, requested->name)) + return -EINVAL; } - if (!found) { - pr_err("Platform device '%s' not available in root pool\n", - req_dev->name); - not_found++; + self_device = NULL; + list_for_each_entry(other, &mk_self->platform_devices, list) { + if (!strcmp(other->name, requested->name)) { + self_device = other; + break; + } } + if (!self_device) + return -ENOENT; } + if (actual_count != requested_count) + return -EINVAL; - if (not_found > 0) { - pr_err("Instance %d (%s): %d platform devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; - } - - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(self_dev, tmp, &mk_self->platform_devices, list) { - if (strcmp(self_dev->name, req_dev->name) == 0) { - list_del(&self_dev->list); - list_add_tail(&self_dev->list, &instance->platform_devices); - mk_self->platform_device_count--; - instance->platform_device_count++; - transferred++; - - pr_debug("Transferred platform device '%s' to instance %d\n", - self_dev->name, instance->id); + list_for_each_entry(requested, requested_devices, list) { + self_device = NULL; + list_for_each_entry(other, &mk_self->platform_devices, list) { + if (!strcmp(other->name, requested->name)) { + self_device = other; break; } } + if (!self_device) + goto rollback; + list_move_tail(&self_device->list, &instance->platform_devices); + mk_self->platform_device_count--; + instance->platform_device_count++; + transferred++; } instance->platform_devices_valid = true; - pr_info("Transferred %d platform devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, mk_self->platform_device_count); + pr_info("Transferred %d platform devices from self to instance %d (%s)\n", + transferred, instance->id, instance->name); return 0; + +rollback: + mk_instance_return_platform_devices(instance); + return -EIO; } static int mk_instance_reserve_platform_devices(struct mk_instance *instance, const struct mk_dt_config *config) { - if (!config->platform_devices_valid || config->platform_device_count == 0) { + if (!config->platform_devices_valid) { + if (config->platform_device_count || + !list_empty(&config->platform_devices)) + return -EINVAL; + instance->platform_devices_valid = true; + return 0; + } + if (!config->platform_device_count) { + if (!list_empty(&config->platform_devices)) + return -EINVAL; instance->platform_devices_valid = true; instance->platform_device_count = 0; pr_debug("No platform devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->platform_devices)) + return -EINVAL; return mk_instance_transfer_platform_devices(instance, &config->platform_devices, @@ -842,49 +961,9 @@ static int mk_instance_reserve_platform_devices(struct mk_instance *instance, int mk_instance_add_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *self_dev, *tmp; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!mk_self || !mk_self->pci_devices_valid) { - pr_err("No self instance or PCI devices not initialized\n"); - return -EINVAL; - } - - if (mk_manifest_phys()) { - int ret = mk_of_pci_lend(instance, domain, bus, devfn); - - if (ret) - pr_err("PCI device %04x:%02x:%02x.%x is not this kernel's to lend\n", - domain, bus, slot, func); - else - pr_info("Lent PCI device %04x:%02x:%02x.%x to instance %d\n", - domain, bus, slot, func, instance->id); - return ret; - } - - list_for_each_entry_safe(self_dev, tmp, &mk_self->pci_devices, list) { - if (self_dev->domain == domain && - self_dev->bus == bus && - self_dev->slot == slot && - self_dev->func == func) { - - list_del(&self_dev->list); - list_add_tail(&self_dev->list, &instance->pci_devices); - mk_self->pci_device_count--; - instance->pci_device_count++; - instance->pci_devices_valid = true; - - pr_info("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - self_dev->vendor, self_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } - - pr_err("PCI device %04x:%02x:%02x.%x not found in root pool\n", - domain, bus, slot, func); - return -ENOENT; + if (mk_manifest_phys()) + return -EOPNOTSUPP; + return mk_pci_assign_device(instance, domain, bus, devfn); } /** @@ -902,62 +981,9 @@ int mk_instance_add_pci_device(struct mk_instance *instance, int mk_instance_remove_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *inst_dev, *tmp; - struct mk_pci_device *self_dev; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!instance->pci_devices_valid) { - pr_err("Instance %d PCI devices not initialized\n", instance->id); - return -EINVAL; - } - - if (!mk_self) { - pr_err("Cannot return PCI device: no self instance\n"); - return -EINVAL; - } - - list_for_each_entry_safe(inst_dev, tmp, &instance->pci_devices, list) { - if (inst_dev->domain == domain && - inst_dev->bus == bus && - inst_dev->slot == slot && - inst_dev->func == func) { - - if (mk_manifest_phys()) { - mk_of_pci_take_back(domain, bus, devfn); - list_del(&inst_dev->list); - kfree(inst_dev); - instance->pci_device_count--; - pr_info("Took PCI device %04x:%02x:%02x.%x back from instance %d\n", - domain, bus, slot, func, instance->id); - return 0; - } - - self_dev = kzalloc_obj(*self_dev, GFP_KERNEL); - if (!self_dev) - return -ENOMEM; - - *self_dev = *inst_dev; - INIT_LIST_HEAD(&self_dev->list); - - list_add_tail(&self_dev->list, &mk_self->pci_devices); - mk_self->pci_device_count++; - mk_self->pci_devices_valid = true; - - list_del(&inst_dev->list); - kfree(inst_dev); - instance->pci_device_count--; - - pr_info("Returned PCI device %04x:%04x@%04x:%02x:%02x.%x from instance %d to root\n", - self_dev->vendor, self_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } - - pr_err("PCI device %04x:%02x:%02x.%x not found in instance %d\n", - domain, bus, slot, func, instance->id); - return -ENOENT; + if (mk_manifest_phys()) + return -EOPNOTSUPP; + return mk_pci_unassign_device(instance, domain, bus, devfn); } /** @@ -1272,8 +1298,20 @@ void mk_instance_free_memory(struct mk_instance *instance) instance->id, instance->name); } +static bool mk_instance_resources_empty(const struct mk_instance *instance) +{ + return list_empty(&instance->memory_regions) && + !instance->instance_pool && !instance->region_count && + mk_cpu_set_empty(instance->cpus) && + list_empty(&instance->pci_devices) && + list_empty(&instance->pci_assignments) && + !instance->pci_device_count && + list_empty(&instance->platform_devices) && + !instance->platform_device_count; +} + /** - * mk_instance_reserve_resources() - Reserve memory and CPU resources for an instance + * mk_instance_reserve_resources() - Atomically reserve instance resources * @instance: Instance to reserve resources for * @config: Device tree configuration with memory regions and CPU assignment * @@ -1285,52 +1323,56 @@ void mk_instance_free_memory(struct mk_instance *instance) int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config) { + const char *failed_resource; + int release_ret; int ret; - if (!config || !instance) { + if (!config || !instance || !instance->cpus) { pr_err("Invalid parameters to mk_instance_reserve_resources\n"); return -EINVAL; } - - /* Free any existing memory regions first */ - mk_instance_free_memory(instance); + if (!mk_instance_resources_empty(instance)) + return -EBUSY; /* Reserve memory regions */ + failed_resource = "memory"; ret = mk_instance_reserve_memory(instance, config); - if (ret) { - pr_err("Failed to reserve memory regions for instance %d (%s): %d\n", - instance->id, instance->name, ret); - return ret; - } + if (ret) + goto rollback; /* Reserve CPU resources */ + failed_resource = "CPU"; ret = mk_instance_reserve_cpus(instance, config); - if (ret) { - pr_err("Failed to reserve CPU resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for CPU reservation failure */ - pr_warn("Continuing without CPU assignment\n"); - } - - /* Reserve PCI device resources */ - ret = mk_instance_reserve_pci_devices(instance, config); - if (ret) { - pr_err("Failed to reserve PCI device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for PCI reservation failure */ - pr_warn("Continuing without PCI device assignment\n"); - } + if (ret) + goto rollback; /* Reserve platform device resources */ + failed_resource = "platform device"; ret = mk_instance_reserve_platform_devices(instance, config); - if (ret) { - pr_err("Failed to reserve platform device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for platform reservation failure */ - pr_warn("Continuing without platform device assignment\n"); - } + if (ret) + goto rollback; + + /* Commit fallible exclusive VF leases last. */ + failed_resource = "PCI device"; + ret = mk_instance_reserve_pci_devices(instance, config); + if (ret) + goto rollback; return 0; + +rollback: + pr_err("Failed to reserve %s resources for instance %d (%s): %d\n", + failed_resource, instance->id, instance->name, ret); + release_ret = mk_instance_release_resources(instance); + if (release_ret) { + mk_instance_set_state(instance, MK_STATE_FAILED); + return release_ret; + } + if (WARN_ON_ONCE(!mk_instance_resources_empty(instance))) { + mk_instance_set_state(instance, MK_STATE_FAILED); + return -EIO; + } + return ret; } /** @@ -1499,10 +1541,12 @@ void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size) struct mk_shutdown_work { struct work_struct work; - u32 flags; int sender_instance_id; }; +static struct mk_shutdown_work mk_shutdown_work; +static atomic_t mk_shutdown_work_pending = ATOMIC_INIT(0); + /* * Notify @target_id that this kernel is going down, while messaging @@ -1542,15 +1586,20 @@ static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) */ void __noreturn mk_halt_to_pool(void) { - mk_notify_down_and_park(0, MK_SYS_HALTED); + int parent_id; + + if (!host_instance) + panic("multikernel: spawned kernel has no parent instance"); + parent_id = READ_ONCE(host_instance->id); + mk_notify_down_and_park(parent_id, MK_SYS_HALTED); } static void mk_shutdown_work_fn(struct work_struct *work) { - struct mk_shutdown_work *sw = container_of(work, struct mk_shutdown_work, work); + struct mk_shutdown_work *sw = + container_of(work, struct mk_shutdown_work, work); int sender_instance_id = sw->sender_instance_id; - kfree(sw); mk_notify_down_and_park(sender_instance_id, MK_SYS_SHUTDOWN_ACK); } @@ -1561,18 +1610,72 @@ static void mk_shutdown_work_fn(struct work_struct *work) * it corrupts the single-producer mailbox. The kexec path confirms the * CPUs are parked before it rewrites the image. */ -static void mk_instance_settle_halted(struct mk_instance *instance) +static int mk_instance_settle_halted(struct mk_instance *instance) { + int ret; + pr_info("Instance %d (%s) halted, CPUs parking in pool\n", instance->id, instance->name); - mk_instance_set_state(instance, MK_STATE_LOADED); + ret = mk_instance_confirm_parked(instance); + if (ret) + return ret; + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + ret = mk_pci_quiesce_instance_irqs(instance, true); + mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); + mk_instance_set_state(instance, ret ? MK_STATE_FAILED : MK_STATE_LOADED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + return ret; } struct mk_halted_work { struct work_struct work; + unsigned int pool_slot; int instance_id; }; +static struct mk_halted_work *mk_halted_work_pool; +static unsigned long *mk_halted_work_pool_busy; +static unsigned int mk_halted_work_pool_size; +static atomic_t mk_halted_work_pool_cursor = ATOMIC_INIT(0); +static struct workqueue_struct *mk_system_workqueue; +static atomic64_t mk_system_work_dropped = ATOMIC64_INIT(0); +static DEFINE_RATELIMIT_STATE(mk_system_work_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + +static void mk_system_work_drop(void) +{ + atomic64_inc(&mk_system_work_dropped); + if (__ratelimit(&mk_system_work_rs)) + printk_deferred("WARNING: Multikernel system work request dropped\n"); +} + +static struct mk_halted_work *mk_halted_work_get(void) +{ + unsigned int start; + unsigned int slot; + unsigned int i; + + start = (unsigned int)atomic_inc_return(&mk_halted_work_pool_cursor); + for (i = 0; i < mk_halted_work_pool_size; i++) { + slot = (start + i) % mk_halted_work_pool_size; + if (!test_and_set_bit_lock(slot, mk_halted_work_pool_busy)) + return &mk_halted_work_pool[slot]; + } + + mk_system_work_drop(); + return NULL; +} + +static void mk_halted_work_put(struct mk_halted_work *aw) +{ + clear_bit_unlock(aw->pool_slot, mk_halted_work_pool_busy); +} + static void mk_halted_work_fn(struct work_struct *work) { struct mk_halted_work *aw = @@ -1581,40 +1684,40 @@ static void mk_halted_work_fn(struct work_struct *work) instance = mk_instance_find(aw->instance_id); if (instance) { - mk_instance_settle_halted(instance); + if (mk_instance_settle_halted(instance)) + pr_err("Instance %d halted but could not be made reusable\n", + instance->id); mk_instance_put(instance); } else { pr_warn("Shutdown ACK from unknown instance %d\n", aw->instance_id); } - kfree(aw); + mk_halted_work_put(aw); } static void mk_system_msg_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx) { if (msg_type != MK_MSG_SYSTEM) return; switch (subtype) { case MK_SYS_SHUTDOWN: { - struct mk_shutdown_payload *req = payload; - struct mk_shutdown_work *sw; - - if (payload_len < sizeof(*req)) + if (payload_len < sizeof(struct mk_shutdown_payload)) return; - pr_info("Shutdown requested by instance %d\n", req->sender_instance_id); + pr_info("Shutdown requested by instance %d\n", sender_instance_id); - sw = kmalloc(sizeof(*sw), GFP_ATOMIC); - if (!sw) + if (atomic_cmpxchg(&mk_shutdown_work_pending, 0, 1)) { + mk_system_work_drop(); return; + } - INIT_WORK(&sw->work, mk_shutdown_work_fn); - sw->flags = req->flags; - sw->sender_instance_id = req->sender_instance_id; - schedule_work(&sw->work); + mk_shutdown_work.sender_instance_id = sender_instance_id; + if (!queue_work(mk_system_workqueue, &mk_shutdown_work.work)) + mk_system_work_drop(); break; } case MK_SYS_SHUTDOWN_ACK: { @@ -1622,6 +1725,8 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, if (payload_len < sizeof(*ack)) return; + if (ack->resource_id != sender_instance_id) + return; /* * Reply to a shutdown this kernel requested: wake the * requester, which waits for the instance's CPUs to park @@ -1637,6 +1742,8 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, if (payload_len < sizeof(*ack)) return; + if (ack->resource_id != sender_instance_id) + return; /* * The instance halted itself; nobody is waiting on it, so @@ -1645,13 +1752,15 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, * need time to reach the park loop, while this runs in IPI * context. */ - aw = kmalloc(sizeof(*aw), GFP_ATOMIC); + aw = mk_halted_work_get(); if (!aw) break; - INIT_WORK(&aw->work, mk_halted_work_fn); - aw->instance_id = ack->resource_id; - schedule_work(&aw->work); + aw->instance_id = sender_instance_id; + if (!queue_work(mk_system_workqueue, &aw->work)) { + mk_halted_work_put(aw); + mk_system_work_drop(); + } break; } default: @@ -1705,19 +1814,16 @@ int multikernel_halt_by_id(int mk_id) ret = mk_msg_pending_wait(pending, 30000); if (ret == 0) { - if (mk_instance_confirm_parked(instance)) - pr_warn("Multikernel instance %d halted with CPUs unaccounted for\n", - mk_id); - - mk_instance_set_state(instance, MK_STATE_LOADED); - pr_info("Multikernel instance %d halted (graceful)\n", mk_id); + ret = mk_instance_settle_halted(instance); + if (!ret) + pr_info("Multikernel instance %d halted (graceful)\n", mk_id); } mk_instance_put(instance); return ret; } -/** +/* * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI * @mk_id: Instance ID to halt * @@ -1788,7 +1894,7 @@ EXPORT_SYMBOL_GPL(mk_cpu_rank); static int mk_fence_missing(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing = 0; @@ -1813,7 +1919,7 @@ static int mk_fence_missing(struct mk_instance *instance, static int mk_confirm_fenced(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing, ret; @@ -1847,22 +1953,20 @@ static int mk_confirm_fenced(struct mk_instance *instance, return ret; } -int multikernel_force_halt_by_id(int mk_id) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { - struct mk_instance *instance; struct mk_cpu_set *targets; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; + return -EINVAL; if (instance == mk_self) { - pr_err("Cannot force halt this kernel (id %d)\n", mk_id); - mk_instance_put(instance); + pr_err("Cannot force halt this kernel (id %d)\n", instance->id); return -EINVAL; } @@ -1873,29 +1977,27 @@ int multikernel_force_halt_by_id(int mk_id) * a rerun the instance is stuck for good. The parent is ACTIVE. */ if (instance->state != MK_STATE_ACTIVE && - instance->state != MK_STATE_LOADED) { + (!allow_loaded || instance->state != MK_STATE_LOADED)) { pr_err("Instance %d not running (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } targets = mk_cpu_set_alloc(); - if (!targets) { - mk_instance_put(instance); + if (!targets) return -ENOMEM; - } ret = mk_force_halt_targets(instance, targets); if (!ret && mk_cpu_set_empty(targets)) ret = -EINVAL; if (ret) { - pr_err("Instance %d: no force-halt targets: %d\n", mk_id, ret); + pr_err("Instance %d: no force-halt targets: %d\n", + instance->id, ret); mk_cpu_set_free(targets); - mk_instance_put(instance); return ret; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + pr_info("Force halting multikernel instance %d via NMI\n", + instance->id); ret = mk_arm_force_halt(instance); if (ret) @@ -1906,7 +2008,8 @@ int multikernel_force_halt_by_id(int mk_id) cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); /* * A child instance parks on its own context, so wait for it to @@ -1918,30 +2021,151 @@ int multikernel_force_halt_by_id(int mk_id) * fence, so a baseline cannot claim a machine with a CPU still * running the dead host. */ - if (instance == host_instance) + if (instance == host_instance) { ret = mk_confirm_fenced(instance, targets); - else - mk_instance_settle_halted(instance); + } else { + ret = mk_instance_confirm_parked(instance); + if (ret) + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + else + ret = mk_instance_settle_halted(instance); + } mk_cpu_set_free(targets); + return ret; +} + +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + mk_ipi_endpoint_close(instance); + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_mark_failed(instance); + return ret; +} + +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by arming the persistent force-halt + * marker and sending NMIs directly to each CPU. The NMI handler checks the + * marker and parks the CPU if it is set. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); mk_instance_put(instance); return ret; } +static int __init mk_system_work_init(void) +{ + unsigned int pool_size; + unsigned int i; + + INIT_WORK(&mk_shutdown_work.work, mk_shutdown_work_fn); + atomic_set(&mk_shutdown_work_pending, 0); + + /* Allow every possible CPU to have all reply slots in flight. */ + pool_size = max_t(unsigned int, num_possible_cpus(), 1) * + MK_REPLY_SLOTS; + mk_halted_work_pool = kcalloc(pool_size, sizeof(*mk_halted_work_pool), + GFP_KERNEL); + if (!mk_halted_work_pool) + return -ENOMEM; + + mk_halted_work_pool_busy = bitmap_zalloc(pool_size, GFP_KERNEL); + if (!mk_halted_work_pool_busy) + goto free_pool; + mk_halted_work_pool_size = pool_size; + atomic_set(&mk_halted_work_pool_cursor, 0); + for (i = 0; i < pool_size; i++) { + INIT_WORK(&mk_halted_work_pool[i].work, mk_halted_work_fn); + mk_halted_work_pool[i].pool_slot = i; + } + + mk_system_workqueue = + alloc_workqueue("mk-system", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!mk_system_workqueue) + goto free_pool_bitmap; + + return 0; + +free_pool_bitmap: + bitmap_free(mk_halted_work_pool_busy); + mk_halted_work_pool_busy = NULL; + mk_halted_work_pool_size = 0; +free_pool: + kfree(mk_halted_work_pool); + mk_halted_work_pool = NULL; + return -ENOMEM; +} + +static void mk_system_work_cleanup(void) +{ + if (mk_system_workqueue) { + destroy_workqueue(mk_system_workqueue); + mk_system_workqueue = NULL; + } + WARN_ON_ONCE(mk_halted_work_pool_busy && + !bitmap_empty(mk_halted_work_pool_busy, + mk_halted_work_pool_size)); + bitmap_free(mk_halted_work_pool_busy); + mk_halted_work_pool_busy = NULL; + mk_halted_work_pool_size = 0; + kfree(mk_halted_work_pool); + mk_halted_work_pool = NULL; +} + static int __init multikernel_init(void) { int ret; + ret = mk_pci_lease_system_init(); + if (ret) + return ret; + ret = mk_messaging_init(); if (ret < 0) { pr_err("Failed to initialize multikernel messaging: %d\n", ret); + mk_pci_lease_system_cleanup(); + return ret; + } + + ret = mk_system_work_init(); + if (ret < 0) { + pr_err("Failed to initialize multikernel system work: %d\n", ret); + mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } ret = mk_register_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler, NULL); if (ret < 0) { pr_err("Failed to register system message handler: %d\n", ret); + mk_system_work_cleanup(); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -1949,7 +2173,9 @@ static int __init multikernel_init(void) if (ret < 0) { pr_err("Failed to initialize multikernel hotplug: %d\n", ret); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + mk_system_work_cleanup(); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -1958,10 +2184,14 @@ static int __init multikernel_init(void) pr_err("Failed to initialize multikernel sysfs interface: %d\n", ret); mk_hotplug_cleanup(); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + mk_system_work_cleanup(); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } + mk_ipi_handlers_enable(); + pr_info("Multikernel support initialized\n"); return 0; } diff --git a/kernel/multikernel/cpuset.c b/kernel/multikernel/cpuset.c index ad36a4f94fc65a..f3a1a918fe4573 100644 --- a/kernel/multikernel/cpuset.c +++ b/kernel/multikernel/cpuset.c @@ -13,31 +13,22 @@ #include #include -struct mk_cpu_set *mk_cpu_set_alloc(void) -{ - return kzalloc(sizeof(struct mk_cpu_set), GFP_KERNEL); -} - -void mk_cpu_set_free(struct mk_cpu_set *set) +static void mk_cpu_set_lock(const struct mk_cpu_set *set, unsigned long *flags) { - if (!set) - return; - - kfree(set->ids); - kfree(set); + raw_spin_lock_irqsave((raw_spinlock_t *)&set->lock, *flags); } -void mk_cpu_set_clear(struct mk_cpu_set *set) +static void mk_cpu_set_unlock(const struct mk_cpu_set *set, unsigned long flags) { - if (set) - set->nr = 0; + raw_spin_unlock_irqrestore((raw_spinlock_t *)&set->lock, flags); } -static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) +static int mk_cpu_set_index_locked(const struct mk_cpu_set *set, + mk_phys_cpu_t id) { unsigned int i; - for (i = 0; set && i < set->nr; i++) { + for (i = 0; i < set->nr; i++) { if (set->ids[i] == id) return i; } @@ -45,9 +36,35 @@ static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) return -1; } -bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +struct mk_cpu_set *mk_cpu_set_alloc(void) +{ + struct mk_cpu_set *set; + + set = kzalloc_obj(*set, GFP_KERNEL); + if (set) + raw_spin_lock_init(&set->lock); + return set; +} + +void mk_cpu_set_free(struct mk_cpu_set *set) +{ + if (!set) + return; + + kfree(set->ids); + kfree(set); +} + +void mk_cpu_set_clear(struct mk_cpu_set *set) { - return mk_cpu_set_index(set, id) >= 0; + unsigned long flags; + + if (!set) + return; + + mk_cpu_set_lock(set, &flags); + set->nr = 0; + mk_cpu_set_unlock(set, flags); } /** @@ -61,64 +78,209 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) */ int mk_cpu_set_reserve(struct mk_cpu_set *set, unsigned int extra) { - unsigned int cap = set->nr + extra; - mk_phys_cpu_t *ids; + mk_phys_cpu_t *ids = NULL; + mk_phys_cpu_t *old_ids; + unsigned int cap; + unsigned long flags; - if (cap <= set->cap) - return 0; + if (!set) + return -EINVAL; + + for (;;) { + mk_cpu_set_lock(set, &flags); + cap = set->nr + extra; + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + mk_cpu_set_unlock(set, flags); - cap = max_t(unsigned int, cap, 8); - ids = krealloc_array(set->ids, cap, sizeof(*ids), GFP_KERNEL); - if (!ids) - return -ENOMEM; + cap = max_t(unsigned int, cap, 8); + kfree(ids); + ids = kcalloc(cap, sizeof(*ids), GFP_KERNEL); + if (!ids) + return -ENOMEM; - set->ids = ids; - set->cap = cap; - return 0; + mk_cpu_set_lock(set, &flags); + if (set->nr + extra > cap) { + mk_cpu_set_unlock(set, flags); + continue; + } + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + + memcpy(ids, set->ids, set->nr * sizeof(*ids)); + old_ids = set->ids; + set->ids = ids; + set->cap = cap; + mk_cpu_set_unlock(set, flags); + kfree(old_ids); + return 0; + } } /* Idempotent: adding an ID already in the set succeeds without effect */ int mk_cpu_set_add(struct mk_cpu_set *set, mk_phys_cpu_t id) { + unsigned long flags; int ret; - if (mk_cpu_set_contains(set, id)) - return 0; + if (!set) + return -EINVAL; - ret = mk_cpu_set_reserve(set, 1); - if (ret) - return ret; + for (;;) { + mk_cpu_set_lock(set, &flags); + if (mk_cpu_set_index_locked(set, id) >= 0) { + mk_cpu_set_unlock(set, flags); + return 0; + } + if (set->nr < set->cap) { + set->ids[set->nr++] = id; + mk_cpu_set_unlock(set, flags); + return 0; + } + mk_cpu_set_unlock(set, flags); - set->ids[set->nr++] = id; - return 0; + ret = mk_cpu_set_reserve(set, 1); + if (ret) + return ret; + } } bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id) { - int idx = mk_cpu_set_index(set, id); + unsigned long flags; + int idx; + + if (!set) + return false; - if (idx < 0) + mk_cpu_set_lock(set, &flags); + idx = mk_cpu_set_index_locked(set, id); + if (idx < 0) { + mk_cpu_set_unlock(set, flags); return false; + } memmove(&set->ids[idx], &set->ids[idx + 1], (set->nr - idx - 1) * sizeof(set->ids[0])); set->nr--; + mk_cpu_set_unlock(set, flags); return true; } +bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +{ + unsigned long flags; + bool found; + + if (!set) + return false; + + mk_cpu_set_lock(set, &flags); + found = mk_cpu_set_index_locked(set, id) >= 0; + mk_cpu_set_unlock(set, flags); + return found; +} + +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) +{ + unsigned long flags; + unsigned int nr; + + if (!set) + return 0; + + mk_cpu_set_lock(set, &flags); + nr = set->nr; + mk_cpu_set_unlock(set, flags); + return nr; +} + +bool mk_cpu_set_empty(const struct mk_cpu_set *set) +{ + return mk_cpu_set_count(set) == 0; +} + +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) +{ + unsigned long flags; + mk_phys_cpu_t id; + + if (!set) + return MK_PHYS_CPU_INVALID; + + mk_cpu_set_lock(set, &flags); + id = set->nr ? set->ids[0] : MK_PHYS_CPU_INVALID; + mk_cpu_set_unlock(set, flags); + return id; +} + +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id) +{ + unsigned long flags; + bool found = false; + + if (!set || !id) + return false; + + mk_cpu_set_lock(set, &flags); + if (index < set->nr) { + *id = set->ids[index]; + found = true; + } + mk_cpu_set_unlock(set, flags); + return found; +} + int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) { - unsigned int nr = mk_cpu_set_count(src); + unsigned long src_flags; + unsigned long dst_flags; + mk_phys_cpu_t *ids; + unsigned int nr; int ret; - dst->nr = 0; - ret = mk_cpu_set_reserve(dst, nr); - if (ret) - return ret; + if (!dst || !src) + return -EINVAL; + if (dst == src) + return 0; - memcpy(dst->ids, src->ids, nr * sizeof(dst->ids[0])); - dst->nr = nr; - return 0; + for (;;) { + nr = mk_cpu_set_count(src); + ids = nr ? kmalloc_array(nr, sizeof(*ids), GFP_KERNEL) : NULL; + if (nr && !ids) + return -ENOMEM; + + mk_cpu_set_lock(src, &src_flags); + if (src->nr > nr) { + mk_cpu_set_unlock(src, src_flags); + kfree(ids); + continue; + } + nr = src->nr; + if (nr) + memcpy(ids, src->ids, nr * sizeof(*ids)); + mk_cpu_set_unlock(src, src_flags); + + ret = mk_cpu_set_reserve(dst, nr); + if (ret) { + kfree(ids); + return ret; + } + mk_cpu_set_lock(dst, &dst_flags); + if (nr) + memcpy(dst->ids, ids, nr * sizeof(dst->ids[0])); + dst->nr = nr; + mk_cpu_set_unlock(dst, dst_flags); + kfree(ids); + return 0; + } } /** @@ -127,22 +289,31 @@ int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) * @size: Buffer size * @set: Set to format * - * Writes "none" for an empty set, a comma-separated list of physical + * Writes none for an empty set, a comma-separated list of physical * IDs otherwise. Output is truncated to @size. Returns the number of * characters written. */ int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set) { + unsigned long flags; unsigned int i; int len = 0; - if (mk_cpu_set_empty(set)) + if (!buf || !size) + return 0; + if (!set) return scnprintf(buf, size, "none"); - for (i = 0; i < set->nr; i++) { + mk_cpu_set_lock(set, &flags); + if (!set->nr) { + mk_cpu_set_unlock(set, flags); + return scnprintf(buf, size, "none"); + } + + for (i = 0; i < set->nr && len < size; i++) { len += scnprintf(buf + len, size - len, "%s%llu", i ? "," : "", set->ids[i]); } - + mk_cpu_set_unlock(set, flags); return len; } diff --git a/kernel/multikernel/dts.c b/kernel/multikernel/dts.c index a62aae701f8ab9..37bd0f7c6c3052 100644 --- a/kernel/multikernel/dts.c +++ b/kernel/multikernel/dts.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #if defined(CONFIG_X86) && defined(CONFIG_PCI_MMCONFIG) #include @@ -28,6 +29,48 @@ #include "internal.h" +static int mk_pci_parse_hex(const char *str, size_t digits, u32 *value) +{ + size_t i; + u32 parsed = 0; + + for (i = 0; i < digits; i++) { + int digit = hex_to_bin(str[i]); + + if (digit < 0) + return -EINVAL; + parsed = (parsed << 4) | digit; + } + + *value = parsed; + return 0; +} + +int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, + u8 *slot, u8 *func) +{ + u32 parsed_domain, parsed_bus, parsed_slot, parsed_func; + + if (len != (int)sizeof("0000:00:00.0") || pci_id[12] != '\0' || + pci_id[4] != ':' || pci_id[7] != ':' || pci_id[10] != '.') + return -EINVAL; + + if (mk_pci_parse_hex(pci_id, 4, &parsed_domain) || + mk_pci_parse_hex(pci_id + 5, 2, &parsed_bus) || + mk_pci_parse_hex(pci_id + 8, 2, &parsed_slot) || + mk_pci_parse_hex(pci_id + 11, 1, &parsed_func)) + return -EINVAL; + if (parsed_domain > U16_MAX || parsed_bus > U8_MAX || + parsed_slot > 31 || parsed_func > 7) + return -ERANGE; + + *domain = (u16)parsed_domain; + *bus = (u8)parsed_bus; + *slot = (u8)parsed_slot; + *func = (u8)parsed_func; + return 0; +} + /** * mk_dt_node_alias() - Find the /aliases entry naming a node * @fdt: Device tree blob @@ -263,9 +306,12 @@ static int mk_dt_add_pci_device(const void *source_fdt, int dev_node, unsigned int func) { const fdt32_t *vendor_prop, *device_prop; + const fdt64_t *resources_prop; + struct mk_pci_device *existing; struct mk_pci_device *pci_dev; const char *node_name, *alias; - int len; + u32 vendor, device; + int len, i; node_name = fdt_get_name(source_fdt, dev_node, NULL); @@ -282,6 +328,16 @@ static int mk_dt_add_pci_device(const void *source_fdt, int dev_node, device_name, node_name ? node_name : ""); return -EINVAL; } + vendor = fdt32_to_cpu(*vendor_prop); + device = fdt32_to_cpu(*device_prop); + if (vendor > U16_MAX || device > U16_MAX || domain > U16_MAX || + bus > U8_MAX || slot > 31 || func > 7) + return -ERANGE; + list_for_each_entry(existing, &config->pci_devices, list) { + if (existing->domain == domain && existing->bus == bus && + existing->slot == slot && existing->func == func) + return -EEXIST; + } pci_dev = kzalloc(sizeof(*pci_dev), GFP_KERNEL); if (!pci_dev) { @@ -289,8 +345,8 @@ static int mk_dt_add_pci_device(const void *source_fdt, int dev_node, return -ENOMEM; } - pci_dev->vendor = (u16)fdt32_to_cpu(*vendor_prop); - pci_dev->device = (u16)fdt32_to_cpu(*device_prop); + pci_dev->vendor = (u16)vendor; + pci_dev->device = (u16)device; pci_dev->domain = (u16)domain; pci_dev->bus = (u8)bus; pci_dev->slot = (u8)slot; @@ -298,6 +354,28 @@ static int mk_dt_add_pci_device(const void *source_fdt, int dev_node, alias = mk_dt_node_alias(source_fdt, dev_node); if (alias) strscpy(pci_dev->alias, alias, sizeof(pci_dev->alias)); + resources_prop = fdt_getprop(source_fdt, dev_node, "bar-resources", &len); + if (resources_prop) { + if (len != MK_PCI_RESOURCE_COUNT * 3 * sizeof(*resources_prop)) { + kfree(pci_dev); + return -EINVAL; + } + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start = fdt64_to_cpu(resources_prop[i * 3]); + u64 end = fdt64_to_cpu(resources_prop[i * 3 + 1]); + u64 flags = fdt64_to_cpu(resources_prop[i * 3 + 2]); + + if ((start || end) && + (end < start || !(flags & (IORESOURCE_IO | IORESOURCE_MEM)))) { + kfree(pci_dev); + return -EINVAL; + } + pci_dev->resources[i].start = start; + pci_dev->resources[i].end = end; + pci_dev->resources[i].flags = flags; + } + pci_dev->resources_valid = true; + } list_add_tail(&pci_dev->list, &config->pci_devices); config->pci_device_count++; @@ -313,9 +391,10 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, struct mk_dt_config *config, const char *device_name) { - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; const char *pci_id_str; - int len; + int len, ret; pci_id_str = fdt_getprop(source_fdt, dev_node, "pci-id", &len); if (!pci_id_str) { @@ -323,11 +402,9 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format: '%s' (expected domain:bus:slot.func)\n", - pci_id_str); - return -EINVAL; - } + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, &slot, &func); + if (ret) + return ret; return mk_dt_add_pci_device(source_fdt, dev_node, config, device_name, domain, bus, slot, func); @@ -1093,6 +1170,44 @@ static struct pci_bus *mk_dt_bus_toward(struct pci_bus *parent, u16 domain, return bus; } +static int mk_dt_emit_pci_resources(void *fdt, + const struct mk_pci_device *device) +{ + fdt64_t resources[MK_PCI_RESOURCE_COUNT * 3]; + struct pci_dev *live_dev = NULL; + int i, ret; + + if (!device->resources_valid) { + live_dev = pci_get_domain_bus_and_slot(device->domain, device->bus, + PCI_DEVFN(device->slot, + device->func)); + if (!live_dev) + return -ENODEV; + } + + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start, end, flags; + + if (live_dev) { + start = pci_resource_start(live_dev, i); + end = pci_resource_end(live_dev, i); + flags = pci_resource_flags(live_dev, i); + } else { + start = device->resources[i].start; + end = device->resources[i].end; + flags = device->resources[i].flags; + } + resources[i * 3] = cpu_to_fdt64(start); + resources[i * 3 + 1] = cpu_to_fdt64(end); + resources[i * 3 + 2] = cpu_to_fdt64(flags); + } + + if (live_dev) + pci_dev_put(live_dev); + ret = fdt_property(fdt, "bar-resources", resources, sizeof(resources)); + return ret; +} + /* * Describe what @instance owns below @bus: its devices on this bus as * leaves, and one bridge node, recursed into, per child bus that leads @@ -1121,6 +1236,8 @@ static int mk_dt_emit_pci_bus(struct mk_instance *instance, ret = fdt_property_u32(fdt, "vendor-id", dev->vendor); if (!ret) ret = fdt_property_u32(fdt, "device-id", dev->device); + if (!ret) + ret = mk_dt_emit_pci_resources(fdt, dev); if (!ret) ret = fdt_end_node(fdt); if (ret) diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 91ff72e0a168d6..19e400b195e5d9 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,8 @@ #include #include #include +#include +#include #include "internal.h" static const char mk_mem_resource_name[] = "System RAM (multikernel)"; @@ -65,37 +68,98 @@ struct mk_hotplug_op { static DEFINE_MUTEX(mk_hotplug_mutex); static LIST_HEAD(mk_hotplug_ops); -/* - * CPU Hotplug Operations - */ -struct mk_cpu_hotplug_work { +struct mk_hotplug_work { struct work_struct work; - mk_phys_cpu_t cpu_id; - u32 numa_node; - u32 flags; - int sender_instance_id; /* For sending ACK back */ - u32 operation; /* MK_RES_CPU_ADD or MK_RES_CPU_REMOVE */ + work_func_t dispatch; + unsigned int pool_slot; + int sender_instance_id; + u32 operation; + union { + struct { + mk_phys_cpu_t cpu_id; + u32 numa_node; + u32 flags; + } cpu; + struct { + u64 start_pfn; + u64 nr_pages; + u32 numa_node; + u32 mem_type; + } mem; + struct { + u16 domain; + u8 bus; + u8 devfn; + u32 flags; + char driver_override[64]; + } device; + }; }; -/* - * Ownership tracking for this kernel's own hotplug: mk_self->cpus - * is the set of CPUs this kernel owns, in the host and in spawn kernels - * alike. The assignable-pool bookkeeping (mk_pool) is not done here; - * it belongs to the mk_send_cpu_* initiator paths of the kernel that - * manages the pool. - */ -static void mk_account_cpu_online(mk_phys_cpu_t cpu_id) +static struct mk_hotplug_work *mk_hotplug_work_pool; +static unsigned long *mk_hotplug_work_pool_busy; +static unsigned int mk_hotplug_work_pool_size; +static atomic_t mk_hotplug_work_pool_cursor = ATOMIC_INIT(0); +static struct workqueue_struct *mk_hotplug_workqueue; +static atomic64_t mk_hotplug_work_dropped = ATOMIC64_INIT(0); +static DEFINE_RATELIMIT_STATE(mk_hotplug_work_pool_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + +static void mk_hotplug_work_drop(void) { - if (mk_self->cpus && mk_cpu_set_add(mk_self->cpus, cpu_id)) - pr_warn("Multikernel hotplug: Failed to track CPU %llu\n", - cpu_id); + atomic64_inc(&mk_hotplug_work_dropped); + if (__ratelimit(&mk_hotplug_work_pool_rs)) + printk_deferred("WARNING: Multikernel hotplug work request dropped\n"); } -static void mk_account_cpu_offline(mk_phys_cpu_t cpu_id) +static struct mk_hotplug_work *mk_hotplug_work_get(void) { - mk_cpu_set_del(mk_self->cpus, cpu_id); + unsigned int start; + unsigned int slot; + unsigned int i; + + start = (unsigned int)atomic_inc_return(&mk_hotplug_work_pool_cursor); + for (i = 0; i < mk_hotplug_work_pool_size; i++) { + slot = (start + i) % mk_hotplug_work_pool_size; + if (!test_and_set_bit_lock(slot, mk_hotplug_work_pool_busy)) + return &mk_hotplug_work_pool[slot]; + } + + mk_hotplug_work_drop(); + return NULL; +} + +static void mk_hotplug_work_put(struct mk_hotplug_work *hp_work) +{ + clear_bit_unlock(hp_work->pool_slot, mk_hotplug_work_pool_busy); +} + +static void mk_hotplug_work_fn(struct work_struct *work) +{ + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); + + hp_work->dispatch(work); + mk_hotplug_work_put(hp_work); +} + +static int mk_hotplug_work_queue(struct mk_hotplug_work *hp_work, + work_func_t dispatch) +{ + hp_work->dispatch = dispatch; + if (queue_work(mk_hotplug_workqueue, &hp_work->work)) + return 0; + + mk_hotplug_work_put(hp_work); + mk_hotplug_work_drop(); + return -EBUSY; } +/* + * CPU Hotplug Operations + */ + /** * Search present CPUs (not possible CPUs) to find the logical CPU with matching * physical ID. Using present CPUs is important because topology can change during @@ -124,6 +188,7 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) pr_info("Multikernel hotplug: Adding CPU %llu (numa=%d, flags=0x%x)\n", cpu_id, (int)numa_node, flags); + mk_cpu_transaction_lock(); logical_cpu = mk_cpu_to_logical(cpu_id); if (logical_cpu < 0) { /* @@ -136,7 +201,8 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (logical_cpu < 0) { pr_err("Multikernel hotplug: CPU %llu is not in this kernel's pool\n", cpu_id); - return -ENODEV; + ret = -ENODEV; + goto unlock_transaction; } set_cpu_present(logical_cpu, true); } @@ -144,18 +210,37 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (cpu_online(logical_cpu)) { pr_warn("Multikernel hotplug: CPU %d (phys %llu) already online\n", logical_cpu, cpu_id); - mk_account_cpu_online(cpu_id); - return 0; + mk_cpu_ownership_lock(); + if (mk_self->cpus) { + ret = mk_cpu_set_add(mk_self->cpus, cpu_id); + if (ret) + pr_warn("Multikernel hotplug: Failed to track CPU %llu in root pool\n", + cpu_id); + } + mk_cpu_ownership_unlock(); + ret = 0; + goto unlock_transaction; } + mk_cpu_ownership_lock(); + if (mk_self->cpus) { + ret = mk_cpu_set_reserve(mk_self->cpus, 1); + if (ret) { + mk_cpu_ownership_unlock(); + goto unlock_transaction; + } + } + mk_cpu_ownership_unlock(); + if (!get_cpu_device(logical_cpu)) { struct cpu *c = &per_cpu(cpu_devices, logical_cpu); + c->hotpluggable = true; ret = register_cpu(c, logical_cpu); if (ret) { pr_err("Multikernel hotplug: Failed to register CPU %d: %d\n", logical_cpu, ret); - return ret; + goto unlock_transaction; } } @@ -163,10 +248,19 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (ret < 0) { pr_err("Multikernel hotplug: Failed to add CPU %d (phys %llu): %d\n", logical_cpu, cpu_id, ret); - return ret; + goto unlock_transaction; } - mk_account_cpu_online(cpu_id); + mk_cpu_ownership_lock(); + if (mk_self->cpus) { + ret = mk_cpu_set_add(mk_self->cpus, cpu_id); + if (ret) + pr_warn("Multikernel hotplug: Failed to track CPU %llu in root pool\n", + cpu_id); + } + mk_cpu_ownership_unlock(); + if (ret) + goto unlock_transaction; /* Track the operation for potential rollback */ op = kzalloc(sizeof(*op), GFP_KERNEL); @@ -183,7 +277,10 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) pr_info("Multikernel hotplug: Successfully added CPU %d (phys %llu)\n", logical_cpu, cpu_id); - return 0; + ret = 0; +unlock_transaction: + mk_cpu_transaction_unlock(); + return ret; } static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) @@ -191,34 +288,56 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) int logical_cpu; int ret; struct mk_hotplug_op *op; + bool tracked; + mk_cpu_transaction_lock(); logical_cpu = mk_cpu_to_logical(cpu_id); if (logical_cpu < 0) { pr_err("Multikernel hotplug: Physical CPU %llu not found\n", cpu_id); - return -ENODEV; + ret = -ENODEV; + goto unlock_transaction; } + mk_cpu_ownership_lock(); + tracked = mk_self->cpus && + mk_cpu_set_contains(mk_self->cpus, cpu_id); + if (!cpu_online(logical_cpu)) { pr_warn("Multikernel hotplug: CPU %d (phys %llu) already offline\n", logical_cpu, cpu_id); - mk_account_cpu_offline(cpu_id); - return 0; + mk_cpu_set_del(mk_self->cpus, cpu_id); + mk_cpu_ownership_unlock(); + ret = 0; + goto unlock_transaction; + } + + if (!tracked) { + pr_err("Multikernel hotplug: CPU %llu is not tracked in root pool\n", + cpu_id); + mk_cpu_ownership_unlock(); + ret = -EINVAL; + goto unlock_transaction; } /* Don't allow removing CPU 0 (boot processor) */ if (logical_cpu == 0) { pr_err("Multikernel hotplug: Cannot remove boot CPU\n"); - return -EINVAL; + mk_cpu_ownership_unlock(); + ret = -EINVAL; + goto unlock_transaction; } + mk_cpu_ownership_unlock(); ret = depart_cpu(logical_cpu); if (ret < 0) { pr_err("Multikernel hotplug: Failed to remove CPU %d (phys %llu): %d\n", logical_cpu, cpu_id, ret); - return ret; + goto unlock_transaction; } - mk_account_cpu_offline(cpu_id); + mk_cpu_ownership_lock(); + mk_cpu_set_del(mk_self->cpus, cpu_id); + mk_cpu_ownership_unlock(); /* * Clear CPU from present mask to prevent host kernel from trying @@ -241,127 +360,121 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) pr_info("Multikernel hotplug: Successfully removed CPU %d (phys %llu)\n", logical_cpu, cpu_id); - return 0; + ret = 0; +unlock_transaction: + mk_cpu_transaction_unlock(); + return ret; } static void mk_cpu_add_work_fn(struct work_struct *work) { - struct mk_cpu_hotplug_work *hp_work = container_of(work, struct mk_cpu_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_cpu_add(hp_work->cpu_id, hp_work->numa_node, hp_work->flags); + ret = mk_do_cpu_add(hp_work->cpu.cpu_id, hp_work->cpu.numa_node, + hp_work->cpu.flags); /* Send ACK back to sender */ ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->cpu_id; + ack.resource_id = hp_work->cpu.cpu_id; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for CPU %llu: %d\n", - hp_work->cpu_id, ack_ret); + hp_work->cpu.cpu_id, ack_ret); } - - kfree(hp_work); } static void mk_cpu_remove_work_fn(struct work_struct *work) { - struct mk_cpu_hotplug_work *hp_work = container_of(work, struct mk_cpu_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_cpu_remove(hp_work->cpu_id); + ret = mk_do_cpu_remove(hp_work->cpu.cpu_id); /* Send ACK back to sender */ ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->cpu_id; + ack.resource_id = hp_work->cpu.cpu_id; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for CPU %llu: %d\n", - hp_work->cpu_id, ack_ret); + hp_work->cpu.cpu_id, ack_ret); } - - kfree(hp_work); } /** * mk_handle_cpu_add - Handle CPU addition request * @payload: CPU resource payload * @payload_len: Payload length + * @sender_instance_id: Authenticated sending instance * * Brings a CPU online in the receiving kernel. This is called on the spawn * kernel side when the host kernel transfers a CPU to it. * * Returns 0 on success, negative error code on failure */ -static int mk_handle_cpu_add(struct mk_cpu_resource_payload *payload, u32 payload_len) +static int mk_handle_cpu_add(struct mk_cpu_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_cpu_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid CPU add payload size: %u\n", payload_len); return -EINVAL; } - /* - * We're in IRQ context (IPI handler), so we can't call add_cpu() directly. - */ - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); + hp_work = mk_hotplug_work_get(); if (!hp_work) return -ENOMEM; - INIT_WORK(&hp_work->work, mk_cpu_add_work_fn); - hp_work->cpu_id = payload->cpu_id; - hp_work->numa_node = payload->numa_node; - hp_work->flags = payload->flags; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->cpu.cpu_id = payload->cpu_id; + hp_work->cpu.numa_node = payload->numa_node; + hp_work->cpu.flags = payload->flags; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_CPU_ADD; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_cpu_add_work_fn); } /** * mk_handle_cpu_remove - Handle CPU removal request * @payload: CPU resource payload * @payload_len: Payload length + * @sender_instance_id: Authenticated sending instance * * Takes a CPU offline in the current kernel. This is called on the host * kernel side before transferring a CPU to a spawn kernel. * * Returns 0 on success, negative error code on failure */ -int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, u32 payload_len) +int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_cpu_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid CPU remove payload size: %u\n", payload_len); return -EINVAL; } - /* - * We're in IRQ context (IPI handler), so we can't call remove_cpu() directly. - */ - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); + hp_work = mk_hotplug_work_get(); if (!hp_work) return -ENOMEM; - INIT_WORK(&hp_work->work, mk_cpu_remove_work_fn); - hp_work->cpu_id = payload->cpu_id; - hp_work->numa_node = payload->numa_node; - hp_work->flags = payload->flags; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->cpu.cpu_id = payload->cpu_id; + hp_work->cpu.numa_node = payload->numa_node; + hp_work->cpu.flags = payload->flags; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_CPU_REMOVE; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_cpu_remove_work_fn); } /* @@ -520,119 +633,98 @@ static int mk_do_mem_remove(u64 start_pfn, u64 nr_pages) return 0; } -struct mk_mem_hotplug_work { - struct work_struct work; - u64 start_pfn; - u64 nr_pages; - u32 numa_node; - u32 mem_type; - int sender_instance_id; - u32 operation; -}; - static void mk_mem_add_work_fn(struct work_struct *work) { - struct mk_mem_hotplug_work *hp_work = container_of(work, struct mk_mem_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_mem_add(hp_work->start_pfn, hp_work->nr_pages, - hp_work->numa_node, hp_work->mem_type); + ret = mk_do_mem_add(hp_work->mem.start_pfn, hp_work->mem.nr_pages, + hp_work->mem.numa_node, hp_work->mem.mem_type); ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->start_pfn; + ack.resource_id = hp_work->mem.start_pfn; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for mem add at 0x%llx: %d\n", - (u64)hp_work->start_pfn, ack_ret); + hp_work->mem.start_pfn, ack_ret); } - - kfree(hp_work); } static void mk_mem_remove_work_fn(struct work_struct *work) { - struct mk_mem_hotplug_work *hp_work = container_of(work, struct mk_mem_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_mem_remove(hp_work->start_pfn, hp_work->nr_pages); + ret = mk_do_mem_remove(hp_work->mem.start_pfn, hp_work->mem.nr_pages); ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->start_pfn; + ack.resource_id = hp_work->mem.start_pfn; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for mem remove at 0x%llx: %d\n", - (u64)hp_work->start_pfn, ack_ret); + hp_work->mem.start_pfn, ack_ret); } - - kfree(hp_work); } -static int mk_handle_mem_add(struct mk_mem_resource_payload *payload, u32 payload_len) +static int mk_handle_mem_add(struct mk_mem_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_mem_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid memory add payload size: %u\n", payload_len); return -EINVAL; } - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work for mem add at 0x%llx\n", - (u64)payload->start_pfn); + hp_work = mk_hotplug_work_get(); + if (!hp_work) return -ENOMEM; - } - INIT_WORK(&hp_work->work, mk_mem_add_work_fn); - hp_work->start_pfn = payload->start_pfn; - hp_work->nr_pages = payload->nr_pages; - hp_work->numa_node = payload->numa_node; - hp_work->mem_type = payload->mem_type; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->mem.start_pfn = payload->start_pfn; + hp_work->mem.nr_pages = payload->nr_pages; + hp_work->mem.numa_node = payload->numa_node; + hp_work->mem.mem_type = payload->mem_type; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_MEM_ADD; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_mem_add_work_fn); } -static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 payload_len) +static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_mem_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid memory remove payload size: %u\n", payload_len); return -EINVAL; } - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work for mem remove at 0x%llx\n", - (u64)payload->start_pfn); + hp_work = mk_hotplug_work_get(); + if (!hp_work) return -ENOMEM; - } - INIT_WORK(&hp_work->work, mk_mem_remove_work_fn); - hp_work->start_pfn = payload->start_pfn; - hp_work->nr_pages = payload->nr_pages; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->mem.start_pfn = payload->start_pfn; + hp_work->mem.nr_pages = payload->nr_pages; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_MEM_REMOVE; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_mem_remove_work_fn); } /* * PCI Device Hotplug Operations */ +#if IS_ENABLED(CONFIG_PCI) static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { @@ -802,121 +894,117 @@ static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) return 0; } -struct mk_device_hotplug_work { - struct work_struct work; - u16 domain; - u8 bus; - u8 devfn; - u32 flags; - char driver_override[64]; - int sender_instance_id; - u32 operation; -}; +#else /* CONFIG_PCI */ +static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, + const char *driver_override, u32 flags) +{ + return -ENODEV; +} + +static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) +{ + return -ENODEV; +} +#endif /* CONFIG_PCI */ static void mk_device_add_work_fn(struct work_struct *work) { - struct mk_device_hotplug_work *hp_work = container_of(work, struct mk_device_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_device_add(hp_work->domain, hp_work->bus, hp_work->devfn, - hp_work->driver_override, hp_work->flags); + ret = mk_do_device_add(hp_work->device.domain, hp_work->device.bus, + hp_work->device.devfn, + hp_work->device.driver_override, + hp_work->device.flags); ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = (hp_work->domain << 16) | (hp_work->bus << 8) | hp_work->devfn; + ack.resource_id = (hp_work->device.domain << 16) | + (hp_work->device.bus << 8) | hp_work->device.devfn; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for device %04x:%02x:%02x.%x: %d\n", - hp_work->domain, hp_work->bus, - PCI_SLOT(hp_work->devfn), PCI_FUNC(hp_work->devfn), ack_ret); + hp_work->device.domain, hp_work->device.bus, + PCI_SLOT(hp_work->device.devfn), + PCI_FUNC(hp_work->device.devfn), ack_ret); } - - kfree(hp_work); } static void mk_device_remove_work_fn(struct work_struct *work) { - struct mk_device_hotplug_work *hp_work = container_of(work, struct mk_device_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_device_remove(hp_work->domain, hp_work->bus, hp_work->devfn); + ret = mk_do_device_remove(hp_work->device.domain, hp_work->device.bus, + hp_work->device.devfn); ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = (hp_work->domain << 16) | (hp_work->bus << 8) | hp_work->devfn; + ack.resource_id = (hp_work->device.domain << 16) | + (hp_work->device.bus << 8) | hp_work->device.devfn; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for device %04x:%02x:%02x.%x: %d\n", - hp_work->domain, hp_work->bus, - PCI_SLOT(hp_work->devfn), PCI_FUNC(hp_work->devfn), ack_ret); + hp_work->device.domain, hp_work->device.bus, + PCI_SLOT(hp_work->device.devfn), + PCI_FUNC(hp_work->device.devfn), ack_ret); } - - kfree(hp_work); } -static int mk_handle_device_add(struct mk_device_resource_payload *payload, u32 payload_len) +static int mk_handle_device_add(struct mk_device_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_device_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid device add payload size: %u\n", payload_len); return -EINVAL; } - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work structure for device %04x:%02x:%02x.%x\n", - payload->domain, payload->bus, - PCI_SLOT(payload->devfn), PCI_FUNC(payload->devfn)); + hp_work = mk_hotplug_work_get(); + if (!hp_work) return -ENOMEM; - } - INIT_WORK(&hp_work->work, mk_device_add_work_fn); - hp_work->domain = payload->domain; - hp_work->bus = payload->bus; - hp_work->devfn = payload->devfn; - hp_work->flags = payload->flags; - strscpy(hp_work->driver_override, payload->driver_override, sizeof(hp_work->driver_override)); - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->device.domain = payload->domain; + hp_work->device.bus = payload->bus; + hp_work->device.devfn = payload->devfn; + hp_work->device.flags = payload->flags; + strscpy(hp_work->device.driver_override, payload->driver_override, + sizeof(hp_work->device.driver_override)); + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_DEVICE_ADD; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_device_add_work_fn); } -static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u32 payload_len) +static int mk_handle_device_remove(struct mk_device_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_device_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid device remove payload size: %u\n", payload_len); return -EINVAL; } - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work structure for device %04x:%02x:%02x.%x\n", - payload->domain, payload->bus, - PCI_SLOT(payload->devfn), PCI_FUNC(payload->devfn)); + hp_work = mk_hotplug_work_get(); + if (!hp_work) return -ENOMEM; - } - INIT_WORK(&hp_work->work, mk_device_remove_work_fn); - hp_work->domain = payload->domain; - hp_work->bus = payload->bus; - hp_work->devfn = payload->devfn; - hp_work->flags = payload->flags; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->device.domain = payload->domain; + hp_work->device.bus = payload->bus; + hp_work->device.devfn = payload->devfn; + hp_work->device.flags = payload->flags; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_DEVICE_REMOVE; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_device_remove_work_fn); } /* @@ -929,6 +1017,7 @@ static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u * @subtype: Message subtype (CPU_ADD, CPU_REMOVE, MEM_ADD, MEM_REMOVE, DEVICE_ADD, DEVICE_REMOVE) * @payload: Payload data * @payload_len: Payload length + * @sender_instance_id: Authenticated sending instance * @ctx: Context (unused) * * This is the main message handler registered with the multikernel @@ -936,7 +1025,8 @@ static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u * message subtype. */ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx) { int ret = 0; @@ -947,27 +1037,31 @@ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, switch (subtype) { case MK_RES_CPU_ADD: - ret = mk_handle_cpu_add((struct mk_cpu_resource_payload *)payload, payload_len); + ret = mk_handle_cpu_add(payload, payload_len, sender_instance_id); break; case MK_RES_CPU_REMOVE: - ret = mk_handle_cpu_remove((struct mk_cpu_resource_payload *)payload, payload_len); + ret = mk_handle_cpu_remove(payload, payload_len, + sender_instance_id); break; case MK_RES_MEM_ADD: - ret = mk_handle_mem_add((struct mk_mem_resource_payload *)payload, payload_len); + ret = mk_handle_mem_add(payload, payload_len, sender_instance_id); break; case MK_RES_MEM_REMOVE: - ret = mk_handle_mem_remove((struct mk_mem_resource_payload *)payload, payload_len); + ret = mk_handle_mem_remove(payload, payload_len, + sender_instance_id); break; case MK_RES_DEVICE_ADD: - ret = mk_handle_device_add((struct mk_device_resource_payload *)payload, payload_len); + ret = mk_handle_device_add(payload, payload_len, + sender_instance_id); break; case MK_RES_DEVICE_REMOVE: - ret = mk_handle_device_remove((struct mk_device_resource_payload *)payload, payload_len); + ret = mk_handle_device_remove(payload, payload_len, + sender_instance_id); break; case MK_RES_ACK: @@ -1009,19 +1103,65 @@ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, */ int __init mk_hotplug_init(void) { + unsigned int pool_size; + unsigned int i; int ret; pr_info("Initializing multikernel hotplug subsystem\n"); + /* + * Match the PCI control-plane reserve. Exhaustion is still handled + * explicitly because resource requests do not consume reply-slot credits. + */ + pool_size = max_t(unsigned int, num_possible_cpus(), 1) * + MK_REPLY_SLOTS; + mk_hotplug_work_pool = kcalloc(pool_size, + sizeof(*mk_hotplug_work_pool), + GFP_KERNEL); + if (!mk_hotplug_work_pool) + return -ENOMEM; + + mk_hotplug_work_pool_busy = bitmap_zalloc(pool_size, GFP_KERNEL); + if (!mk_hotplug_work_pool_busy) { + ret = -ENOMEM; + goto free_pool; + } + mk_hotplug_work_pool_size = pool_size; + atomic_set(&mk_hotplug_work_pool_cursor, 0); + for (i = 0; i < pool_size; i++) { + INIT_WORK(&mk_hotplug_work_pool[i].work, + mk_hotplug_work_fn); + mk_hotplug_work_pool[i].pool_slot = i; + } + + mk_hotplug_workqueue = + alloc_workqueue("mk-hotplug", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!mk_hotplug_workqueue) { + ret = -ENOMEM; + goto free_pool_bitmap; + } + /* Register handler for resource management messages */ ret = mk_register_msg_handler(MK_MSG_RESOURCE, mk_resource_msg_handler, NULL); if (ret < 0) { pr_err("Failed to register multikernel hotplug message handler: %d\n", ret); - return ret; + goto destroy_workqueue; } pr_info("Multikernel hotplug subsystem initialized\n"); return 0; + +destroy_workqueue: + destroy_workqueue(mk_hotplug_workqueue); + mk_hotplug_workqueue = NULL; +free_pool_bitmap: + bitmap_free(mk_hotplug_work_pool_busy); + mk_hotplug_work_pool_busy = NULL; + mk_hotplug_work_pool_size = 0; +free_pool: + kfree(mk_hotplug_work_pool); + mk_hotplug_work_pool = NULL; + return ret; } /** @@ -1037,6 +1177,18 @@ void mk_hotplug_cleanup(void) /* Unregister message handler */ mk_unregister_msg_handler(MK_MSG_RESOURCE, mk_resource_msg_handler); + if (mk_hotplug_workqueue) { + destroy_workqueue(mk_hotplug_workqueue); + mk_hotplug_workqueue = NULL; + } + WARN_ON_ONCE(mk_hotplug_work_pool_busy && + !bitmap_empty(mk_hotplug_work_pool_busy, + mk_hotplug_work_pool_size)); + bitmap_free(mk_hotplug_work_pool_busy); + mk_hotplug_work_pool_busy = NULL; + mk_hotplug_work_pool_size = 0; + kfree(mk_hotplug_work_pool); + mk_hotplug_work_pool = NULL; /* Free operation tracking list */ mutex_lock(&mk_hotplug_mutex); @@ -1198,6 +1350,7 @@ int mk_pool_device_remove(u16 domain, u8 bus, u8 devfn, */ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) { + struct mk_cpu_set removing = { .nr = 1, .cap = 1, .ids = &cpu_id }; struct mk_cpu_resource_payload payload = { .cpu_id = cpu_id, .numa_node = 0, @@ -1208,6 +1361,8 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) struct mk_instance *target_instance; int ret; + raw_spin_lock_init(&removing.lock); + /* For self-removal, execute directly (we're in process context) */ if (instance_id == mk_self->id) { if (mk_pool) @@ -1223,6 +1378,7 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (target_instance->state != MK_STATE_ACTIVE) { struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; + raw_spin_lock_init(&cpus.lock); /* * A CPU the instance has already run on is parked on that * instance's context. Bring it back to the host slot before @@ -1248,26 +1404,46 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) goto out; } - ret = mk_cpu_set_reserve(mk_pool->cpus, 1); - if (ret) - goto out; - + mk_cpu_transaction_lock(); pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, cpu_id); if (!pending) { ret = -ENOMEM; - goto out; + goto unlock_transaction; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, - &payload, sizeof(payload)); + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { + pr_err("Multikernel hotplug: CPU %llu not assigned to instance %d\n", + cpu_id, instance_id); + ret = -EINVAL; + mk_cpu_ownership_unlock(); + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; + } + + ret = mk_cpu_set_reserve(mk_pool->cpus, 1); + if (ret) { + mk_cpu_ownership_unlock(); + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; + } + mk_cpu_ownership_unlock(); + ret = mk_instance_migrate_irq_route(target_instance, &removing); + if (ret) { + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; + } + ret = mk_send_message_to_instance(target_instance, MK_MSG_RESOURCE, + MK_RES_CPU_REMOVE, &payload, + sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ - goto out; + goto unlock_transaction; } ret = mk_msg_pending_wait(pending, 10000); if (ret < 0) - goto out; + goto unlock_transaction; /* * The spawn kernel parked the CPU on its own context when it went @@ -1281,13 +1457,29 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (ret < 0) { pr_err("Multikernel hotplug: CPU %llu offline in instance %d but not reparked to host: %d\n", cpu_id, instance_id, ret); - goto out; + goto unlock_transaction; } + down_write(&target_instance->control_route_sem); + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { + ret = -ESTALE; + goto unlock_ownership; + } + ret = mk_cpu_set_add(mk_pool->cpus, cpu_id); + if (ret) { + pr_warn("Multikernel hotplug: Failed to track CPU %llu in pool\n", + cpu_id); + goto unlock_ownership; + } mk_cpu_set_del(target_instance->cpus, cpu_id); - mk_cpu_set_add(mk_pool->cpus, cpu_id); ret = 0; +unlock_ownership: + mk_cpu_ownership_unlock(); + up_write(&target_instance->control_route_sem); +unlock_transaction: + mk_cpu_transaction_unlock(); out: mk_instance_put(target_instance); return ret; @@ -1332,26 +1524,44 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl instance_id); return -ENODEV; } + if (arch_cpu_from_physical_id(cpu_id) == 0) { + pr_err("Multikernel hotplug: CPU %llu is reserved for host control\n", + cpu_id); + ret = -EINVAL; + goto out; + } /* For non-running instances, transfer CPU from root using existing API */ if (target_instance->state != MK_STATE_ACTIVE) { struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; + raw_spin_lock_init(&cpus.lock); ret = mk_instance_transfer_cpus(target_instance, &cpus); goto out; } - /* - * Only a CPU from the assignable pool is parked on the host slot; - * publishing a wakeup for any other CPU can only time out. - */ + if (!mk_pool) { + ret = -ENODEV; + goto out; + } + + mk_cpu_transaction_lock(); + mk_cpu_ownership_lock(); if (!mk_cpu_set_contains(mk_pool->cpus, cpu_id)) { - pr_err("Multikernel hotplug: CPU %llu is not in this kernel's pool\n", + pr_err("Multikernel hotplug: CPU %llu not available in the pool\n", cpu_id); ret = -EBUSY; - goto out; + mk_cpu_ownership_unlock(); + goto unlock_transaction; } + ret = mk_cpu_set_reserve(target_instance->cpus, 1); + if (ret) { + mk_cpu_ownership_unlock(); + goto unlock_transaction; + } + mk_cpu_ownership_unlock(); + /* * The CPU is parked on the host slot, where the spawn kernel's * secondary wakeup cannot reach it. Point it at the instance's @@ -1361,22 +1571,23 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl if (ret < 0) { pr_err("Multikernel hotplug: Failed to repark CPU %llu to instance %d: %d\n", cpu_id, instance_id, ret); - goto out; + goto unlock_transaction; } pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_ADD, cpu_id); if (!pending) { mk_repark_cpu_to_host(target_instance, cpu_id); ret = -ENOMEM; - goto out; + goto unlock_transaction; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_ADD, - &payload, sizeof(payload)); + ret = mk_send_message_to_instance(target_instance, MK_MSG_RESOURCE, + MK_RES_CPU_ADD, &payload, + sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ mk_repark_cpu_to_host(target_instance, cpu_id); - goto out; + goto unlock_transaction; } ret = mk_msg_pending_wait(pending, 10000); @@ -1388,20 +1599,51 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl * watching the context and this times out harmlessly. */ mk_repark_cpu_to_host(target_instance, cpu_id); - goto out; + goto unlock_transaction; } - if (mk_cpu_set_add(target_instance->cpus, cpu_id)) + down_write(&target_instance->control_route_sem); + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(mk_pool->cpus, cpu_id)) { + ret = -ESTALE; + goto unlock_ownership; + } + ret = mk_cpu_set_add(target_instance->cpus, cpu_id); + if (ret) { pr_warn("Multikernel hotplug: Failed to track CPU %llu in instance %d\n", cpu_id, instance_id); + goto unlock_ownership; + } mk_cpu_set_del(mk_pool->cpus, cpu_id); + if (mk_instance_irq_route_load(target_instance) == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(target_instance, cpu_id); ret = 0; +unlock_ownership: + mk_cpu_ownership_unlock(); + up_write(&target_instance->control_route_sem); +unlock_transaction: + mk_cpu_transaction_unlock(); out: mk_instance_put(target_instance); return ret; } +static int mk_memory_change_allowed(struct mk_instance *instance) +{ + bool iommu_active; + + mutex_lock(&instance->resource_mutex); + iommu_active = mk_pci_iommu_lease_active_locked(instance); + mutex_unlock(&instance->resource_mutex); + if (!iommu_active) + return 0; + + pr_err("Cannot change memory for instance %d while an IOMMU lease is active\n", + instance->id); + return -EBUSY; +} + /** * mk_send_mem_add - Add memory to instance * @instance_id: Target instance ID @@ -1438,6 +1680,11 @@ int mk_send_mem_add(int instance_id, u64 start_pfn, u64 nr_pages, target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, allocate memory from pool and add to instance */ if (target_instance->state != MK_STATE_ACTIVE) { @@ -1506,6 +1753,11 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, just remove the memory region from the instance */ if (target_instance->state != MK_STATE_ACTIVE) { diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 8503733372da18..bdb3ad89f40151 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -42,6 +43,19 @@ EXPORT_SYMBOL_GPL(mk_self); struct mk_instance *host_instance; EXPORT_SYMBOL_GPL(host_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /** * mk_dt_extract_instance_info() - Extract instance ID and name from DTB * @dtb_data: Device tree blob data @@ -211,6 +225,7 @@ static int __init mk_instance_alloc_ipi(struct mk_instance *instance) instance->id); return -ENOMEM; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -264,6 +279,17 @@ static int __init mk_copy_platform_devices(const struct mk_dt_config *config, return 0; } +static void __init mk_take_pci_devices(struct mk_dt_config *config, + struct mk_instance *instance) +{ + list_splice_tail_init(&config->pci_devices, &instance->pci_devices); + instance->pci_device_count = config->pci_device_count; + instance->pci_devices_valid = config->pci_devices_valid; + + config->pci_device_count = 0; + config->pci_devices_valid = false; +} + /* A message ring the host describes in /chosen: its address and size */ static bool __init mk_chosen_ring(const char *what, phys_addr_t *phys, u32 *pages) @@ -297,6 +323,11 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) return 0; } ipi_size = (size_t)ipi_pages << PAGE_SHIFT; + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small: %zu < %zu\n", ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -316,37 +347,83 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) static int __init mk_restore_host_instance(void) { struct mk_instance *hi; - phys_addr_t host_ipi_phys; - u32 host_ipi_pages; - size_t host_ipi_size; - u64 host_ipi_cpu; + struct mk_shared_data *shared; + mk_phys_cpu_t parent_cpu; + mk_phys_cpu_t chosen_parent_cpu; + phys_addr_t halt_phys; + size_t halt_size; + int parent_id; + int child_id; + u32 halt_pages; + int ret; - if (!mk_chosen_ring("host-ipi", &host_ipi_phys, &host_ipi_pages) || - of_property_read_u64(of_chosen, "multikernel,host-ipi-cpu", - &host_ipi_cpu)) { - pr_warn("No host IPI buffer in the boot tree (spawn won't be able to send to host)\n"); + if (!mk_self || !mk_self->ipi_data) { + pr_err("No parent/child IPI link in the boot tree\n"); return -ENOENT; } - host_ipi_size = (size_t)host_ipi_pages << PAGE_SHIFT; + shared = mk_self->ipi_data; + if (!of_chosen || + of_property_read_u64(of_chosen, "multikernel,host-ipi-cpu", + &chosen_parent_cpu)) { + pr_err("No parent IPI CPU in the boot tree\n"); + return -EPROTO; + } + parent_id = READ_ONCE(shared->parent_id); + child_id = READ_ONCE(shared->child_id); + parent_cpu = READ_ONCE(shared->parent_doorbell_cpu); + if (parent_id < 0 || parent_id == mk_self->id || + child_id != mk_self->id || parent_cpu == MK_PHYS_CPU_INVALID || + parent_cpu != chosen_parent_cpu) { + pr_err("Invalid parent/child IPI identity: parent=%d child=%d self=%d shared-cpu=%llu chosen-cpu=%llu\n", + parent_id, child_id, mk_self->id, + (unsigned long long)parent_cpu, + (unsigned long long)chosen_parent_cpu); + return -EPROTO; + } - hi = mk_instance_alloc(0, "host"); + hi = mk_instance_alloc(parent_id, "host"); if (!hi) return -ENOMEM; - - hi->ipi_target = host_ipi_cpu; - - hi->ipi_data = memremap(host_ipi_phys, host_ipi_size, MEMREMAP_WB); - if (!hi->ipi_data) { - pr_err("Failed to map host IPI buffer at 0x%llx\n", - (unsigned long long)host_ipi_phys); + hi->ipi_target = parent_cpu; + mk_instance_irq_route_store(hi, parent_cpu); + ret = mk_cpu_set_add(hi->cpus, parent_cpu); + if (ret) + goto err_free; + if (!mk_chosen_ring("host-ipi", &halt_phys, &halt_pages)) { + pr_err("No host force-halt area in the boot tree\n"); + ret = -EPROTO; goto err_free; } - hi->ipi_phys = host_ipi_phys; - hi->ipi_pages = host_ipi_pages; + halt_size = (size_t)halt_pages << PAGE_SHIFT; + if (halt_size < sizeof(struct mk_shared_data) || + halt_phys == mk_self->ipi_phys) { + pr_err("Invalid host force-halt area: phys=0x%llx, pages=%u\n", + (unsigned long long)halt_phys, halt_pages); + ret = -EPROTO; + goto err_free; + } + hi->halt_data = memremap(halt_phys, halt_size, MEMREMAP_WB); + if (!hi->halt_data) { + pr_err("Failed to map host force-halt area at 0x%llx\n", + (unsigned long long)halt_phys); + ret = -ENOMEM; + goto err_free; + } + hi->ipi_data = mk_self->ipi_data; + hi->ipi_phys = mk_self->ipi_phys; + hi->ipi_pages = mk_self->ipi_pages; - if (mk_instance_publish(hi)) { - memunmap(hi->ipi_data); + ret = mk_instance_publish(hi); + if (ret) goto err_free; + ret = mk_ipi_endpoint_init(hi, false); + if (ret) { + mutex_lock(&mk_instance_mutex); + idr_remove(&mk_instance_idr, hi->id); + list_del(&hi->list); + mutex_unlock(&mk_instance_mutex); + mk_instance_free(hi); + return ret; } /* The host is running, or this kernel would not be */ mk_instance_set_state(hi, MK_STATE_ACTIVE); @@ -362,15 +439,13 @@ static int __init mk_restore_host_instance(void) host_instance = hi; - pr_info("Restored host IPI buffer: phys=0x%llx, pages=%u\n", - (unsigned long long)host_ipi_phys, host_ipi_pages); - pr_info("Registered host instance (ID 0) for spawn→host communication\n"); + pr_info("Registered parent instance %d on duplex IPI link\n", parent_id); return 0; err_free: mk_instance_free(hi); - return -ENOMEM; + return ret; } /** @@ -393,6 +468,9 @@ int __init mk_instance_restore_from_manifest(void) int instance_id; const char *instance_name; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + if (!mk_manifest_phys()) { pr_info("No manifest available for multikernel DTB restoration\n"); @@ -401,12 +479,19 @@ int __init mk_instance_restore_from_manifest(void) pr_err("Failed to allocate the self instance\n"); return -ENOMEM; } - /* Initially, root owns all online CPUs (physical IDs) */ - for_each_online_cpu(cpu) { - if (mk_cpu_set_add(instance->cpus, - arch_cpu_physical_id(cpu))) - pr_warn("Failed to add CPU %d to the self instance\n", - cpu); + /* + * Root owns every enumerated CPU, including APs that become online + * only after early initcalls complete. + */ + for_each_present_cpu(cpu) { + ret = mk_cpu_set_add(instance->cpus, + arch_cpu_physical_id(cpu)); + if (ret) { + pr_err("Failed to track CPU %d in the self instance: %d\n", + cpu, ret); + mk_instance_free(instance); + return ret; + } } mk_cpu_set_format(cpus_buf, sizeof(cpus_buf), instance->cpus); pr_info("Self instance initialized with CPUs (physical): %s\n", @@ -435,7 +520,7 @@ int __init mk_instance_restore_from_manifest(void) dtb_virt = initial_boot_params; if (!dtb_virt || !of_have_populated_dt()) { pr_err("Boot device tree from the manifest was not unflattened\n"); - return -ENOENT; + mk_manifest_reject_and_park(-ENOENT); } dtb_len = fdt_totalsize(dtb_virt); @@ -444,7 +529,7 @@ int __init mk_instance_restore_from_manifest(void) ret = mk_dt_extract_instance_info(dtb_virt, dtb_len, &instance_id, &instance_name); if (ret) { pr_err("Failed to extract instance info from DTB: %d\n", ret); - return ret; + mk_manifest_reject_and_park(ret); } pr_info("DTB contains instance ID %d, name '%s'\n", instance_id, instance_name); @@ -477,8 +562,8 @@ int __init mk_instance_restore_from_manifest(void) goto cleanup_instance; } - /* The tree is the record of this kernel's PCI devices; the list stays empty */ - instance->pci_devices_valid = true; + /* Config entries become this kernel's assigned-device allowlist. */ + mk_take_pci_devices(&config, instance); ret = mk_copy_platform_devices(&config, instance); if (ret) { @@ -493,26 +578,33 @@ int __init mk_instance_restore_from_manifest(void) } ret = mk_instance_publish(instance); - if (ret) { - if (instance->ipi_data) - memunmap(instance->ipi_data); + if (ret) goto cleanup_instance; - } mk_self = instance; - if (mk_restore_host_instance()) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); + ret = mk_restore_host_instance(); + if (ret) + mk_manifest_reject_and_park(ret); + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); pr_info("Successfully restored multikernel self instance %d ('%s') from the boot tree (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); return 0; cleanup_instance: + if (instance->ipi_data) + memunmap(instance->ipi_data); mk_instance_free(instance); config_free: mk_dt_config_free(&config); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } @@ -532,6 +624,7 @@ early_initcall(mk_instance_restore_from_manifest); * * Returns: true if probing should proceed, false to skip entirely */ +#if IS_ENABLED(CONFIG_PCI) bool mk_pci_should_probe(struct pci_bus *bus, int devfn) { struct device_node *np; @@ -549,6 +642,7 @@ bool mk_pci_should_probe(struct pci_bus *bus, int devfn) return available; } EXPORT_SYMBOL_GPL(mk_pci_should_probe); +#endif /* CONFIG_PCI */ /* * A netdev's alias is its interface name, the one the device had in the diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 563bad63aa0f7f..8926eb181bdb9f 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -16,6 +16,18 @@ struct mk_instance *mk_instance_alloc(int id, const char *name); int mk_instance_publish(struct mk_instance *instance); void mk_instance_free(struct mk_instance *instance); +/* core.c */ +int mk_instance_force_halt(struct mk_instance *instance); + +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance); +void mk_poll_ipi_messages(void); +int mk_reply_publish_route_locked(struct mk_instance *instance, + const struct mk_reply_handle *reply, + s32 status, u32 value); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; @@ -23,6 +35,13 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, int chosen_node, int resources_node); struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); +int mk_instance_release_resources(struct mk_instance *instance); +void mk_cpu_ownership_lock(void); +void mk_cpu_ownership_unlock(void); +void mk_cpu_ownership_assert_held(void); +/* Caller serializes CPU ownership changes with mk_cpu_transaction_lock(). */ +int mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing); /* dts.c */ int mk_dt_parse_chosen(const void *fdt, int chosen_node, @@ -34,7 +53,85 @@ int mk_dt_emit_boot_tree(struct mk_instance *instance, void *fdt, size_t size, int (*chosen)(void *fdt, void *data), void *data); int mk_dt_generate_instance_dtb(struct mk_instance *instance, void **out_dtb, size_t *out_size); +int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, + u8 *slot, u8 *func); + +/* pci.c */ +#if defined(CONFIG_MULTIKERNEL) && defined(CONFIG_PCI) +int mk_pci_lease_system_init(void); +void mk_pci_lease_system_cleanup(void); +void mk_pci_lease_instance_init(struct mk_instance *instance); +void mk_pci_irq_retry_disable_sync(struct mk_instance *instance); +void mk_pci_irq_retry_enable(struct mk_instance *instance); +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance); +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count); +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_release_assignments(struct mk_instance *instance); +int mk_pci_quiesce_instance_irqs(struct mk_instance *instance, + bool parked_force); +unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance); +#else +static inline int mk_pci_lease_system_init(void) { return 0; } +static inline void mk_pci_lease_system_cleanup(void) { } +static inline void mk_pci_lease_instance_init(struct mk_instance *instance) { } +static inline void +mk_pci_irq_retry_disable_sync(struct mk_instance *instance) +{ +} + +static inline void mk_pci_irq_retry_enable(struct mk_instance *instance) +{ +} + +static inline bool +mk_pci_iommu_lease_active_locked(struct mk_instance *instance) +{ + return false; +} + +static inline int +mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count) +{ + return requested_count ? -EOPNOTSUPP : 0; +} + +static inline int +mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) +{ + return -EOPNOTSUPP; +} + +static inline int +mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + return -EOPNOTSUPP; +} + +static inline int mk_pci_release_assignments(struct mk_instance *instance) +{ + return 0; +} + +static inline int +mk_pci_quiesce_instance_irqs(struct mk_instance *instance, bool parked_force) +{ + return 0; +} +static inline unsigned int +mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + return 0; +} +#endif /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; extern struct mutex mk_overlay_mutex; @@ -48,7 +145,8 @@ int mk_arm_force_halt(struct mk_instance *instance); /* hotplug.c */ int mk_hotplug_init(void); void mk_hotplug_cleanup(void); -int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, u32 payload_len); +int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, + u32 payload_len, s32 sender_instance_id); /* * Move primitives between this kernel and the pool it manages. Valid diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index b9a2a74bbf1ca4..18962c6e64f416 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -1,356 +1,797 @@ // SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2025 Multikernel Technologies, Inc. All rights reserved - */ -#include -#include #include -#include -#include -#include -#include -#include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "internal.h" -/* Callback management */ static struct mk_ipi_handler *mk_handlers; -static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); +static DEFINE_RAW_SPINLOCK(mk_handlers_lock); +static LIST_HEAD(mk_ipi_endpoints); +DEFINE_STATIC_SRCU(mk_ipi_srcu); +static DEFINE_RAW_SPINLOCK(mk_ipi_endpoints_lock); +static bool mk_handlers_ready; +static DEFINE_RATELIMIT_STATE(mk_ipi_full_rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); +static DEFINE_RATELIMIT_STATE(mk_ipi_bad_size_rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); +static DECLARE_WAIT_QUEUE_HEAD(mk_reply_waitq); +static void mk_reply_wake_workfn(struct work_struct *work); +static DECLARE_WORK(mk_reply_wake_work, mk_reply_wake_workfn); + +static void mk_reply_wake_workfn(struct work_struct *work) +{ + wake_up_all(&mk_reply_waitq); +} -static void mk_ipi_drain_ring(void); +static void mk_reply_wake_all(void) +{ + /* + * Workqueue clears PENDING before running the callback. A false return + * therefore means a future callback is already queued; a notification + * racing with the running callback queues another execution. + */ + schedule_work(&mk_reply_wake_work); +} -/* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. - */ -static inline unsigned int mk_ring_idx(unsigned int i) +#define MK_REPLY_STATE_MASK (BIT(MK_REPLY_STATE_BITS) - 1) +#define MK_REPLY_GENERATION_MAX (U64_MAX >> MK_REPLY_STATE_BITS) + +static u64 mk_reply_token(u64 generation, enum mk_reply_state state) { - return i & (MK_IPI_RING_SIZE - 1); + return generation << MK_REPLY_STATE_BITS | state; } -/** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring - * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. - */ -void mk_ipi_ring_drop_pending(void) +static u64 mk_reply_generation(u64 token) { - struct mk_ipi_ring *ring; - unsigned int head, tail; + return token >> MK_REPLY_STATE_BITS; +} + +static enum mk_reply_state mk_reply_state(u64 token) +{ + return token & MK_REPLY_STATE_MASK; +} - if (!mk_self || !mk_self->ipi_data) +static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) +{ + return READ_ONCE(instance->ipi_data); +} + +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance) +{ + struct mk_shared_data *shared = READ_ONCE(instance->halt_data); + + return shared ? shared : mk_instance_ipi_area(instance); +} + +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; + + if (!mk_instance_ipi_area(instance)) + return -ENODEV; + if (endpoint->registered) + return 0; + endpoint->peer = instance; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx = parent_side ? &instance->ipi_data->to_child : + &instance->ipi_data->to_parent; + endpoint->tx_head = 0; + endpoint->tx_epoch++; + endpoint->tx_enabled = true; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + endpoint->rx = parent_side ? &instance->ipi_data->to_parent : + &instance->ipi_data->to_child; + endpoint->rx_tail = 0; + endpoint->rx_dispatching = false; + endpoint->parent_side = parent_side; + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + list_add_tail_rcu(&endpoint->rx_node, &mk_ipi_endpoints); + endpoint->registered = true; + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + return 0; +} + +void mk_ipi_endpoint_close(struct mk_instance *instance) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; + + if (!endpoint->registered) return; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx_enabled = false; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); +} + +void mk_ipi_endpoint_unregister(struct mk_instance *instance) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; - ring = &mk_self->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); + if (!endpoint->registered) + return; + mk_ipi_endpoint_close(instance); + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + if (endpoint->registered) { + list_del_rcu(&endpoint->rx_node); + endpoint->registered = false; + } + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + synchronize_srcu(&mk_ipi_srcu); +} - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu) +{ + struct mk_shared_data *shared = mk_instance_ipi_area(instance); + u64 epoch; - atomic_set(&ring->tail, head); + if (!shared) + return; + epoch = READ_ONCE(shared->spawn_epoch) + 1; + if (!epoch) + epoch = 1; + mk_ipi_endpoint_unregister(instance); + mk_shared_data_reset(shared); + WRITE_ONCE(shared->spawn_epoch, epoch); + WRITE_ONCE(shared->parent_id, parent_id); + WRITE_ONCE(shared->child_id, child_id); + WRITE_ONCE(shared->parent_doorbell_cpu, parent_cpu); + WRITE_ONCE(shared->child_doorbell_cpu, child_cpu); + mk_ipi_endpoint_init(instance, true); } -/** - * multikernel_register_handler - Register a callback for multikernel IPI - * @callback: Function to call when IPI is received - * @ctx: Context pointer passed to the callback - * @ipi_type: IPI type this handler should process - * - * Returns pointer to handler on success, NULL on failure - */ -struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, unsigned int ipi_type) +int mk_reply_reserve(struct mk_shared_data *shared, u32 kind, u64 request_id, + struct mk_reply_handle *reply) +{ + struct mk_reply_table *table; + unsigned int i; + int ret; + + if (!reply || !request_id || !kind) + return -EINVAL; + ret = shared ? 0 : -ENODEV; + if (ret) + return ret; + + table = &shared->replies; + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &table->slots[i]; + u64 generation; + u64 claim; + u64 old; + + old = atomic64_read(&slot->state_generation); + if (mk_reply_state(old) != MK_REPLY_FREE) + continue; + generation = mk_reply_generation(old) + 1; + if (!generation || generation > MK_REPLY_GENERATION_MAX) + generation = 1; + claim = mk_reply_token(generation, MK_REPLY_WRITING); + if (atomic64_cmpxchg_acquire(&slot->state_generation, old, + claim) != old) + continue; + + WRITE_ONCE(slot->request_id, request_id); + WRITE_ONCE(slot->kind, kind); + WRITE_ONCE(slot->status, -ETIMEDOUT); + WRITE_ONCE(slot->value, ~0U); + atomic64_set_release(&slot->state_generation, + mk_reply_token(generation, + MK_REPLY_RESERVED)); + reply->slot = i; + reply->kind = kind; + reply->request_id = request_id; + reply->generation = generation; + return 0; + } + + atomic_inc(&table->occupied_failures); + return -ENOSPC; +} + +static int mk_reply_take_ready(struct mk_shared_data *shared, + struct mk_reply_handle *reply, + s32 *status, u32 *value) +{ + struct mk_reply_slot *slot = &shared->replies.slots[reply->slot]; + u64 ready = mk_reply_token(reply->generation, MK_REPLY_READY); + u64 free = mk_reply_token(reply->generation, MK_REPLY_FREE); + + if (atomic64_read_acquire(&slot->state_generation) != ready) + return -EAGAIN; + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) + return -EPROTO; + if (status) + *status = READ_ONCE(slot->status); + if (value) + *value = READ_ONCE(slot->value); + if (atomic64_cmpxchg_release(&slot->state_generation, ready, free) != + ready) + return -EAGAIN; + return 0; +} + +static bool mk_reply_cancel(struct mk_shared_data *shared, + struct mk_reply_handle *reply, bool atomic_timeout) +{ + struct mk_reply_table *table = &shared->replies; + struct mk_reply_slot *slot = &table->slots[reply->slot]; + u64 reserved = mk_reply_token(reply->generation, MK_REPLY_RESERVED); + u64 writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + u64 executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + u64 committed = mk_reply_token(reply->generation, MK_REPLY_COMMITTED); + u64 abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + u64 ready = mk_reply_token(reply->generation, MK_REPLY_READY); + u64 free = mk_reply_token(reply->generation, MK_REPLY_FREE); + u64 token; + + if (atomic64_cmpxchg_release(&slot->state_generation, reserved, free) == + reserved) + goto cancelled; + + for (;;) { + token = atomic64_read_acquire(&slot->state_generation); + if (token == ready) + return true; + if (token == committed) + goto timed_out; + if (token == writing && + atomic64_cmpxchg_release(&slot->state_generation, writing, + abandoned) == writing) + break; + if (token == executing && + atomic64_cmpxchg_release(&slot->state_generation, executing, + committed) == executing) { + atomic_inc(&table->indeterminate_timeouts); + goto timed_out; + } + if (token != writing && token != executing) + break; + } + +cancelled: + atomic_inc(&table->cancelled_slots); +timed_out: + if (atomic_timeout) + atomic_inc(&table->atomic_timeouts); + return false; +} + +static bool mk_reply_wait_done(struct mk_shared_data *shared, + const struct mk_reply_handle *reply) +{ + struct mk_reply_slot *slot = &shared->replies.slots[reply->slot]; + u64 token; + + token = atomic64_read_acquire(&slot->state_generation); + return token == mk_reply_token(reply->generation, MK_REPLY_READY) || + mk_reply_generation(token) != reply->generation || + mk_reply_state(token) == MK_REPLY_FREE; +} + +int mk_reply_wait_atomic(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_us, + s32 *status, u32 *value) +{ + u64 deadline; + + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + deadline = ktime_get_mono_fast_ns() + (u64)timeout_us * NSEC_PER_USEC; + for (;;) { + int ret = mk_reply_take_ready(shared, reply, status, value); + + if (!ret) + return 0; + if (ret != -EAGAIN) + return ret; + if (ktime_get_mono_fast_ns() >= deadline) + break; + cpu_relax(); + } + + if (mk_reply_cancel(shared, reply, true)) + return mk_reply_take_ready(shared, reply, status, value); + return -ETIMEDOUT; +} + +int mk_reply_wait(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_ms, + s32 *status, u32 *value) +{ + long waited; + int ret; + + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + waited = wait_event_timeout(mk_reply_waitq, + mk_reply_wait_done(shared, reply), + msecs_to_jiffies(timeout_ms)); + if (!waited) { + if (mk_reply_cancel(shared, reply, false)) + return mk_reply_take_ready(shared, reply, status, value); + return -ETIMEDOUT; + } + ret = mk_reply_take_ready(shared, reply, status, value); + if (ret == -EAGAIN) + ret = -ESTALE; + return ret; +} + +void mk_reply_release(struct mk_shared_data *shared, + struct mk_reply_handle *reply) +{ + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return; + mk_reply_cancel(shared, reply, false); +} + +int mk_reply_claim(struct mk_instance *instance, + const struct mk_reply_handle *reply) +{ + struct mk_shared_data *shared; + struct mk_reply_slot *slot; + u64 writing; + u64 abandoned; + u64 reserved; + u64 free; + int ret; + + if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + shared = instance->ipi_data; + ret = shared ? 0 : -ENODEV; + if (ret) + return ret; + slot = &shared->replies.slots[reply->slot]; + reserved = mk_reply_token(reply->generation, MK_REPLY_RESERVED); + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + free = mk_reply_token(reply->generation, MK_REPLY_FREE); + if (atomic64_cmpxchg_acquire(&slot->state_generation, reserved, + writing) != reserved) { + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) { + u64 old; + + old = atomic64_cmpxchg_release(&slot->state_generation, writing, + free); + if (old == abandoned) + old = atomic64_cmpxchg_release(&slot->state_generation, + abandoned, free); + if (old == writing || old == abandoned) + mk_reply_wake_all(); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + return 0; +} + +int mk_reply_begin_execute(struct mk_instance *instance, + const struct mk_reply_handle *reply) +{ + struct mk_reply_slot *slot; + u64 writing; + u64 executing; + u64 old; + + if (!instance || !instance->ipi_data || !reply || + reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + slot = &instance->ipi_data->replies.slots[reply->slot]; + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + old = atomic64_cmpxchg_acquire(&slot->state_generation, writing, + executing); + if (old == writing) + return 0; + if (old == mk_reply_token(reply->generation, MK_REPLY_ABANDONED)) + return -ECANCELED; + return -ESTALE; +} + +int mk_reply_publish_route_locked(struct mk_instance *instance, + const struct mk_reply_handle *reply, + s32 status, u32 value) +{ + struct mk_shared_data *shared; + struct mk_reply_slot *slot; + mk_phys_cpu_t target; + u64 writing; + u64 executing; + u64 committed; + u64 abandoned; + u64 ready; + u64 free; + u64 old; + unsigned long flags; + int ret; + + if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + lockdep_assert_held_read(&instance->control_route_sem); + shared = instance->ipi_data; + ret = shared ? 0 : -ENODEV; + if (ret) + return ret; + slot = &shared->replies.slots[reply->slot]; + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + committed = mk_reply_token(reply->generation, MK_REPLY_COMMITTED); + abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + ready = mk_reply_token(reply->generation, MK_REPLY_READY); + free = mk_reply_token(reply->generation, MK_REPLY_FREE); + + old = atomic64_read_acquire(&slot->state_generation); + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) { + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (old == abandoned) { + atomic64_cmpxchg_release(&slot->state_generation, abandoned, free); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (old == committed) { + atomic64_cmpxchg_release(&slot->state_generation, committed, free); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (old != executing && old != writing) { + atomic_inc(&shared->replies.late_replies); + return -EIO; + } + + WRITE_ONCE(slot->status, status); + WRITE_ONCE(slot->value, value); + if (atomic64_cmpxchg_release(&slot->state_generation, old, ready) != old) { + /* A racing timeout may abandon only this exact generation. */ + old = atomic64_read_acquire(&slot->state_generation); + if (old == abandoned) + atomic64_cmpxchg_release(&slot->state_generation, + abandoned, free); + else if (old == committed) + atomic64_cmpxchg_release(&slot->state_generation, + committed, free); + atomic_inc(&shared->replies.late_replies); + return old == abandoned || old == committed ? -ESTALE : -EIO; + } + + raw_spin_lock_irqsave(&instance->control_route_lock, flags); + target = mk_instance_irq_route_load(instance); + if (target == MK_PHYS_CPU_INVALID) { + ret = -ENODEV; + goto unlock_route; + } + ret = 0; + mk_arch_send_ipi(target); + +unlock_route: + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); + return ret; +} + +int mk_reply_publish(struct mk_instance *instance, + const struct mk_reply_handle *reply, s32 status, u32 value) +{ + int ret; + + if (!instance) + return -EINVAL; + down_read(&instance->control_route_sem); + ret = mk_reply_publish_route_locked(instance, reply, status, value); + up_read(&instance->control_route_sem); + return ret; +} + +void mk_reply_scan(struct mk_shared_data *shared) +{ + unsigned int i; + + if (!shared) + return; + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &shared->replies.slots[i]; + u64 token = atomic64_read_acquire(&slot->state_generation); + + if (mk_reply_state(token) == MK_REPLY_READY) { + mk_reply_wake_all(); + return; + } + } +} + +struct mk_ipi_handler * +multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, + unsigned int ipi_type) { struct mk_ipi_handler *handler; unsigned long flags; if (!callback) return NULL; - handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler) return NULL; - handler->callback = callback; handler->context = ctx; handler->ipi_type = ipi_type; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); handler->next = mk_handlers; mk_handlers = handler; raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - return handler; } EXPORT_SYMBOL(multikernel_register_handler); -/** - * multikernel_unregister_handler - Unregister a multikernel IPI callback - * @handler: Handler pointer returned from multikernel_register_handler - */ void multikernel_unregister_handler(struct mk_ipi_handler *handler) { - struct mk_ipi_handler **pp, *p; + struct mk_ipi_handler **pp, *p = NULL; unsigned long flags; if (!handler) return; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); - pp = &mk_handlers; - while ((p = *pp) != NULL) { + for (pp = &mk_handlers; (p = *pp); pp = &p->next) { if (p == handler) { *pp = p->next; break; } - pp = &p->next; } raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - + if (!p) + return; + synchronize_srcu(&mk_ipi_srcu); kfree(p); } EXPORT_SYMBOL(multikernel_unregister_handler); -/* - * An instance's IPI area is allocated when its image is loaded; the - * instance pointer is filled in lazily on first use. - */ -static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) -{ - struct mk_shared_data *ipi_data; - - if (instance->ipi_data) - return instance->ipi_data; - - if (!instance->kimage || !instance->kimage->mk_ipi) - return NULL; - - ipi_data = phys_to_virt(instance->kimage->mk_ipi); - if (cmpxchg(&instance->ipi_data, NULL, ipi_data) == NULL) - pr_info("Initialized IPI ring buffer for instance %d: phys=0x%llx\n", - instance->id, (unsigned long long)instance->kimage->mk_ipi); - - return instance->ipi_data; -} - -/** - * mk_arm_force_halt - Post the force-halt marker for an instance - * @instance: Instance about to be NMIed - * - * The instance's CPUs test the marker from their NMI handlers, so it - * must be armed before the NMIs are sent. It stays armed until the - * kexec path has confirmed every CPU parked and wipes the shared area - * for the next run, which is what makes the NMI rescue idempotent: a - * repeat force halt still reaches CPUs an earlier one missed. - * - * Returns 0 on success, -ENODEV if the instance has no shared IPI area. - */ int mk_arm_force_halt(struct mk_instance *instance) { - struct mk_shared_data *ipi_data = mk_instance_ipi_area(instance); + struct mk_shared_data *shared = mk_instance_halt_data(instance); - if (!ipi_data) + if (!shared) return -ENODEV; - - WRITE_ONCE(ipi_data->force_halt, 1); - /* The marker must be visible before the NMIs that test it */ + WRITE_ONCE(shared->force_halt, 1); smp_wmb(); return 0; } -/** - * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID - * @data: Pointer to data to send - * @data_size: Size of data - * @type: User-defined type identifier - * - * This function enqueues data into the target instance's IPI ring buffer - * and sends an IPI to notify the target CPU. - * - * Returns 0 on success, negative error code on failure - */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { + struct mk_ipi_endpoint *endpoint; struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; + struct mk_shared_data *shared; mk_phys_cpu_t target; - - if (!instance) + unsigned long route_flags; + unsigned long tx_flags; + u32 idx; + u64 tx_epoch; + int ret = 0; + + if (!instance || data_size > MK_MAX_DATA_SIZE || + (data_size && !data)) return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); - return -EINVAL; - } - - target = instance->ipi_target; - if (target == MK_PHYS_CPU_INVALID) - target = mk_cpu_set_first(instance->cpus); + endpoint = &instance->ipi_endpoint; + if (!READ_ONCE(endpoint->registered)) + return -ESHUTDOWN; + tx_epoch = READ_ONCE(endpoint->tx_epoch); + raw_spin_lock_irqsave(&instance->control_route_lock, route_flags); + target = mk_instance_irq_route_load(instance); if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPU to receive the IPI\n", instance_id); - mk_instance_put(instance); - return -ENODEV; + ret = -ENODEV; + goto unlock_route; } - - if (!mk_instance_ipi_area(instance)) { - pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); - return -ENODEV; + raw_spin_lock_irqsave(&endpoint->tx_lock, tx_flags); + if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled || + tx_epoch != endpoint->tx_epoch) { + ret = -ESHUTDOWN; + goto unlock_endpoint; } - - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; - } - - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); - - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; - - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; - - if (data && data_size > 0) + shared = READ_ONCE(instance->ipi_data); + if (!shared) { + ret = -ENODEV; + goto unlock_endpoint; + } + if (endpoint->parent_side) + WRITE_ONCE(shared->child_doorbell_cpu, target); + idx = endpoint->tx_head & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->tx->entries[idx]; + /* Pair with the receiver's release when it makes the slot reusable. */ + if (smp_load_acquire(&slot->ready)) { + atomic_inc(&endpoint->tx->full_failures); + ret = -ENOSPC; + goto unlock_endpoint; + } + WRITE_ONCE(slot->sender_cpu, arch_cpu_physical_id(smp_processor_id())); + WRITE_ONCE(slot->type, type); + WRITE_ONCE(slot->data_size, data_size); + if (data_size) memcpy(slot->buffer, data, data_size); + /* Publish all message fields before the receiver observes readiness. */ + smp_store_release(&slot->ready, 1); + endpoint->tx_head++; + +unlock_endpoint: + raw_spin_unlock_irqrestore(&endpoint->tx_lock, tx_flags); + if (!ret || ret == -ENOSPC) + mk_arch_send_ipi(target); + +unlock_route: + raw_spin_unlock_irqrestore(&instance->control_route_lock, route_flags); + if (ret == -ENOSPC && __ratelimit(&mk_ipi_full_rs)) + printk_deferred(KERN_WARNING + "multikernel: IPI ring full for instance %d\n", + instance->id); + return ret; +} - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); - - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance; + int ret; + might_sleep(); + instance = mk_instance_find(instance_id); + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } -static void mk_ipi_drain_ring(void) +int multikernel_send_ipi_data_to_host(void *data, size_t data_size, + unsigned long type) { - struct mk_ipi_data *slot; + struct mk_instance *instance = READ_ONCE(host_instance); + + if (!instance) + return -ENODEV; + return mk_send_ipi_data(instance, data, data_size, type); +} +EXPORT_SYMBOL(multikernel_send_ipi_data_to_host); + +static void mk_ipi_dispatch(struct mk_ipi_endpoint *endpoint, + struct mk_ipi_data *slot) +{ + struct mk_instance *instance = container_of(endpoint, struct mk_instance, + ipi_endpoint); struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; + mk_ipi_callback_t callback = NULL; + void *context = NULL; + unsigned long flags; size_t data_size; - int messages_processed = 0; - if (!mk_self || !mk_self->ipi_data) + data_size = READ_ONCE(slot->data_size); + if (data_size > MK_MAX_DATA_SIZE) { + if (__ratelimit(&mk_ipi_bad_size_rs)) + printk_deferred(KERN_WARNING + "multikernel: oversized IPI payload (%zu bytes) for instance %d\n", + data_size, instance->id); return; - - while (1) { - tail = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.head)); - - if (tail == head) - break; - - slot = &mk_self->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + } + raw_spin_lock_irqsave(&mk_handlers_lock, flags); + for (handler = mk_handlers; handler; handler = handler->next) { + if (handler->ipi_type == READ_ONCE(slot->type)) { + callback = handler->callback; + context = handler->context; break; - - if (data_size > MK_MAX_DATA_SIZE) { - pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - continue; } + } + raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); + if (callback) + callback(slot, context); +} - /* Dispatch to registered handler */ - raw_spin_lock(&mk_handlers_lock); - for (handler = mk_handlers; handler; handler = handler->next) { - if (handler->ipi_type == slot->type && handler->callback) { - mk_ipi_callback_t cb = handler->callback; - void *ctx = handler->context; +static bool mk_ipi_drain_endpoint(struct mk_ipi_endpoint *endpoint) +{ + struct mk_ipi_data *slot; + unsigned long flags; + unsigned int drained = 0; + bool pending; + u32 idx; - raw_spin_unlock(&mk_handlers_lock); - cb(slot, ctx); - goto advance_tail; + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + if (endpoint->rx_dispatching) { + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return false; + } + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + /* Bound the SRCU read section so a busy link cannot starve its peers. */ + while (drained < MK_IPI_RING_SIZE) { + mk_reply_scan(endpoint->peer->ipi_data); + idx = endpoint->rx_tail & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->rx->entries[idx]; + /* Pair with the producer's release publication of this slot. */ + if (!smp_load_acquire(&slot->ready)) { + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_dispatching = false; + /* Close the empty-ring handoff race with a new publication. */ + if (smp_load_acquire(&slot->ready)) { + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + continue; } + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return false; } - raw_spin_unlock(&mk_handlers_lock); - -advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; + /* Authenticate provenance from the registered duplex endpoint. */ + WRITE_ONCE(slot->sender_instance_id, endpoint->peer->id); + mk_ipi_dispatch(endpoint, slot); + /* The callback must finish reading before the slot is reusable. */ + smp_store_release(&slot->ready, 0); + endpoint->rx_tail++; + drained++; } + + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_dispatching = false; + idx = endpoint->rx_tail & (MK_IPI_RING_SIZE - 1); + /* Pair with a producer that refilled the ring during this pass. */ + pending = smp_load_acquire(&endpoint->rx->entries[idx].ready); + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return pending; } -/** - * multikernel_interrupt_handler - Handle the multikernel IPI - * - * This function is called when a multikernel IPI is received. - * Messages are drained here, in interrupt context. - */ -static void multikernel_interrupt_handler(void) +static void mk_ipi_drain_workfn(struct work_struct *work); +static DECLARE_WORK(mk_ipi_drain_work, mk_ipi_drain_workfn); + +static void mk_ipi_drain_all(void) { - if (!mk_self || !mk_self->ipi_data) + struct mk_ipi_endpoint *endpoint; + bool pending = false; + int idx; + + if (!READ_ONCE(mk_handlers_ready)) return; + idx = srcu_read_lock(&mk_ipi_srcu); + list_for_each_entry_srcu(endpoint, &mk_ipi_endpoints, rx_node, + srcu_read_lock_held(&mk_ipi_srcu)) + pending |= mk_ipi_drain_endpoint(endpoint); + srcu_read_unlock(&mk_ipi_srcu, idx); + if (pending) + schedule_work(&mk_ipi_drain_work); +} - /* - * Drain here rather than from irq_work. We are already in interrupt - * context and every handler is safe to call from it, and irq_work - * brings a failure mode with it: the work is a single static - * instance, so if it is ever left pending - its self-IPI lost while - * the CPU was bringing its APIC up, say - every later queue attempt - * is a no-op and the ring never drains again. - */ - mk_ipi_drain_ring(); +void mk_ipi_handlers_enable(void) +{ + WRITE_ONCE(mk_handlers_ready, true); + mk_ipi_drain_all(); +} + +static void mk_ipi_drain_workfn(struct work_struct *work) +{ + mk_ipi_drain_all(); } -/** - * Generic multikernel interrupt handler - called by the IPI vector - * - * This is the function that gets called by the IPI vector handler. - */ void generic_multikernel_interrupt(void) { - multikernel_interrupt_handler(); + if (!READ_ONCE(mk_handlers_ready)) + return; + /* x86 vector IRQs must be injected from actual hardirq context. */ + if (mk_self && mk_self->ipi_data) + mk_pci_irq_mailbox_drain(mk_self->ipi_data); + schedule_work(&mk_ipi_drain_work); } /** diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index c8fa6270e1fe6b..a264b9ca8f95f2 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -125,6 +125,155 @@ static int status_seq_show(struct seq_file *sf, void *v) return 0; } +/* + * This is an observational snapshot, not a transaction: producers may update + * fields while the file is read. Shared transport counters are u32 values + * and restart with a new spawn_epoch, so consumers compare modulo-u32 deltas + * only between samples from the same epoch. + */ +static int stats_seq_show(struct seq_file *sf, void *v) +{ + struct kernfs_open_file *of = sf->private; + struct mk_instance *instance = of->kn->priv; + struct mk_shared_data *shared; + struct { + u64 spawn_epoch; + u64 irq_pending_total; + u32 ipi_producer_contention; + u32 ipi_full_failures; + u32 ipi_invalid_state; + u32 ipi_cancelled_writes; + u32 reply_late_replies; + u32 reply_cancelled_slots; + u32 reply_atomic_timeouts; + u32 reply_indeterminate_timeouts; + u32 reply_occupied_failures; + u32 irq_recorded; + u32 irq_coalesced; + u32 irq_masked_deferred; + u32 irq_stale; + u32 irq_dispatch_failed; + u32 irq_saturated; + u32 ipi_slots[4]; + u32 reply_states[MK_REPLY_ABANDONED + 1]; + u32 irq_active; + u32 irq_pending; + u32 irq_masked; + u32 irq_consuming; + u32 reply_busy; + u32 ipi_gate_busy; + bool available; + } stats = {}; + unsigned int i; + + down_read(&instance->control_route_sem); + shared = READ_ONCE(instance->ipi_data); + if (!shared) + goto unlock; + stats.available = true; + stats.spawn_epoch = READ_ONCE(shared->spawn_epoch); + stats.ipi_full_failures = + (u32)atomic_read(&shared->to_child.full_failures) + + (u32)atomic_read(&shared->to_parent.full_failures); + stats.reply_late_replies = atomic_read(&shared->replies.late_replies); + stats.reply_cancelled_slots = atomic_read(&shared->replies.cancelled_slots); + stats.reply_atomic_timeouts = atomic_read(&shared->replies.atomic_timeouts); + stats.reply_indeterminate_timeouts = + atomic_read(&shared->replies.indeterminate_timeouts); + stats.reply_occupied_failures = + atomic_read(&shared->replies.occupied_failures); + stats.irq_recorded = atomic_read(&shared->irq_mailbox.recorded); + stats.irq_coalesced = atomic_read(&shared->irq_mailbox.coalesced); + stats.irq_masked_deferred = + atomic_read(&shared->irq_mailbox.masked_deferred); + stats.irq_stale = atomic_read(&shared->irq_mailbox.stale); + stats.irq_dispatch_failed = + atomic_read(&shared->irq_mailbox.dispatch_failed); + stats.irq_saturated = atomic_read(&shared->irq_mailbox.saturated); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + /* Pair with the producer's release publication. */ + if (smp_load_acquire(&shared->to_child.entries[i].ready)) + stats.ipi_slots[1]++; + /* Pair with the producer's release publication. */ + if (smp_load_acquire(&shared->to_parent.entries[i].ready)) + stats.ipi_slots[1]++; + } + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &shared->replies.slots[i]; + u64 token = atomic64_read_acquire(&slot->state_generation); + u32 state = token & (BIT(MK_REPLY_STATE_BITS) - 1); + + if (state <= MK_REPLY_ABANDONED) + stats.reply_states[state]++; + if (state != MK_REPLY_FREE) + stats.reply_busy++; + } + for (i = 0; i < MK_IRQ_MAILBOX_SLOTS; i++) { + struct mk_irq_mailbox_entry *entry = + &shared->irq_mailbox.entries[i]; + u64 token = atomic64_read_acquire(&entry->pending_generation); + + if (!mk_irq_mailbox_generation(token)) + continue; + stats.irq_active++; + if (mk_irq_mailbox_pending(token)) { + stats.irq_pending++; + stats.irq_pending_total += mk_irq_mailbox_pending(token); + } + if (mk_irq_mailbox_masked(token)) + stats.irq_masked++; + if (mk_irq_mailbox_consuming(token)) + stats.irq_consuming++; + } +unlock: + up_read(&instance->control_route_sem); + + seq_puts(sf, "stats_version 1\n"); + seq_printf(sf, "abi_version %u\n", MK_BOOT_CONTEXT_MAGIC & 0xffff); + seq_puts(sf, "snapshot_atomic 0\n"); + seq_puts(sf, "transport_counter_bits 32\n"); + seq_printf(sf, "transport_available %u\n", stats.available); + seq_printf(sf, "spawn_epoch %llu\n", + (unsigned long long)stats.spawn_epoch); +#define MK_STATS_VALUE(name, value) seq_printf(sf, name " %u\n", value) + MK_STATS_VALUE("ipi.producer_contention", stats.ipi_producer_contention); + MK_STATS_VALUE("ipi.full_failures", stats.ipi_full_failures); + MK_STATS_VALUE("ipi.invalid_state", stats.ipi_invalid_state); + MK_STATS_VALUE("ipi.cancelled_writes", stats.ipi_cancelled_writes); + MK_STATS_VALUE("ipi.gate_busy", stats.ipi_gate_busy); + MK_STATS_VALUE("ipi.slot_writing", stats.ipi_slots[0]); + MK_STATS_VALUE("ipi.slot_ready", stats.ipi_slots[1]); + MK_STATS_VALUE("ipi.slot_consuming", stats.ipi_slots[2]); + MK_STATS_VALUE("ipi.slot_cancelled", stats.ipi_slots[3]); + MK_STATS_VALUE("reply.late_replies", stats.reply_late_replies); + MK_STATS_VALUE("reply.cancelled_slots", stats.reply_cancelled_slots); + MK_STATS_VALUE("reply.atomic_timeouts", stats.reply_atomic_timeouts); + MK_STATS_VALUE("reply.indeterminate_timeouts", + stats.reply_indeterminate_timeouts); + MK_STATS_VALUE("reply.occupied_failures", stats.reply_occupied_failures); + MK_STATS_VALUE("reply.slot_busy", stats.reply_busy); + MK_STATS_VALUE("reply.slot_reserved", stats.reply_states[MK_REPLY_RESERVED]); + MK_STATS_VALUE("reply.slot_writing", stats.reply_states[MK_REPLY_WRITING]); + MK_STATS_VALUE("reply.slot_executing", stats.reply_states[MK_REPLY_EXECUTING]); + MK_STATS_VALUE("reply.slot_committed", stats.reply_states[MK_REPLY_COMMITTED]); + MK_STATS_VALUE("reply.slot_ready", stats.reply_states[MK_REPLY_READY]); + MK_STATS_VALUE("reply.slot_abandoned", stats.reply_states[MK_REPLY_ABANDONED]); + MK_STATS_VALUE("irq.recorded", stats.irq_recorded); + MK_STATS_VALUE("irq.coalesced", stats.irq_coalesced); + MK_STATS_VALUE("irq.masked_deferred", stats.irq_masked_deferred); + MK_STATS_VALUE("irq.stale", stats.irq_stale); + MK_STATS_VALUE("irq.dispatch_failed", stats.irq_dispatch_failed); + MK_STATS_VALUE("irq.saturated", stats.irq_saturated); + MK_STATS_VALUE("irq.slot_active", stats.irq_active); + MK_STATS_VALUE("irq.slot_pending", stats.irq_pending); + MK_STATS_VALUE("irq.slot_masked", stats.irq_masked); + MK_STATS_VALUE("irq.slot_consuming", stats.irq_consuming); +#undef MK_STATS_VALUE + seq_printf(sf, "irq.pending_total %llu\n", + (unsigned long long)stats.irq_pending_total); + return 0; +} + /* Root-level device_tree attribute - shows current baseline pool from kernel structures */ static int root_device_tree_seq_show(struct seq_file *sf, void *v) { @@ -240,6 +389,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, struct mk_instance *instance; struct kernfs_node *kn; struct mk_dt_config config; + int release_ret; int ret; int allocated_id; @@ -310,7 +460,16 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, return 0; err_free_resources: - mk_instance_free_memory(instance); + release_ret = mk_instance_release_resources(instance); + if (release_ret) { + pr_crit("Retaining failed instance '%s' because PCI cleanup failed: %d\n", + name, release_ret); + list_add_tail(&instance->list, &mk_instance_list); + kernfs_activate(kn); + mk_instance_set_state(instance, MK_STATE_FAILED); + mk_dt_config_free(&config); + return release_ret; + } err_free_idr: idr_remove(&mk_instance_idr, instance->id); err_remove_dir: @@ -332,6 +491,10 @@ static const struct kernfs_ops mk_status_ops = { .seq_show = status_seq_show, }; +static const struct kernfs_ops mk_stats_ops = { + .seq_show = stats_seq_show, +}; + /* Root-level device_tree operations */ static const struct kernfs_ops mk_root_device_tree_ops = { .seq_show = root_device_tree_seq_show, @@ -366,6 +529,13 @@ static int mk_create_instance_files(struct mk_instance *instance) pr_err("Failed to create status file for instance %s\n", instance->name); return PTR_ERR(kn); } + kn = __kernfs_create_file(instance->kn, "stats", 0444, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, + &mk_stats_ops, instance, NULL, NULL); + if (IS_ERR(kn)) { + pr_err("Failed to create stats file for instance %s\n", instance->name); + return PTR_ERR(kn); + } kn = __kernfs_create_file(instance->kn, "device_tree", 0444, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, @@ -407,6 +577,8 @@ static int mk_kernfs_rmdir(struct kernfs_node *kn) */ int mk_instance_destroy(struct mk_instance *instance) { + int ret; + lockdep_assert_held(&mk_instance_mutex); if (!instance) { @@ -426,6 +598,14 @@ int mk_instance_destroy(struct mk_instance *instance) return -EBUSY; } + ret = mk_instance_release_resources(instance); + if (ret) { + pr_crit("Cannot remove instance '%s' while PCI cleanup is unsafe: %d\n", + instance->name, ret); + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; + } + list_del(&instance->list); idr_remove(&mk_instance_idr, instance->id); if (instance->kn) { diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index c5f97fbeea75fc..687d3712729b1b 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -28,12 +28,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -55,6 +61,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -73,14 +80,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /* @@ -108,7 +118,7 @@ static int mk_manifest_collect_cpus(struct mk_instance *target, unsigned int i; int ret = 0; - mutex_lock(&mk_instance_mutex); + lockdep_assert_held(&mk_instance_mutex); list_for_each_entry(other, &mk_instance_list, list) { if (other == target) continue; @@ -127,7 +137,6 @@ static int mk_manifest_collect_cpus(struct mk_instance *target, break; } } - mutex_unlock(&mk_instance_mutex); /* * The spawn assigns logical CPU ids in this list's order, and the @@ -229,7 +238,7 @@ static int mk_manifest_add_reserved(void *fdt, struct kimage *image) if (!ret && image->mk_manifest) ret = mk_reserved_add(&r, image->mk_manifest, MK_MANIFEST_SIZE); - mutex_lock(&mk_instance_mutex); + lockdep_assert_held(&mk_instance_mutex); list_for_each_entry(other, &mk_instance_list, list) { if (ret) break; @@ -241,7 +250,6 @@ static int mk_manifest_add_reserved(void *fdt, struct kimage *image) ret = mk_reserved_add(&r, other->kimage->mk_manifest, MK_MANIFEST_SIZE); } - mutex_unlock(&mk_instance_mutex); if (!ret) { int pairs = mk_pool_park_regions(r.pair + 2 * r.n, MK_RESERVED_PAIRS - r.n); @@ -267,6 +275,7 @@ static int mk_manifest_add_reserved(void *fdt, struct kimage *image) struct mk_manifest_ctx { struct kimage *image; struct mk_instance *instance; + mk_phys_cpu_t parent_doorbell_cpu; }; static int mk_manifest_chosen(void *fdt, void *data) @@ -289,7 +298,7 @@ static int mk_manifest_chosen(void *fdt, void *data) /* The boot CPU can never join the pool, so it stays ours */ if (!ret) ret = fdt_property_u64(fdt, "multikernel,host-ipi-cpu", - arch_cpu_physical_id(0)); + ctx->parent_doorbell_cpu); if (ret) return ret; } @@ -323,15 +332,20 @@ static int mk_manifest_chosen(void *fdt, void *data) /** * mk_manifest_finalize - Write the boot tree for a spawn * @image: The multikernel kimage being executed + * @parent_doorbell_cpu: Physical CPU that receives child-to-parent IPIs * * Generates the instance's device tree into the manifest page allocated * at load time, with /chosen carrying the boot handoff. * * Returns: 0 on success, negative error code on failure */ -int mk_manifest_finalize(struct kimage *image) +int mk_manifest_finalize(struct kimage *image, + mk_phys_cpu_t parent_doorbell_cpu) { - struct mk_manifest_ctx ctx = { .image = image }; + struct mk_manifest_ctx ctx = { + .image = image, + .parent_doorbell_cpu = parent_doorbell_cpu, + }; struct mk_instance *instance; void *fdt; int ret; @@ -341,13 +355,20 @@ int mk_manifest_finalize(struct kimage *image) return -EINVAL; } + if (parent_doorbell_cpu == MK_PHYS_CPU_INVALID) + return -EINVAL; + if (!image->mk_manifest) { pr_err("No manifest page allocated for multikernel kimage\n"); return -EINVAL; } - instance = mk_instance_find(image->mk_id); - if (!instance) { + /* + * A multikernel image owns an instance reference until kimage_free(). + * Reuse it here so callers may retain the established instance locks. + */ + instance = image->mk_instance; + if (!instance || instance->id != image->mk_id) { pr_err("Target multikernel instance %d not found\n", image->mk_id); return -ENOENT; } @@ -359,12 +380,10 @@ int mk_manifest_finalize(struct kimage *image) if (ret) { pr_err("Failed to write the boot tree for instance %d: %d\n", image->mk_id, ret); - mk_instance_put(instance); return ret == -FDT_ERR_NOSPACE ? -ENOSPC : ret; } pr_info("multikernel: boot tree for instance %d written (%u bytes)\n", image->mk_id, fdt_totalsize(fdt)); - mk_instance_put(instance); return 0; } diff --git a/kernel/multikernel/mem.c b/kernel/multikernel/mem.c index 2cd43fa2f5b239..22631d0462067f 100644 --- a/kernel/multikernel/mem.c +++ b/kernel/multikernel/mem.c @@ -437,17 +437,30 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size, phys_addr_t phys_addr; int ret; + if (!instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot add memory to instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + phys_addr = multikernel_alloc(size, node); if (!phys_addr) { pr_err("Failed to allocate %zu bytes from multikernel pool for instance %d\n", size, instance->id); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region = kzalloc(sizeof(*region), GFP_KERNEL); if (!region) { multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.name = kasprintf(GFP_KERNEL, "mk-instance-%d-%s-region-%d", @@ -455,7 +468,8 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size, if (!region->res.name) { kfree(region); multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.start = phys_addr; @@ -471,7 +485,7 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size, kfree(region->res.name); kfree(region); multikernel_free(phys_addr, size); - return ret; + goto out_unlock; } INIT_LIST_HEAD(®ion->list); @@ -482,7 +496,10 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size, (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), size >> 20, instance->id, instance->name); - return 0; + ret = 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /* Does [phys_addr, phys_addr+size) back a segment of the loaded image? */ @@ -524,16 +541,26 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, { struct mk_memory_region *region, *tmp; bool found = false; + int ret; if (!instance) return -EINVAL; + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot remove memory from instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + if (mk_range_backs_kimage(instance, phys_addr, size)) { pr_err("Refusing to remove 0x%llx-0x%llx from instance %d (%s): the loaded kernel image lives there\n", (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), instance->id, instance->name); - return -EBUSY; + ret = -EBUSY; + goto out_unlock; } list_for_each_entry_safe(region, tmp, &instance->memory_regions, list) { @@ -563,10 +590,14 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), instance->id, instance->name); - return -ENOENT; + ret = -ENOENT; + } else { + ret = 0; } - return 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /** diff --git a/kernel/multikernel/messaging.c b/kernel/multikernel/messaging.c index 2cd2b7682f8d3e..c56434c9740084 100644 --- a/kernel/multikernel/messaging.c +++ b/kernel/multikernel/messaging.c @@ -11,6 +11,8 @@ #include #include #include +#include +#include "internal.h" /* Pending message tracking for request-response pattern */ struct mk_pending_msg { @@ -19,11 +21,12 @@ struct mk_pending_msg { u64 resource_id; /* Resource identifier (physical CPU ID, PFN, etc.) */ int result; /* Operation result */ struct completion done; /* Completion for waiting */ + struct work_struct complete_work; struct list_head list; /* List linkage */ }; static LIST_HEAD(mk_pending_msgs); -static DEFINE_SPINLOCK(mk_pending_msgs_lock); +static DEFINE_RAW_SPINLOCK(mk_pending_msgs_lock); /* Per-type message handler registry */ struct mk_msg_type_handler { @@ -86,13 +89,23 @@ static void mk_message_type_ipi_callback(struct mk_ipi_data *data, void *ctx) msg_type, msg_subtype, payload_len, data->sender_cpu); /* Call the registered handler for this message type */ - type_handler->msg_handler(msg_type, msg_subtype, payload, payload_len, type_handler->context); + type_handler->msg_handler(msg_type, msg_subtype, payload, payload_len, + data->sender_instance_id, + type_handler->context); } /* * Pending message tracking for request-response pattern */ +static void mk_msg_pending_complete_workfn(struct work_struct *work) +{ + struct mk_pending_msg *pending = + container_of(work, struct mk_pending_msg, complete_work); + + complete(&pending->done); +} + /** * mk_msg_pending_add - Register a pending operation awaiting response * @msg_type: Message type @@ -115,10 +128,11 @@ struct mk_pending_msg *mk_msg_pending_add(u32 msg_type, u32 operation, u64 resou pending->resource_id = resource_id; pending->result = -ETIMEDOUT; /* Default to timeout */ init_completion(&pending->done); + INIT_WORK(&pending->complete_work, mk_msg_pending_complete_workfn); - spin_lock_irqsave(&mk_pending_msgs_lock, flags); + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); list_add(&pending->list, &mk_pending_msgs); - spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); return pending; } @@ -137,17 +151,17 @@ void mk_msg_pending_complete(u32 msg_type, u32 operation, u64 resource_id, int r struct mk_pending_msg *pending; unsigned long flags; - spin_lock_irqsave(&mk_pending_msgs_lock, flags); + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); list_for_each_entry(pending, &mk_pending_msgs, list) { if (pending->msg_type == msg_type && pending->operation == operation && pending->resource_id == resource_id) { pending->result = result; - complete(&pending->done); + schedule_work(&pending->complete_work); break; } } - spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); } /** @@ -161,27 +175,29 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms { unsigned long timeout = msecs_to_jiffies(timeout_ms); unsigned long flags; + bool timed_out; int result; - if (!wait_for_completion_timeout(&pending->done, timeout)) { + timed_out = !wait_for_completion_timeout(&pending->done, timeout); + if (timed_out) { pr_err("Timeout waiting for operation 0x%x on resource %llu\n", pending->operation, pending->resource_id); - result = -ETIMEDOUT; - } else { - result = pending->result; } /* Remove from list and free */ - spin_lock_irqsave(&mk_pending_msgs_lock, flags); + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); list_del(&pending->list); - spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + result = timed_out ? -ETIMEDOUT : pending->result; + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + cancel_work_sync(&pending->complete_work); kfree(pending); return result; } /** - * mk_send_message - Send a message to another multikernel instance + * __mk_send_message - Send a message to another multikernel instance + * @instance: Lifetime-stable target instance, or NULL for ID lookup * @instance_id: Target multikernel instance ID * @msg_type: Message type identifier * @subtype: Message subtype @@ -190,8 +206,9 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms * * Returns 0 on success, negative error code on failure */ -int mk_send_message(int instance_id, u32 msg_type, u32 subtype, - void *payload, u32 payload_len) +static int __mk_send_message(struct mk_instance *instance, int instance_id, + u32 msg_type, u32 subtype, void *payload, + u32 payload_len) { struct mk_message *msg; size_t total_size; @@ -202,8 +219,9 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, /* Check if message fits in IPI buffer */ if (total_size > MK_MAX_DATA_SIZE) { - pr_err("Multikernel message too large: %zu > %d bytes\n", - total_size, MK_MAX_DATA_SIZE); + printk_deferred(KERN_ERR + "Multikernel message too large: %zu > %d bytes\n", + total_size, MK_MAX_DATA_SIZE); return -EMSGSIZE; } @@ -223,21 +241,40 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, memcpy(msg->payload, payload, payload_len); /* Send via IPI using the message type as IPI type */ - ret = multikernel_send_ipi_data(instance_id, msg, total_size, msg_type); + if (instance) + ret = mk_send_ipi_data(instance, msg, total_size, msg_type); + else + ret = multikernel_send_ipi_data(instance_id, msg, total_size, + msg_type); /* Clean up temporary buffer */ kfree(msg); if (ret < 0) { - pr_err("Failed to send multikernel message: %d\n", ret); + printk_deferred(KERN_ERR + "Failed to send multikernel message: %d\n", ret); return ret; } - pr_debug("Multikernel message sent: type=0x%x, subtype=0x%x, len=%u to instance %d\n", - msg_type, subtype, payload_len, instance_id); - return 0; } + +int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, + u32 subtype, void *payload, u32 payload_len) +{ + if (!instance) + return -EINVAL; + return __mk_send_message(instance, instance->id, + msg_type, subtype, payload, payload_len); +} + +int mk_send_message(int instance_id, u32 msg_type, u32 subtype, + void *payload, u32 payload_len) +{ + might_sleep(); + return __mk_send_message(NULL, instance_id, + msg_type, subtype, payload, payload_len); +} EXPORT_SYMBOL(mk_send_message); /** diff --git a/kernel/multikernel/overlay.c b/kernel/multikernel/overlay.c index 3ee26597620c57..e718ee3ab7ddc3 100644 --- a/kernel/multikernel/overlay.c +++ b/kernel/multikernel/overlay.c @@ -565,9 +565,9 @@ static int mk_overlay_parse_pci_id(const void *fdt, int item_node, const char *name, u16 *domain, u8 *bus, u8 *devfn) { - unsigned int d, b, slot, func; + u8 slot, func; const char *pci_id_str; - int len; + int len, ret; pci_id_str = fdt_getprop(fdt, item_node, "pci-id", &len); if (!pci_id_str || len <= 0 || pci_id_str[len - 1] != '\0') { @@ -575,13 +575,11 @@ static int mk_overlay_parse_pci_id(const void *fdt, int item_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &d, &b, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, domain, bus, &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%s'\n", pci_id_str); + return ret; } - - *domain = (u16)d; - *bus = (u8)b; *devfn = PCI_DEVFN(slot, func); return 0; } @@ -1302,8 +1300,24 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, const void *fdt) for (i = 0; i < nr; i++) { ret = mk_overlay_apply_fragment(tx, fdt, frags[i].node); - if (ret) + if (ret) { + int rollback_ret; + + /* + * A later fragment may depend on state created by an earlier + * one. Undo every completed fragment before reporting the + * transaction failure so the visible state stays atomic. + */ + while (--i >= 0) { + rollback_ret = + mk_overlay_rollback_fragment(tx, fdt, + frags[i].node); + if (rollback_ret) + pr_err("Overlay tx%d: failed to undo fragment@%x: %d\n", + tx->id, frags[i].unit, rollback_ret); + } break; + } } kfree(frags); diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c new file mode 100644 index 00000000000000..7a4f2a7c173556 --- /dev/null +++ b/kernel/multikernel/pci.c @@ -0,0 +1,2685 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Multikernel PCI assignment policy + * + * Keeps assigned-device discovery, identity presentation, bridge traversal, + * and resource restoration independent from the manifest that populated + * the current instance. + * + * Configuration-space filtering constrains normal PCI access by a cooperative + * spawn kernel. It is not a security boundary: a privileged kernel can issue + * configuration cycles or map physical configuration windows directly. The + * host-owned IOMMU domain separately constrains DMA initiated by an assigned + * device. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct mk_pci_assignment; + +#define MK_PCI_FLR_SETTLE_MS 100 +#define MK_PCI_MAILBOX_QUIESCE_MS 1000 + +static unsigned long mk_pci_mailbox_deadline(void) +{ + return jiffies + msecs_to_jiffies(MK_PCI_MAILBOX_QUIESCE_MS); +} + +struct mk_pci_irq_vector { + struct mk_pci_assignment *assignment; + unsigned int host_irq; + u32 local_irq; + u32 mailbox_slot; + u32 mailbox_generation; + atomic64_t forwarded; + bool requested; + bool disabled; +}; + +struct mk_pci_assignment { + struct list_head instance_node; + struct list_head active_node; + struct list_head transaction_node; + struct mk_instance *instance; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + const struct device_driver *host_driver; + struct iommu_group *iommu_group; + struct iommu_domain *iommu_domain; + char *host_driver_override; + struct mutex iommu_mutex; /* Serializes IOMMU activation and teardown. */ + unsigned int iommu_mapped_regions; + bool iommu_dma_owner; + bool iommu_attached; + bool iommu_override_active; + struct mk_pci_irq_vector *irq_vectors; + unsigned long *irq_bound_map; + unsigned int nr_irq_vectors; + u64 irq_epoch; + u32 irq_generation; + u32 reset_generation; + u8 irq_state; + bool irq_msix; + bool irq_needs_reprogram; + bool irq_mailbox_failed; + unsigned long irq_flr_deadline; + bool device_enabled; + struct work_struct failure_work; + atomic_t failure_pending; + bool assigned; + bool inventory_moved; + bool expected_unbind; +}; + +static DEFINE_MUTEX(mk_pci_lease_mutex); +static DEFINE_SPINLOCK(mk_pci_active_lock); +static LIST_HEAD(mk_pci_active_assignments); +static bool mk_pci_notifier_registered; +static bool mk_pci_control_registered; +static struct mk_pci_control_work *mk_pci_control_pool; +static unsigned long *mk_pci_control_pool_busy; +static unsigned int mk_pci_control_pool_size; +static atomic_t mk_pci_control_pool_cursor = ATOMIC_INIT(0); +static struct workqueue_struct *mk_pci_control_wq; +static DECLARE_WAIT_QUEUE_HEAD(mk_pci_control_waitq); +#define MK_PCI_CONTROL_SHUTDOWN BIT(30) +#define MK_PCI_CONTROL_ACTIVE_MASK (MK_PCI_CONTROL_SHUTDOWN - 1) +static atomic_t mk_pci_control_state = ATOMIC_INIT(MK_PCI_CONTROL_SHUTDOWN); +static atomic64_t mk_pci_control_pool_exhausted = ATOMIC64_INIT(0); +static DEFINE_RATELIMIT_STATE(mk_pci_control_pool_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + +struct mk_pci_control_work { + struct work_struct work; + work_func_t dispatch; + unsigned int pool_slot; + s32 sender_instance_id; + union { + struct mk_pci_cfg_request cfg; + struct mk_pci_irq_request irq; + struct mk_pci_reset_request reset; + } request; +}; + +static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment); + +static void mk_pci_control_wake_workfn(struct work_struct *work) +{ + wake_up_all(&mk_pci_control_waitq); +} + +static DECLARE_WORK(mk_pci_control_wake_work, + mk_pci_control_wake_workfn); + +static bool mk_pci_control_handler_get(void) +{ + int state = atomic_read(&mk_pci_control_state); + + for (;;) { + if (state & MK_PCI_CONTROL_SHUTDOWN) + return false; + if ((state & MK_PCI_CONTROL_ACTIVE_MASK) == + MK_PCI_CONTROL_ACTIVE_MASK) + return false; + if (atomic_try_cmpxchg(&mk_pci_control_state, &state, state + 1)) + return true; + } +} + +static void mk_pci_control_handler_put(void) +{ + int state = atomic_read(&mk_pci_control_state); + int new_state; + + for (;;) { + if (!(state & MK_PCI_CONTROL_ACTIVE_MASK)) + return; + new_state = state - 1; + if (atomic_try_cmpxchg(&mk_pci_control_state, &state, new_state)) + break; + } + if (new_state == MK_PCI_CONTROL_SHUTDOWN) + schedule_work(&mk_pci_control_wake_work); +} + +static void mk_pci_control_shutdown_begin(void) +{ + atomic_or(MK_PCI_CONTROL_SHUTDOWN, &mk_pci_control_state); +} + +static void mk_pci_control_shutdown_end(void) +{ + WARN_ON_ONCE(atomic_read(&mk_pci_control_state) != + MK_PCI_CONTROL_SHUTDOWN); + atomic_set(&mk_pci_control_state, 0); +} + +static struct mk_pci_control_work *mk_pci_control_work_get(void) +{ + unsigned int start; + unsigned int slot; + unsigned int i; + + start = (unsigned int)atomic_inc_return(&mk_pci_control_pool_cursor); + for (i = 0; i < mk_pci_control_pool_size; i++) { + slot = (start + i) % mk_pci_control_pool_size; + if (!test_and_set_bit_lock(slot, mk_pci_control_pool_busy)) + return &mk_pci_control_pool[slot]; + } + return NULL; +} + +static void +mk_pci_control_work_put(struct mk_pci_control_work *control_work) +{ + clear_bit_unlock(control_work->pool_slot, mk_pci_control_pool_busy); +} + +static bool mk_pci_device_live(struct pci_dev *pdev) +{ + return device_is_registered(&pdev->dev) && + !pci_dev_is_disconnected(pdev) && + pci_device_is_present(pdev); +} + +static int mk_pci_forwarding_cpu(void) +{ + /* The host IPI manifest publishes logical CPU 0's physical ID. */ + if (!cpu_online(0)) + return -ENODEV; + + return 0; +} + +static bool mk_pci_device_matches_bdf(const struct mk_pci_device *device, + u16 domain, u8 bus, u8 devfn) +{ + return device->domain == domain && device->bus == bus && + device->slot == PCI_SLOT(devfn) && + device->func == PCI_FUNC(devfn); +} + +static struct mk_pci_device * +mk_pci_find_device_bdf(struct list_head *devices, u16 domain, u8 bus, u8 devfn) +{ + struct mk_pci_device *device; + + list_for_each_entry(device, devices, list) { + if (mk_pci_device_matches_bdf(device, domain, bus, devfn)) + return device; + } + + return NULL; +} + +static bool mk_pci_inventory_matches(const struct mk_pci_device *left, + const struct mk_pci_device *right) +{ + return left->vendor == right->vendor && + left->device == right->device && + mk_pci_device_matches_bdf(left, right->domain, right->bus, + PCI_DEVFN(right->slot, right->func)); +} + +static struct mk_pci_device * +mk_pci_find_root_inventory(const struct mk_pci_device *requested) +{ + struct mk_pci_device *device; + + list_for_each_entry(device, &mk_self->pci_devices, list) { + if (mk_pci_inventory_matches(device, requested)) + return device; + } + + return NULL; +} + +static struct mk_pci_device * +mk_pci_find_root_bdf(u16 domain, u8 bus, u8 devfn) +{ + return mk_pci_find_device_bdf(&mk_self->pci_devices, domain, bus, + devfn); +} + +static struct mk_pci_assignment * +mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + if (mk_pci_device_matches_bdf(assignment->inventory, domain, bus, + devfn)) + return assignment; + } + + return NULL; +} + +static void mk_pci_mailbox_clear_bit(struct mk_irq_mailbox *mailbox, + unsigned int slot) +{ + atomic64_andnot(BIT_ULL(slot & 63), + &mailbox->pending_bitmap[slot / 64]); +} + +static bool mk_pci_mailbox_quiesce(struct mk_pci_irq_vector *vector, + bool parked_force, + unsigned long deadline) +{ + struct mk_shared_data *shared = vector->assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u64 base, token; + + if (!shared || vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return true; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + base = mk_irq_mailbox_token(vector->mailbox_generation, 0) | + MK_IRQ_MAILBOX_MASKED; + for (;;) { + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return false; + } + if (mk_irq_mailbox_consuming(token)) { + if (parked_force) { + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, base) == token) + break; + continue; + } + if (!mk_irq_mailbox_masked(token)) + atomic64_cmpxchg(&entry->pending_generation, + token, + token | MK_IRQ_MAILBOX_MASKED); + if (time_after_eq(jiffies, deadline)) + return false; + usleep_range(50, 100); + continue; + } + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, base) == token) + break; + cpu_relax(); + } + mk_pci_mailbox_clear_bit(mailbox, vector->mailbox_slot); + WRITE_ONCE(entry->local_irq, 0); + return true; +} + +static void mk_pci_mailbox_release(struct mk_pci_irq_vector *vector) +{ + struct mk_shared_data *shared = vector->assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u64 token; + + if (!shared || vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + for (;;) { + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return; + } + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, 0) == token) + break; + cpu_relax(); + } + mk_pci_mailbox_clear_bit(mailbox, vector->mailbox_slot); + WRITE_ONCE(entry->lifecycle_epoch, 0); + WRITE_ONCE(entry->lifecycle_generation, 0); + WRITE_ONCE(entry->device_id, 0); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, 0); + vector->mailbox_slot = MK_IRQ_MAILBOX_SLOT_INVALID; + vector->mailbox_generation = 0; +} + +static int mk_pci_mailbox_reserve(struct mk_pci_irq_vector *vector, + unsigned int vector_index) +{ + struct mk_pci_assignment *assignment = vector->assignment; + struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u32 generation; + u32 device_id; + unsigned int slot; + u64 base; + + if (!shared) + return -ENODEV; + mailbox = &shared->irq_mailbox; + generation = (u32)atomic_inc_return(&mailbox->next_generation); + if (!generation) + generation = (u32)atomic_inc_return(&mailbox->next_generation); + if (!generation) + return -EOVERFLOW; + device_id = MK_PCI_IRQ_ID(pci_domain_nr(assignment->vf->bus), + assignment->vf->bus->number, + assignment->vf->devfn); + base = mk_irq_mailbox_token(generation, 0) | MK_IRQ_MAILBOX_MASKED; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + entry = &mailbox->entries[slot]; + if (atomic64_read(&entry->pending_generation)) + continue; + if (atomic64_cmpxchg_release(&entry->pending_generation, 0, + base)) + continue; + WRITE_ONCE(entry->lifecycle_epoch, assignment->irq_epoch); + WRITE_ONCE(entry->lifecycle_generation, + assignment->irq_generation); + WRITE_ONCE(entry->device_id, device_id); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, vector_index); + vector->mailbox_slot = slot; + vector->mailbox_generation = generation; + return 0; + } + return -ENOSPC; +} + +static void mk_pci_irq_retry_workfn(struct work_struct *work) +{ + struct mk_instance *instance = container_of(to_delayed_work(work), + struct mk_instance, + irq_retry_work); + struct mk_shared_data *shared; + bool any_pending = false; + bool unmasked_pending = false; + mk_phys_cpu_t target; + unsigned long flags; + unsigned int slot; + + raw_spin_lock_irqsave(&instance->control_route_lock, flags); + shared = READ_ONCE(instance->ipi_data); + if (!shared) + goto unlock; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry; + u64 token; + + entry = &shared->irq_mailbox.entries[slot]; + token = atomic64_read_acquire(&entry->pending_generation); + + if (!mk_irq_mailbox_generation(token)) + continue; + if (mk_irq_mailbox_pending(token)) { + any_pending = true; + atomic64_or(BIT_ULL(slot & 63), + &shared->irq_mailbox.pending_bitmap[slot / 64]); + if (!mk_irq_mailbox_masked(token)) + unmasked_pending = true; + } else if (mk_irq_mailbox_consuming(token)) { + any_pending = true; + } + } + if (unmasked_pending) { + target = mk_instance_irq_route_load(instance); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } +unlock: + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); + if (any_pending) + mod_delayed_work(system_wq, &instance->irq_retry_work, + msecs_to_jiffies(unmasked_pending ? 10 : 100)); +} + +static irqreturn_t mk_pci_forward_irq(int irq, void *data) +{ + struct mk_pci_irq_vector *vector = data; + struct mk_pci_assignment *assignment = vector->assignment; + struct mk_instance *instance = assignment->instance; + struct mk_shared_data *shared; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + bool first_forward = false; + bool schedule_retry = false; + mk_phys_cpu_t target; + u64 old, new; + u32 pending; + u32 local_irq; + unsigned long flags; + + /* Pair with BIND's publication of the shared mailbox identity. */ + local_irq = smp_load_acquire(&vector->local_irq); + + raw_spin_lock_irqsave(&instance->control_route_lock, flags); + shared = READ_ONCE(instance->ipi_data); + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE || + READ_ONCE(assignment->irq_state) != MK_PCI_MSI_ACTIVE || + !local_irq || !shared || + vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + goto unlock; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + for (;;) { + old = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(old) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + goto unlock; + } + pending = mk_irq_mailbox_pending(old); + if (pending == MK_IRQ_MAILBOX_PENDING_MASK) { + atomic_inc(&mailbox->saturated); + break; + } + new = mk_irq_mailbox_token(vector->mailbox_generation, + pending + 1) | + (old & (MK_IRQ_MAILBOX_MASKED | + MK_IRQ_MAILBOX_CONSUMING)); + if (atomic64_cmpxchg(&entry->pending_generation, old, new) == old) + break; + cpu_relax(); + } + atomic64_or(BIT_ULL(vector->mailbox_slot & 63), + &mailbox->pending_bitmap[vector->mailbox_slot / 64]); + atomic_inc(&mailbox->recorded); + if (!pending) + schedule_retry = true; + first_forward = atomic64_inc_return(&vector->forwarded) == 1; + target = mk_instance_irq_route_load(instance); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); +unlock: + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); + if (schedule_retry) + mod_delayed_work(system_wq, &instance->irq_retry_work, + msecs_to_jiffies(10)); + if (first_forward) + printk_deferred(KERN_INFO + "Forwarding host IRQ %u as instance IRQ %u for %s vector %u\n", + irq, local_irq, pci_name(assignment->vf), + (unsigned int)(vector - assignment->irq_vectors)); + return IRQ_HANDLED; +} + +static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment, + bool parked_force, + unsigned long deadline) +{ + unsigned int disabled = 0; + unsigned int i; + + assignment->irq_mailbox_failed = false; + for (i = 0; i < assignment->nr_irq_vectors; i++) + WRITE_ONCE(assignment->irq_vectors[i].local_irq, 0); + + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->requested && !vector->disabled) { + disable_irq(vector->host_irq); + vector->disabled = true; + disabled++; + } + } + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!mk_pci_mailbox_quiesce(vector, parked_force, deadline)) + assignment->irq_mailbox_failed = true; + } + if (assignment->irq_bound_map) + bitmap_zero(assignment->irq_bound_map, + assignment->nr_irq_vectors); + if (assignment->irq_state != MK_PCI_MSI_IDLE && + assignment->irq_state != MK_PCI_MSI_FAILED) + assignment->irq_state = MK_PCI_MSI_PREPARED; + + return disabled; +} + +static int mk_pci_release_irqs(struct mk_pci_assignment *assignment, + bool parked_force) +{ + unsigned long deadline = mk_pci_mailbox_deadline(); + unsigned int i; + + mk_pci_quiesce_irqs(assignment, parked_force, deadline); + if (assignment->irq_mailbox_failed) + return -ETIMEDOUT; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->requested) + free_irq(vector->host_irq, vector); + mk_pci_mailbox_release(vector); + } + if (assignment->nr_irq_vectors) + pci_free_irq_vectors(assignment->vf); + kfree(assignment->irq_vectors); + bitmap_free(assignment->irq_bound_map); + assignment->irq_vectors = NULL; + assignment->irq_bound_map = NULL; + assignment->nr_irq_vectors = 0; + assignment->irq_msix = false; + assignment->irq_needs_reprogram = false; + assignment->irq_flr_deadline = 0; + assignment->irq_mailbox_failed = false; + return 0; +} + +int mk_pci_quiesce_instance_irqs(struct mk_instance *instance, + bool parked_force) +{ + struct mk_pci_assignment *assignment; + unsigned long deadline = mk_pci_mailbox_deadline(); + unsigned int disabled = 0; + int ret = 0; + + if (!instance || instance == mk_self) + return 0; + + lockdep_assert_held(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + disabled += mk_pci_quiesce_irqs(assignment, parked_force, + deadline); + if (assignment->irq_mailbox_failed) + ret = -ETIMEDOUT; + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + if (disabled) + pr_info("Quiesced %u host-owned PCI IRQ vectors for instance %d\n", + disabled, instance->id); + return ret; +} + +unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + unsigned int requested = 0; + unsigned int i; + + lockdep_assert_held(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = + &assignment->irq_vectors[i]; + + if (vector->requested) { + requested++; + synchronize_irq(vector->host_irq); + } + } + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + if (requested) { + mk_phys_cpu_t target; + unsigned long flags; + + raw_spin_lock_irqsave(&instance->control_route_lock, flags); + target = mk_instance_irq_route_load(instance); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); + } + return requested; +} + +static bool mk_pci_generation_after(u32 generation, u32 last) +{ + return !last || (s32)(generation - last) > 0; +} + +static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + struct mk_pci_irq_vector *vectors; + unsigned long *bound_map; + unsigned int flags; + int forwarding_cpu; + int i; + int nvec; + int ret; + + if (!request->nr_vectors || + request->nr_vectors > MK_IRQ_MAILBOX_SLOTS) + return -EINVAL; + if (!request->lifecycle_generation) + return -EINVAL; + if (assignment->irq_epoch && + assignment->irq_epoch != request->lifecycle_epoch) + return -ESTALE; + forwarding_cpu = mk_pci_forwarding_cpu(); + if (forwarding_cpu < 0) { + pr_err("Host control CPU is unavailable for %s MSI forwarding\n", + pci_name(assignment->vf)); + return forwarding_cpu; + } + if (assignment->irq_generation == request->lifecycle_generation) { + if ((assignment->irq_state == MK_PCI_MSI_PREPARED || + assignment->irq_state == MK_PCI_MSI_COMMITTED || + assignment->irq_state == MK_PCI_MSI_ACTIVE) && + assignment->nr_irq_vectors == request->nr_vectors && + assignment->irq_msix == request->msix) + return 0; + return -ESTALE; + } + if (!mk_pci_generation_after(request->lifecycle_generation, + assignment->irq_generation)) + return -ESTALE; + if (assignment->nr_irq_vectors) { + ret = mk_pci_release_irqs(assignment, false); + if (ret) + return ret; + } + assignment->irq_epoch = request->lifecycle_epoch; + assignment->irq_generation = request->lifecycle_generation; + assignment->irq_state = MK_PCI_MSI_FAILED; + + vectors = kcalloc(request->nr_vectors, sizeof(*vectors), GFP_KERNEL); + if (!vectors) + return -ENOMEM; + bound_map = bitmap_zalloc(request->nr_vectors, GFP_KERNEL); + if (!bound_map) { + kfree(vectors); + return -ENOMEM; + } + flags = request->msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; + nvec = pci_alloc_irq_vectors(assignment->vf, request->nr_vectors, + request->nr_vectors, flags); + if (nvec < 0) { + bitmap_free(bound_map); + kfree(vectors); + return nvec; + } + + assignment->irq_vectors = vectors; + assignment->irq_bound_map = bound_map; + assignment->nr_irq_vectors = nvec; + assignment->irq_msix = request->msix; + for (i = 0; i < nvec; i++) { + vectors[i].assignment = assignment; + vectors[i].host_irq = pci_irq_vector(assignment->vf, i); + vectors[i].mailbox_slot = MK_IRQ_MAILBOX_SLOT_INVALID; + ret = mk_pci_mailbox_reserve(&vectors[i], i); + if (ret) + goto err_release; + } + for (i = 0; i < nvec; i++) { + ret = request_irq(vectors[i].host_irq, mk_pci_forward_irq, + IRQF_NO_AUTOEN | IRQF_NOBALANCING, + "multikernel-pci-forward", &vectors[i]); + if (ret) + goto err_release; + vectors[i].requested = true; + vectors[i].disabled = true; + ret = irq_set_affinity(vectors[i].host_irq, + cpumask_of(forwarding_cpu)); + if (ret) + goto err_release; + } + assignment->irq_state = MK_PCI_MSI_PREPARED; + pr_info("Allocated %d host-owned %s vectors for %s (instance %d, CPU %d)\n", + nvec, request->msix ? "MSI-X" : "MSI", + pci_name(assignment->vf), assignment->instance->id, + forwarding_cpu); + return 0; + +err_release: + pr_err("Failed to configure host-owned IRQ for %s vector %d: %d\n", + pci_name(assignment->vf), i, ret); + if (mk_pci_release_irqs(assignment, false)) + return -ETIMEDOUT; + return ret; +} + +static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_pci_irq_vector *vector; + unsigned long delay; + unsigned int count; + unsigned int i; + u32 last_irq; + u32 local_irq; + + if (mk_instance_irq_route_load(assignment->instance) == + MK_PHYS_CPU_INVALID) + return -ENODEV; + if (!request->lifecycle_generation || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_epoch != request->lifecycle_epoch || + (assignment->irq_state != MK_PCI_MSI_PREPARED && + assignment->irq_state != MK_PCI_MSI_ACTIVE)) + return -ESTALE; + + if (!assignment->irq_vectors || + assignment->irq_msix != request->msix) + return -EINVAL; + count = request->msix ? 1 : request->nr_vectors; + if (!count || request->vector >= assignment->nr_irq_vectors || + count > assignment->nr_irq_vectors - request->vector) + return -EINVAL; + last_irq = request->local_irq + count - 1; + if (!request->local_irq || last_irq < request->local_irq) + return -EINVAL; + + for (i = 0; i < count; i++) { + struct mk_irq_mailbox_entry *entry; + u64 token; + + vector = &assignment->irq_vectors[request->vector + i]; + local_irq = READ_ONCE(vector->local_irq); + if (!vector->requested || + vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return -EINVAL; + entry = &shared->irq_mailbox.entries[vector->mailbox_slot]; + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation || + READ_ONCE(entry->lifecycle_epoch) != assignment->irq_epoch || + READ_ONCE(entry->lifecycle_generation) != + assignment->irq_generation || + READ_ONCE(entry->vector) != request->vector + i) + return -ESTALE; + if (local_irq && local_irq != request->local_irq + i) + return -EBUSY; + } + + if (assignment->irq_needs_reprogram) { + if (time_before(jiffies, assignment->irq_flr_deadline)) { + delay = assignment->irq_flr_deadline - jiffies; + msleep(jiffies_to_msecs(delay) + 1); + } + pci_restore_msi_state(assignment->vf); + assignment->irq_needs_reprogram = false; + assignment->irq_flr_deadline = 0; + pr_info("Reprogrammed host-owned MSI state for %s during restore\n", + pci_name(assignment->vf)); + } + + for (i = 0; i < count; i++) { + struct mk_irq_mailbox_entry *entry; + + vector = &assignment->irq_vectors[request->vector + i]; + entry = &shared->irq_mailbox.entries[vector->mailbox_slot]; + WRITE_ONCE(entry->local_irq, request->local_irq + i); + /* Make the shared identity visible before routing this vector. */ + smp_store_release(&vector->local_irq, request->local_irq + i); + } + bitmap_set(assignment->irq_bound_map, request->vector, count); + return 0; +} + +static int +mk_pci_restore_irqs_begin(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_state == MK_PCI_MSI_PREPARED && + assignment->irq_needs_reprogram) + return 0; + if (assignment->irq_state != MK_PCI_MSI_ACTIVE) + return -ESTALE; + + mk_pci_quiesce_irqs(assignment, false, mk_pci_mailbox_deadline()); + if (assignment->irq_mailbox_failed) + return -ETIMEDOUT; + assignment->irq_needs_reprogram = true; + return 0; +} + +static int mk_pci_commit_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + unsigned int i; + + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation || + (assignment->irq_state != MK_PCI_MSI_PREPARED && + assignment->irq_state != MK_PCI_MSI_ACTIVE) || + !assignment->irq_bound_map || + !bitmap_full(assignment->irq_bound_map, + assignment->nr_irq_vectors)) + return -ESTALE; + + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!READ_ONCE(vector->local_irq)) + return -EINVAL; + } + if (assignment->irq_state == MK_PCI_MSI_ACTIVE) + return 0; + assignment->irq_state = MK_PCI_MSI_COMMITTED; + return 0; +} + +static int mk_pci_activate_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + unsigned int i; + + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_state == MK_PCI_MSI_ACTIVE) + return 0; + if (assignment->irq_state != MK_PCI_MSI_COMMITTED || + !assignment->nr_irq_vectors || !assignment->irq_bound_map || + !bitmap_full(assignment->irq_bound_map, + assignment->nr_irq_vectors)) + return -ESTALE; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!vector->requested || !READ_ONCE(vector->local_irq)) + return -EINVAL; + } + assignment->irq_state = MK_PCI_MSI_ACTIVE; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->disabled) { + enable_irq(vector->host_irq); + vector->disabled = false; + } + } + return 0; +} + +static int mk_pci_teardown_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + if (assignment->irq_epoch && + assignment->irq_epoch != request->lifecycle_epoch) + return -ESTALE; + if (assignment->irq_generation != request->lifecycle_generation && + !mk_pci_generation_after(request->lifecycle_generation, + assignment->irq_generation)) + return -ESTALE; + + if (assignment->nr_irq_vectors) { + int ret = mk_pci_release_irqs(assignment, false); + + if (ret) + return ret; + } + assignment->irq_epoch = request->lifecycle_epoch; + assignment->irq_generation = request->lifecycle_generation; + assignment->irq_state = MK_PCI_MSI_IDLE; + return 0; +} + +static bool mk_pci_is_flr_write(struct pci_dev *vf, + const struct mk_pci_cfg_request *request) +{ + u16 flr_byte; + unsigned int bit; + + if (!request->write || !pci_is_pcie(vf)) + return false; + flr_byte = pci_pcie_cap(vf) + PCI_EXP_DEVCTL + 1; + if (request->reg > flr_byte || + request->reg + request->len <= flr_byte) + return false; + bit = (flr_byte - request->reg) * 8 + 7; + return request->value & BIT(bit); +} + +static bool mk_pci_request_route_stale(struct mk_instance *instance, + s32 sender_instance_id, + u64 lifecycle_epoch) +{ + if (!instance || instance == mk_self || + instance->id != sender_instance_id) + return true; + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE) + return true; + return !instance->ipi_data || !lifecycle_epoch || + READ_ONCE(instance->ipi_data->spawn_epoch) != lifecycle_epoch; +} + +static int mk_pci_config_access(struct mk_instance *instance, + const struct mk_pci_cfg_request *request, + u32 *value, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + struct pci_dev *vf; + bool flr; + int ret; + + if (request->len != 1 && request->len != 2 && request->len != 4) + return PCIBIOS_BAD_REGISTER_NUMBER; + if (request->reg > PCI_CFG_SPACE_EXP_SIZE - request->len) + return PCIBIOS_BAD_REGISTER_NUMBER; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf)) { + ret = PCIBIOS_DEVICE_NOT_FOUND; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + /* Committed: complete and let publish reclaim an indeterminate timeout. */ + + vf = assignment->vf; + flr = mk_pci_is_flr_write(vf, request); + if (flr) { + ret = PCIBIOS_SET_FAILED; + goto out; + } + if (request->write) { + switch (request->len) { + case 1: + ret = pci_write_config_byte(vf, request->reg, + request->value); + break; + case 2: + ret = pci_write_config_word(vf, request->reg, + request->value); + break; + default: + ret = pci_write_config_dword(vf, request->reg, + request->value); + break; + } + } else { + switch (request->len) { + case 1: { + u8 data; + + ret = pci_read_config_byte(vf, request->reg, &data); + *value = data; + break; + } + case 2: { + u16 data; + + ret = pci_read_config_word(vf, request->reg, &data); + *value = data; + break; + } + default: + ret = pci_read_config_dword(vf, request->reg, value); + break; + } + } +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_cfg_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_cfg_request *request = &control_work->request.cfg; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_CFG, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + u32 value = ~0U; + s32 status; + + instance = mk_instance_find(control_work->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + status = mk_pci_request_route_stale(instance, + control_work->sender_instance_id, + request->lifecycle_epoch) ? + -ESTALE : 0; + /* An untrusted payload ID must never select another instance's slot. */ + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_config_access(instance, request, &value, &reply); + if (mk_reply_publish_route_locked(instance, &reply, status, value)) + pr_warn_ratelimited("Failed to return PCI config response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + return; +} + +static int mk_pci_irq_access(struct mk_instance *instance, + const struct mk_pci_irq_request *request, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + int ret; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + if (request->operation != MK_PCI_IRQ_TEARDOWN && + READ_ONCE(instance->state) != MK_STATE_ACTIVE) { + ret = -ESHUTDOWN; + goto out; + } + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf)) { + ret = -ENODEV; + goto out; + } + if (!instance->ipi_data || !request->lifecycle_epoch || + request->lifecycle_epoch != + READ_ONCE(instance->ipi_data->spawn_epoch)) { + ret = -ESTALE; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + /* + * This CAS is the sole cancellation boundary. IRQ programming is + * non-cancellable once EXECUTING; a timed-out waiter moves the exact + * generation to COMMITTED for mk_reply_publish() to reclaim afterwards. + */ + + switch (request->operation) { + case MK_PCI_IRQ_SETUP: + ret = mk_pci_setup_irqs(assignment, request); + break; + case MK_PCI_IRQ_RESTORE_BEGIN: + ret = mk_pci_restore_irqs_begin(assignment, request); + break; + case MK_PCI_IRQ_BIND: + ret = mk_pci_bind_irq(assignment, request); + break; + case MK_PCI_IRQ_COMMIT: + ret = mk_pci_commit_irqs(assignment, request); + break; + case MK_PCI_IRQ_ACTIVATE: + ret = mk_pci_activate_irqs(assignment, request); + break; + case MK_PCI_IRQ_TEARDOWN: + ret = mk_pci_teardown_irqs(assignment, request); + break; + default: + ret = -EINVAL; + break; + } +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_irq_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_irq_request *request = &control_work->request.irq; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_IRQ, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + s32 status; + + instance = mk_instance_find(control_work->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + status = mk_pci_request_route_stale(instance, + control_work->sender_instance_id, + request->lifecycle_epoch) ? + -ESTALE : 0; + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_irq_access(instance, request, &reply); + if (mk_reply_publish_route_locked(instance, &reply, status, 0)) + pr_warn_ratelimited("Failed to return PCI IRQ response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + return; +} + +static int mk_pci_reset_access(struct mk_instance *instance, + const struct mk_pci_reset_request *request, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + int ret; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE) { + ret = -ESHUTDOWN; + goto out; + } + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf) || !assignment->vf->is_virtfn) { + ret = -ENODEV; + goto out; + } + if (!instance->ipi_data || !request->lifecycle_epoch || + request->lifecycle_epoch != + READ_ONCE(instance->ipi_data->spawn_epoch) || + !request->reset_generation || + !mk_pci_generation_after(request->reset_generation, + assignment->reset_generation)) { + ret = -ESTALE; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + + /* Tombstone this serial before side effects so delayed replays reject. */ + assignment->reset_generation = request->reset_generation; + mk_pci_quiesce_irqs(assignment, false, mk_pci_mailbox_deadline()); + if (assignment->irq_mailbox_failed) { + assignment->irq_state = MK_PCI_MSI_FAILED; + mk_pci_schedule_failure(assignment); + ret = -ETIMEDOUT; + goto out; + } + ret = pcie_reset_flr(assignment->vf, false); + if (ret) { + assignment->irq_state = MK_PCI_MSI_FAILED; + mk_pci_schedule_failure(assignment); + goto out; + } + if (assignment->nr_irq_vectors) { + assignment->irq_needs_reprogram = true; + assignment->irq_flr_deadline = + jiffies + msecs_to_jiffies(MK_PCI_FLR_SETTLE_MS); + } + pr_info("Invalidated host-owned MSI bindings for spawn-triggered FLR of %s\n", + pci_name(assignment->vf)); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_reset_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_reset_request *request = &control_work->request.reset; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_RESET, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + s32 status; + + instance = mk_instance_find(control_work->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + status = mk_pci_request_route_stale(instance, + control_work->sender_instance_id, + request->lifecycle_epoch) ? + -ESTALE : 0; + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_reset_access(instance, request, &reply); + if (mk_reply_publish_route_locked(instance, &reply, status, 0)) + pr_warn_ratelimited("Failed to return PCI reset response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + return; +} + +static void mk_pci_control_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + + control_work->dispatch(work); + mk_pci_control_work_put(control_work); +} + +static void mk_pci_control_msg_handler(u32 msg_type, u32 subtype, + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx) +{ + struct mk_pci_control_work *control_work; + size_t request_size; + work_func_t work_fn; + + if (msg_type != MK_MSG_PCI) + return; + if (!mk_pci_control_handler_get()) + return; + + switch (subtype) { + case MK_PCI_CFG_REQUEST: + request_size = sizeof(control_work->request.cfg); + work_fn = mk_pci_cfg_work_fn; + break; + case MK_PCI_IRQ_REQUEST: + request_size = sizeof(control_work->request.irq); + work_fn = mk_pci_irq_work_fn; + break; + case MK_PCI_RESET_REQUEST: + request_size = sizeof(control_work->request.reset); + work_fn = mk_pci_reset_work_fn; + break; + default: + goto out; + } + if (payload_len != request_size) + goto out; + /* + * Valid senders reserve one of MK_REPLY_SLOTS before publishing. Every + * active instance owns at least one disjoint possible CPU, so the pool + * covers the maximum number of valid requests across all instances. This + * keeps the hardirq receive path allocation-safe without changing reply + * or route validation. Duplicate traffic is outside the cooperative ABI. + */ + control_work = mk_pci_control_work_get(); + if (!control_work) { + atomic64_inc(&mk_pci_control_pool_exhausted); + if (__ratelimit(&mk_pci_control_pool_rs)) + printk_deferred(KERN_WARNING + "Multikernel PCI control work pool exhausted\n"); + goto out; + } + control_work->dispatch = work_fn; + memcpy(&control_work->request, payload, request_size); + control_work->sender_instance_id = sender_instance_id; + switch (subtype) { + case MK_PCI_CFG_REQUEST: + control_work->request.cfg.sender_instance_id = sender_instance_id; + break; + case MK_PCI_IRQ_REQUEST: + control_work->request.irq.sender_instance_id = sender_instance_id; + break; + case MK_PCI_RESET_REQUEST: + control_work->request.reset.sender_instance_id = sender_instance_id; + break; + } + if (!queue_work(mk_pci_control_wq, &control_work->work)) + mk_pci_control_work_put(control_work); +out: + mk_pci_control_handler_put(); +} + +static void mk_pci_assignment_failure_work(struct work_struct *work) +{ + struct mk_pci_assignment *assignment = + container_of(work, struct mk_pci_assignment, failure_work); + struct mk_instance *instance = assignment->instance; + int ret; + + pr_err("PCI assignment lease for %s was lost by instance %d (%s)\n", + pci_name(assignment->vf), instance->id, instance->name); + + if (READ_ONCE(instance->state) == MK_STATE_ACTIVE) { + ret = mk_instance_force_halt(instance); + if (ret) + pr_err("Failed to force halt instance %d after PCI lease loss: %d\n", + instance->id, ret); + } + + mk_instance_mark_failed(instance); +} + +static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment) +{ + if (atomic_cmpxchg(&assignment->failure_pending, 0, 1)) + return; + + if (!schedule_work(&assignment->failure_work)) + atomic_set(&assignment->failure_pending, 0); +} + +static int mk_pci_bus_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct pci_dev *pdev = to_pci_dev(data); + struct mk_pci_assignment *assignment; + unsigned long flags; + + if (action != BUS_NOTIFY_DEL_DEVICE && + action != BUS_NOTIFY_REMOVED_DEVICE && + action != BUS_NOTIFY_UNBOUND_DRIVER) + return NOTIFY_DONE; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_for_each_entry(assignment, &mk_pci_active_assignments, + active_node) { + if (pdev != assignment->vf && pdev != assignment->pf) + continue; + if (pdev == assignment->vf && + action == BUS_NOTIFY_UNBOUND_DRIVER && + assignment->expected_unbind) + continue; + mk_pci_schedule_failure(assignment); + } + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + return NOTIFY_OK; +} + +static struct notifier_block mk_pci_bus_notifier = { + .notifier_call = mk_pci_bus_notify, +}; + +static void +mk_pci_release_bound_driver(struct mk_pci_assignment *assignment) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = true; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + device_release_driver(&assignment->vf->dev); + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); +} + +#if IS_ENABLED(CONFIG_IOMMU_API) +#define MK_PCI_ASSIGNMENT_DRIVER_NAME "multikernel-pci-assignment" + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id); +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev); + +static struct pci_driver mk_pci_assignment_driver = { + .name = MK_PCI_ASSIGNMENT_DRIVER_NAME, + .probe = mk_pci_iommu_assignment_probe, + .remove = mk_pci_iommu_assignment_remove, + .driver_managed_dma = true, +}; + +struct mk_pci_iommu_group_check { + struct device *vf; + unsigned int count; +}; + +static int mk_pci_iommu_check_group_device(struct device *dev, void *data) +{ + struct mk_pci_iommu_group_check *check = data; + + check->count++; + return dev == check->vf ? 0 : -EXDEV; +} + +static void mk_pci_iommu_free_resv_regions(struct list_head *regions) +{ + struct iommu_resv_region *region, *tmp; + + list_for_each_entry_safe(region, tmp, regions, list) { + list_del(®ion->list); + kfree(region); + } +} + +static int mk_pci_iommu_validate_group(struct mk_pci_assignment *assignment) +{ + struct mk_pci_iommu_group_check check = { + .vf = &assignment->vf->dev, + }; + int ret; + + ret = iommu_group_for_each_dev(assignment->iommu_group, &check, + mk_pci_iommu_check_group_device); + if (ret || check.count != 1) { + pr_err("IOMMU group %d for %s is not an isolated singleton group\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return ret ?: -EXDEV; + } + + if (!iommu_group_has_isolated_msi(assignment->iommu_group)) { + pr_err("IOMMU group %d for %s lacks isolated MSI delivery\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return -EPERM; + } + + return 0; +} + +static int +mk_pci_iommu_validate_resv_regions(struct mk_pci_assignment *assignment) +{ + struct iommu_resv_region *region; + struct mk_memory_region *memory; + LIST_HEAD(resv_regions); + u64 memory_end, resv_end; + int ret; + + ret = iommu_get_group_resv_regions(assignment->iommu_group, + &resv_regions); + if (ret) + goto out; + + list_for_each_entry(region, &resv_regions, list) { + if (region->type == IOMMU_RESV_DIRECT_RELAXABLE) + continue; + if (!region->length || + check_add_overflow((u64)region->start, + (u64)region->length - 1, &resv_end)) { + pr_err("IOMMU group %d for %s has invalid reserved region at %#llx\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf), + (unsigned long long)region->start); + ret = -EOVERFLOW; + goto out; + } + + list_for_each_entry(memory, + &assignment->instance->memory_regions, list) { + resource_size_t size = resource_size(&memory->res); + + if (!size || + check_add_overflow((u64)memory->res.start, + (u64)size - 1, &memory_end)) { + ret = -EOVERFLOW; + goto out; + } + if ((u64)memory->res.start > resv_end || + memory_end < (u64)region->start) + continue; + + pr_err("Instance %d IOVA %#llx-%#llx overlaps IOMMU reserved region %#llx-%#llx type %u for %s\n", + assignment->instance->id, + (unsigned long long)memory->res.start, + (unsigned long long)memory_end, + (unsigned long long)region->start, + (unsigned long long)resv_end, region->type, + pci_name(assignment->vf)); + ret = -EPERM; + goto out; + } + } + + ret = 0; +out: + mk_pci_iommu_free_resv_regions(&resv_regions); + return ret; +} + +static int +mk_pci_iommu_validate_region(struct mk_pci_assignment *assignment, + const struct mk_memory_region *region) +{ + struct iommu_domain *domain = assignment->iommu_domain; + resource_size_t start = region->res.start; + resource_size_t size = resource_size(®ion->res); + u64 dma_mask = dma_get_mask(&assignment->vf->dev); + u64 end; + unsigned long min_page_size; + + if (!size || check_add_overflow((u64)start, (u64)size - 1, &end)) + return -EOVERFLOW; + if (!domain->pgsize_bitmap) + return -EOPNOTSUPP; + + min_page_size = 1UL << __ffs(domain->pgsize_bitmap); + if (!IS_ALIGNED(start, min_page_size) || + !IS_ALIGNED(size, min_page_size)) { + pr_err("Instance %d memory %#llx-%#llx is not aligned to IOMMU page size %#lx\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, min_page_size); + return -EINVAL; + } + if (start > ULONG_MAX || end > ULONG_MAX || end > dma_mask) { + pr_err("Instance %d memory %#llx-%#llx exceeds DMA addressability of %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + if (domain->geometry.force_aperture && + (start < domain->geometry.aperture_start || + end > domain->geometry.aperture_end)) { + pr_err("Instance %d memory %#llx-%#llx is outside the IOMMU aperture for %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + + return 0; +} + +static void mk_pci_iommu_unmap_regions(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + unsigned int remaining = assignment->iommu_mapped_regions; + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size; + size_t unmapped; + + if (!remaining) + break; + size = resource_size(®ion->res); + unmapped = iommu_unmap(assignment->iommu_domain, + region->res.start, size); + if (unmapped != size) + pr_err("IOMMU unmapped only %#zx of %#llx bytes for instance %d at %#llx\n", + unmapped, (unsigned long long)size, + assignment->instance->id, + (unsigned long long)region->res.start); + remaining--; + } + if (remaining) + pr_err("IOMMU lease for %s lost %u mapped instance regions\n", + pci_name(assignment->vf), remaining); + assignment->iommu_mapped_regions = 0; +} + +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + bool transactions_drained; + int ret; + + ret = mk_pci_release_irqs(assignment, false); + if (ret) + return ret; + if (!mk_pci_device_live(vf)) + return 0; + + /* + * Releasing DMA ownership restores the group's default domain. Stop new + * DMA first, drain requests already issued, and reset the VF while the + * assignment domain still contains any stragglers. + */ + pci_clear_master(vf); + transactions_drained = pci_wait_for_pending_transaction(vf); + ret = pcie_reset_flr(vf, false); + if (!ret) + return 0; + if (ret == -ENOTTY && transactions_drained) + return 0; + + if (!transactions_drained) + pr_err("Timed out draining DMA from assigned VF %s\n", + pci_name(vf)); + if (ret != -ENOTTY) + pr_err("Failed to reset assigned VF %s: %d\n", + pci_name(vf), ret); + + /* + * Keep the assignment domain attached when the device cannot be made + * safe. The lease owner can retry teardown after the instance halts. + */ + return ret == -ENOTTY ? -ETIMEDOUT : ret; +} + +static int +mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + int ret; + + ret = mk_pci_release_irqs(assignment, true); + if (ret) + return ret; + assignment->irq_epoch = 0; + assignment->irq_generation = 0; + assignment->reset_generation = 0; + assignment->irq_state = MK_PCI_MSI_IDLE; + if (!assignment->assigned || !assignment->iommu_attached) + return -EINVAL; + if (!mk_pci_device_live(vf)) + return -ENODEV; + + /* + * A stopped instance may have left DMA active. Keep its restrictive + * domain attached while stopping new requests, draining old ones, and + * resetting device state before the instance image is reused. + */ + pci_clear_master(vf); + if (!pci_wait_for_pending_transaction(vf)) { + pr_err("Timed out draining assigned VF %s before instance restart\n", + pci_name(vf)); + return -ETIMEDOUT; + } + + ret = pcie_reset_flr(vf, false); + if (ret) { + pr_err("Failed to reset assigned VF %s before instance restart: %d\n", + pci_name(vf), ret); + return ret == -ENOTTY ? -EOPNOTSUPP : ret; + } + return 0; +} + +static void +__mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + if (assignment->iommu_attached) { + iommu_detach_group(assignment->iommu_domain, + assignment->iommu_group); + assignment->iommu_attached = false; + } + if (assignment->iommu_dma_owner) { + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + } +} + +static void +mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + mutex_unlock(&assignment->iommu_mutex); +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + + if (assignment->iommu_domain) { + mk_pci_iommu_unmap_regions(assignment); + iommu_domain_free(assignment->iommu_domain); + assignment->iommu_domain = NULL; + } + if (assignment->iommu_group) { + iommu_group_put(assignment->iommu_group); + assignment->iommu_group = NULL; + } + mutex_unlock(&assignment->iommu_mutex); +} + +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + int ret; + + if (!device_iommu_mapped(&assignment->vf->dev)) { + pr_err("Cannot assign %s without an active hardware IOMMU\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; + } + if (!device_iommu_capable(&assignment->vf->dev, + IOMMU_CAP_CACHE_COHERENCY)) { + pr_err("Cannot assign %s without coherent IOMMU mappings\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; + } + if (!assignment->instance->region_count || + list_empty(&assignment->instance->memory_regions)) + return -EINVAL; + + assignment->iommu_group = iommu_group_get(&assignment->vf->dev); + if (!assignment->iommu_group) + return -ENODEV; + + ret = mk_pci_iommu_validate_group(assignment); + if (ret) + goto err_release; + ret = mk_pci_iommu_validate_resv_regions(assignment); + if (ret) + goto err_release; + + assignment->iommu_domain = + iommu_paging_domain_alloc(&assignment->vf->dev); + if (IS_ERR(assignment->iommu_domain)) { + ret = PTR_ERR(assignment->iommu_domain); + assignment->iommu_domain = NULL; + goto err_release; + } + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size = resource_size(®ion->res); + + ret = mk_pci_iommu_validate_region(assignment, region); + if (ret) + goto err_release; + ret = iommu_map(assignment->iommu_domain, region->res.start, + region->res.start, size, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, GFP_KERNEL); + if (ret) + goto err_release; + assignment->iommu_mapped_regions++; + } + + /* + * The domain blocks DMA outside these mappings, but translation-fault + * notification is not portable. In particular, Intel VT-d reports primary + * faults through dmar_fault() without invoking a legacy domain handler. + * Do not claim automatic instance failure on an IOMMU fault here. + */ + pr_info("Prepared host IOMMU domain for %s with %u instance regions\n", + pci_name(assignment->vf), assignment->iommu_mapped_regions); + return 0; + +err_release: + mk_pci_iommu_release_assignment(assignment); + return ret; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + int ret; + + if (!assignment->iommu_domain) + return 0; + + if (assignment->vf->driver_override) { + assignment->host_driver_override = + kstrdup(assignment->vf->driver_override, GFP_KERNEL); + if (!assignment->host_driver_override) + return -ENOMEM; + } + + ret = driver_set_override(&assignment->vf->dev, + &assignment->vf->driver_override, + MK_PCI_ASSIGNMENT_DRIVER_NAME, + strlen(MK_PCI_ASSIGNMENT_DRIVER_NAME)); + if (ret) + return ret; + assignment->iommu_override_active = true; + pci_set_drvdata(assignment->vf, assignment); + ret = device_driver_attach(&mk_pci_assignment_driver.driver, + &assignment->vf->dev); + if (ret) + return ret; + if (assignment->vf->dev.driver != &mk_pci_assignment_driver.driver) + return -ENODEV; + return 0; +} + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; + + if (!assignment || assignment->vf != pdev || !assignment->iommu_domain) + return -ENODEV; + ret = iommu_device_claim_dma_owner(&assignment->vf->dev, assignment); + if (ret) + return ret; + assignment->iommu_dma_owner = true; + + ret = iommu_attach_group(assignment->iommu_domain, + assignment->iommu_group); + if (ret) { + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + return ret; + } + assignment->iommu_attached = true; + ret = pci_enable_device(pdev); + if (ret) { + iommu_detach_group(assignment->iommu_domain, + assignment->iommu_group); + assignment->iommu_attached = false; + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + return ret; + } + assignment->device_enabled = true; + pr_info("Attached %s to host-owned IOMMU domain for instance %d\n", + pci_name(assignment->vf), assignment->instance->id); + return 0; +} + +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; + + if (!assignment || assignment->vf != pdev) + return; + + if (READ_ONCE(assignment->expected_unbind)) { + if (assignment->device_enabled) { + pci_disable_device(pdev); + assignment->device_enabled = false; + } + pci_set_drvdata(pdev, NULL); + return; + } + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) { + pr_crit("Keeping IOMMU containment for %s after unsafe driver removal: %d\n", + pci_name(pdev), ret); + mk_pci_schedule_failure(assignment); + } else { + mk_pci_iommu_deactivate_assignment(assignment); + } + if (assignment->device_enabled) { + pci_disable_device(pdev); + assignment->device_enabled = false; + } + pci_set_drvdata(pdev, NULL); +} + +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + const char *override = assignment->host_driver_override ?: ""; + int ret = 0; + + if (vf->dev.driver == &mk_pci_assignment_driver.driver) { + mk_pci_release_bound_driver(assignment); + } else if (vf->dev.driver) { + pr_err("Cannot release assignment driver from %s: device is bound to %s\n", + pci_name(vf), vf->dev.driver->name); + return -EBUSY; + } + + pci_set_drvdata(vf, NULL); + if (assignment->iommu_override_active) { + ret = driver_set_override(&vf->dev, &vf->driver_override, + override, strlen(override)); + if (ret) + return ret; + assignment->iommu_override_active = false; + } + + mk_pci_iommu_deactivate_assignment(assignment); + if (assignment->host_driver && mk_pci_device_live(vf)) { + if (!vf->dev.driver) { + ret = device_driver_attach(assignment->host_driver, &vf->dev); + if (ret) { + pr_err("Failed to restore driver %s to %s: %d\n", + assignment->host_driver->name, + pci_name(vf), ret); + return ret; + } + } else if (vf->dev.driver != assignment->host_driver) { + pr_err("Cannot restore driver %s to %s: device is bound to %s\n", + assignment->host_driver->name, pci_name(vf), + vf->dev.driver->name); + return -EBUSY; + } + } + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return pci_register_driver(&mk_pci_assignment_driver); +} + +static void mk_pci_iommu_system_cleanup(void) +{ + pci_unregister_driver(&mk_pci_assignment_driver); +} +#else +static int +mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) +{ + return -EOPNOTSUPP; +} + +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + pr_err("Cannot assign %s without CONFIG_IOMMU_API\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ +} + +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return 0; +} + +static void mk_pci_iommu_system_cleanup(void) +{ +} +#endif + +static int +mk_pci_prepare_assignment(struct mk_instance *instance, + const struct mk_pci_device *requested, + struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + struct pci_dev *physfn; + int ret; + + inventory = mk_pci_find_root_inventory(requested); + if (!inventory) { + pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x is not available in the root pool\n", + requested->vendor, requested->device, requested->domain, + requested->bus, requested->slot, requested->func); + return -ENOENT; + } + + vf = pci_get_domain_bus_and_slot(inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func)); + if (!vf) + return -ENODEV; + + if (vf->vendor != inventory->vendor || + vf->device != inventory->device) { + pr_err("PCI identity changed for %s: expected %04x:%04x, found %04x:%04x\n", + pci_name(vf), inventory->vendor, inventory->device, + vf->vendor, vf->device); + pci_dev_put(vf); + return -ENODEV; + } + + physfn = pci_physfn(vf); + if (!vf->is_virtfn || physfn == vf) { + pr_err("PCI assignment only supports SR-IOV VFs, rejecting %s\n", + pci_name(vf)); + pci_dev_put(vf); + return -EOPNOTSUPP; + } + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(physfn)) { + pci_dev_put(vf); + return -ENODEV; + } + ret = pcie_reset_flr(vf, true); + if (ret) { + pr_err("PCI assignment requires FLR for safe instance restart, rejecting %s\n", + pci_name(vf)); + pci_dev_put(vf); + return -EOPNOTSUPP; + } + + if (pci_is_dev_assigned(vf) || + mk_pci_find_assignment(instance, inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func))) { + pci_dev_put(vf); + return -EBUSY; + } + + pf = pci_dev_get(physfn); + assignment = kzalloc_obj(*assignment, GFP_KERNEL); + if (!assignment) { + pci_dev_put(pf); + pci_dev_put(vf); + return -ENOMEM; + } + + assignment->instance = instance; + assignment->inventory = inventory; + assignment->vf = vf; + assignment->pf = pf; + assignment->host_driver = vf->dev.driver; + if (assignment->host_driver && assignment->host_driver->owner && + !try_module_get(assignment->host_driver->owner)) { + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return -ENODEV; + } + + INIT_LIST_HEAD(&assignment->instance_node); + INIT_LIST_HEAD(&assignment->active_node); + INIT_LIST_HEAD(&assignment->transaction_node); + mutex_init(&assignment->iommu_mutex); + INIT_WORK(&assignment->failure_work, mk_pci_assignment_failure_work); + atomic_set(&assignment->failure_pending, 0); + + ret = mk_pci_iommu_prepare_assignment(assignment); + if (ret) + goto err_module; + + list_add_tail(&assignment->instance_node, &instance->pci_assignments); + list_add_tail(&assignment->transaction_node, transaction); + + return 0; + +err_module: + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return ret; +} + +static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int ret; + int i; + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(assignment->pf)) + return -ENODEV; + + if (vf->dev.driver != assignment->host_driver || + pci_is_dev_assigned(vf)) + return -EBUSY; + + pci_set_dev_assigned(vf); + assignment->assigned = true; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_add_tail(&assignment->active_node, &mk_pci_active_assignments); + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (assignment->host_driver) + mk_pci_release_bound_driver(assignment); + + if (vf->dev.driver) + return -EBUSY; + + ret = mk_pci_iommu_commit_assignment(assignment); + if (ret) + return ret; + + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + assignment->inventory->resources[i].start = + vf->resource[i].start; + assignment->inventory->resources[i].end = + vf->resource[i].end; + assignment->inventory->resources[i].flags = + vf->resource[i].flags; + } + assignment->inventory->resources_valid = true; + + list_move_tail(&assignment->inventory->list, + &assignment->instance->pci_devices); + mk_self->pci_device_count--; + assignment->instance->pci_device_count++; + assignment->instance->pci_devices_valid = true; + assignment->inventory_moved = true; + + pr_info("Leased SR-IOV VF %s to instance %d (%s)\n", + pci_name(vf), assignment->instance->id, + assignment->instance->name); + return 0; +} + +static int mk_pci_release_assignment(struct mk_pci_assignment *assignment, + struct list_head *released) +{ + struct mk_instance *instance = assignment->instance; + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int ret; + + if (!assignment->assigned) + goto release_resources; + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) + return ret; + + ret = mk_pci_restore_host_binding(assignment); + if (ret) + return ret; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + if (!list_empty(&assignment->active_node)) + list_del_init(&assignment->active_node); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (assignment->assigned) { + pci_clear_dev_assigned(vf); + assignment->assigned = false; + } + +release_resources: + mk_pci_iommu_release_assignment(assignment); + + if (assignment->inventory_moved && mk_self) { + assignment->inventory->resources_valid = false; + list_move_tail(&assignment->inventory->list, + &mk_self->pci_devices); + instance->pci_device_count--; + mk_self->pci_device_count++; + mk_self->pci_devices_valid = true; + assignment->inventory_moved = false; + } + + if (!list_empty(&assignment->transaction_node)) + list_del_init(&assignment->transaction_node); + list_del_init(&assignment->instance_node); + list_add_tail(&assignment->transaction_node, released); + + return 0; +} + +static void mk_pci_finalize_releases(struct list_head *released) +{ + struct mk_pci_assignment *assignment, *tmp; + + list_for_each_entry_safe(assignment, tmp, released, transaction_node) { + list_del_init(&assignment->transaction_node); + cancel_work_sync(&assignment->failure_work); + kfree(assignment->host_driver_override); + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + pci_dev_put(assignment->pf); + pci_dev_put(assignment->vf); + kfree(assignment); + } +} + +static int mk_pci_rollback_transaction(struct list_head *transaction, + struct list_head *released) +{ + struct mk_pci_assignment *assignment, *tmp; + int rollback_ret = 0; + int ret; + + list_for_each_entry_safe_reverse(assignment, tmp, transaction, + transaction_node) { + ret = mk_pci_release_assignment(assignment, released); + if (!ret) + continue; + pr_crit("Failed to roll back PCI assignment for %s: %d\n", + pci_name(assignment->vf), ret); + list_del_init(&assignment->transaction_node); + if (!rollback_ret) + rollback_ret = ret; + } + + return rollback_ret; +} + +static int mk_pci_commit_transaction(struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + int ret; + + list_for_each_entry(assignment, transaction, transaction_node) { + ret = mk_pci_commit_assignment(assignment); + if (ret) + return ret; + } + + while (!list_empty(transaction)) { + assignment = list_first_entry(transaction, + struct mk_pci_assignment, + transaction_node); + list_del_init(&assignment->transaction_node); + } + + return 0; +} + +void mk_pci_lease_instance_init(struct mk_instance *instance) +{ + INIT_DELAYED_WORK(&instance->irq_retry_work, mk_pci_irq_retry_workfn); +} + +void mk_pci_irq_retry_disable_sync(struct mk_instance *instance) +{ + disable_delayed_work_sync(&instance->irq_retry_work); +} + +void mk_pci_irq_retry_enable(struct mk_instance *instance) +{ + enable_delayed_work(&instance->irq_retry_work); +} + +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance) +{ + if (!instance) + return false; + + lockdep_assert_held(&instance->resource_mutex); + return !list_empty(&instance->pci_assignments); +} + +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count) +{ + struct mk_pci_device *requested; + LIST_HEAD(released); + LIST_HEAD(transaction); + int prepared = 0; + int ret = 0; + int rollback_ret; + + if (!instance || instance == mk_self || !requested_devices || + requested_count < 0) + return -EINVAL; + if (!mk_self || !mk_self->pci_devices_valid) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + list_for_each_entry(requested, requested_devices, list) { + ret = mk_pci_prepare_assignment(instance, requested, + &transaction); + if (ret) + goto rollback; + prepared++; + } + + if (prepared != requested_count) { + ret = -EINVAL; + goto rollback; + } + + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + rollback_ret = mk_pci_rollback_transaction(&transaction, &released); + if (rollback_ret) + ret = rollback_ret; +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); + return ret; +} + +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_device *inventory; + LIST_HEAD(released); + LIST_HEAD(transaction); + int ret; + int rollback_ret; + + if (!instance || instance == mk_self) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + if (!mk_self || !mk_self->pci_devices_valid) { + ret = -EINVAL; + goto out; + } + + inventory = mk_pci_find_root_bdf(domain, bus, devfn); + if (!inventory) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_prepare_assignment(instance, inventory, &transaction); + if (ret) + goto rollback; + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + rollback_ret = mk_pci_rollback_transaction(&transaction, &released); + if (rollback_ret) + ret = rollback_ret; +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); + return ret; +} + +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + LIST_HEAD(released); + int ret; + + if (!instance || instance == mk_self) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + + assignment = mk_pci_find_assignment(instance, domain, bus, devfn); + if (!assignment) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_release_assignment(assignment, &released); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); + return ret; +} + +int mk_pci_release_assignments(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + LIST_HEAD(released); + int ret = 0; + + if (!instance || instance == mk_self) + return 0; + + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + while (!list_empty(&instance->pci_assignments)) { + assignment = list_last_entry(&instance->pci_assignments, + struct mk_pci_assignment, + instance_node); + ret = mk_pci_release_assignment(assignment, &released); + if (ret) { + pr_crit("Instance %d retains unsafe PCI lease for %s: %d\n", + instance->id, pci_name(assignment->vf), ret); + break; + } + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + /* + * The assignment is no longer reachable by routed requests. Cancel its + * failure work after dropping the route and transaction locks because the + * worker may itself force-halt the instance and take both locks. + */ + mk_pci_finalize_releases(&released); + if (!ret) + cancel_delayed_work_sync(&instance->irq_retry_work); + return ret; +} + +int mk_pci_prepare_instance_start(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + int ret = 0; + + if (!instance || instance == mk_self) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + ret = mk_pci_reset_assignment_for_start(assignment); + if (ret) + break; + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +int mk_pci_lease_system_init(void) +{ + unsigned int i; + unsigned int pool_size; + int ret; + + ret = mk_pci_iommu_system_init(); + if (ret) + return ret; + ret = bus_register_notifier(&pci_bus_type, &mk_pci_bus_notifier); + if (ret) { + mk_pci_iommu_system_cleanup(); + return ret; + } + mk_pci_notifier_registered = true; + if (mk_self && mk_self->id == 0) { + pool_size = max_t(unsigned int, num_possible_cpus(), 1) * + MK_REPLY_SLOTS; + mk_pci_control_pool = kcalloc(pool_size, + sizeof(*mk_pci_control_pool), + GFP_KERNEL); + if (!mk_pci_control_pool) { + ret = -ENOMEM; + goto unregister_notifier; + } + mk_pci_control_pool_busy = bitmap_zalloc(pool_size, GFP_KERNEL); + if (!mk_pci_control_pool_busy) { + ret = -ENOMEM; + goto free_pool; + } + mk_pci_control_pool_size = pool_size; + atomic_set(&mk_pci_control_pool_cursor, 0); + for (i = 0; i < pool_size; i++) { + INIT_WORK(&mk_pci_control_pool[i].work, + mk_pci_control_work_fn); + mk_pci_control_pool[i].pool_slot = i; + } + mk_pci_control_wq = + alloc_workqueue("mk-pci-control", + WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!mk_pci_control_wq) { + ret = -ENOMEM; + goto free_pool_bitmap; + } + mk_pci_control_shutdown_end(); + ret = mk_register_msg_handler(MK_MSG_PCI, + mk_pci_control_msg_handler, NULL); + if (ret) + goto destroy_workqueue; + mk_pci_control_registered = true; + } + return 0; + +destroy_workqueue: + mk_pci_control_shutdown_begin(); + destroy_workqueue(mk_pci_control_wq); + mk_pci_control_wq = NULL; +free_pool_bitmap: + bitmap_free(mk_pci_control_pool_busy); + mk_pci_control_pool_busy = NULL; + mk_pci_control_pool_size = 0; +free_pool: + kfree(mk_pci_control_pool); + mk_pci_control_pool = NULL; +unregister_notifier: + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; + mk_pci_iommu_system_cleanup(); + return ret; +} + +void mk_pci_lease_system_cleanup(void) +{ + if (mk_pci_control_registered) { + mk_pci_control_shutdown_begin(); + mk_unregister_msg_handler(MK_MSG_PCI, + mk_pci_control_msg_handler); + mk_pci_control_registered = false; + wait_event(mk_pci_control_waitq, + !(atomic_read(&mk_pci_control_state) & + MK_PCI_CONTROL_ACTIVE_MASK)); + } + if (mk_pci_control_wq) { + destroy_workqueue(mk_pci_control_wq); + mk_pci_control_wq = NULL; + } + bitmap_free(mk_pci_control_pool_busy); + mk_pci_control_pool_busy = NULL; + mk_pci_control_pool_size = 0; + kfree(mk_pci_control_pool); + mk_pci_control_pool = NULL; + if (mk_pci_notifier_registered) { + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; + } + mk_pci_iommu_system_cleanup(); +} + +static struct mk_pci_device * +mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) +{ + if (!mk_self || mk_self->id == 0 || + !mk_self->pci_devices_valid) + return NULL; + + return mk_pci_find_device_bdf(&mk_self->pci_devices, + domain, bus, devfn); +} + +static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) +{ + return mk_pci_find_assigned_bdf(pci_domain_nr(bus), bus->number, devfn); +} + +/** + * mk_pci_get_assigned_identity_bdf - Get an assigned function's identity + * @domain: PCI domain number + * @bus: PCI bus number + * @devfn: device/function number + * @vendor: assigned Vendor ID + * @device_id: assigned Device ID + * + * Returns: true when assignment metadata contains an exact location match. + */ +bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, + u16 *device_id) +{ + struct mk_pci_device *device; + + if (domain != (u16)domain || bus != (u8)bus || devfn != (u8)devfn) + return false; + device = mk_pci_find_assigned_bdf(domain, bus, devfn); + + if (!device) + return false; + + if (vendor) + *vendor = device->vendor; + if (device_id) + *device_id = device->device; + return true; +} + +static void mk_pci_restore_resources(struct pci_dev *dev) +{ + struct mk_pci_device *device; + int i; + + device = mk_pci_find_assigned(dev->bus, dev->devfn); + if (!device || !device->resources_valid) + return; + + dev->non_compliant_bars = true; + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + dev->resource[i].start = device->resources[i].start; + dev->resource[i].end = device->resources[i].end; + dev->resource[i].flags = device->resources[i].flags; + } + pr_info("Restored PCI BAR resources for %s\n", pci_name(dev)); +} + +DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, mk_pci_restore_resources); diff --git a/net/vmw_vsock/mk_transport.c b/net/vmw_vsock/mk_transport.c index 84961b2f355aa2..086e1b32702d35 100644 --- a/net/vmw_vsock/mk_transport.c +++ b/net/vmw_vsock/mk_transport.c @@ -297,7 +297,8 @@ static void mk_vsock_rx_work(struct work_struct *work) } static void mk_vsock_ipi_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx) { struct sk_buff *skb; struct virtio_vsock_hdr *hdr;