multikernel: use per-instance duplex IPI links

arch/x86/kernel/smp.c

@@ -342,6 +342,8 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_multikernel)
  */
 void mk_arch_send_ipi(u64 phys_cpu)
 {
+	/* Order shared-memory publication before ringing the x2APIC doorbell. */
+	weak_wrmsr_fence();
 	apic_icr_write(APIC_DM_FIXED | APIC_DEST_PHYSICAL | MULTIKERNEL_VECTOR,
 		       (u32)phys_cpu);
 }

drivers/tty/mktty.c

@@ -337,6 +337,18 @@ static struct tty_driver *mktty_spawn_driver;
 static struct mktty_spawn_state mktty_spawn;
 static struct mk_ipi_handler *mktty_spawn_handler;
 static struct console mktty_spawn_console;
+
+static int mktty_spawn_parent_id(void)
+{
+	if (!mk_self)
+		return -ENODEV;
+	if (mk_self->id == 0)
+		return 0;
+	if (!host_instance)
+		return -ENODEV;
+	return READ_ONCE(host_instance->id);
+}
+
 static bool mktty_console_registered;
 
 static int mktty_spawn_activate(struct tty_port *port, struct tty_struct *tty)
@@ -377,10 +389,13 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf,
 {
 	struct mktty_message *msg;
 	size_t sent = 0, chunk;
-	int ret;
+	int parent_id, ret;
 
 	if (tty->index != 0)
 		return -ENODEV;
+	parent_id = mktty_spawn_parent_id();
+	if (parent_id < 0)
+		return parent_id;
 
 	msg = kmalloc(sizeof(*msg), GFP_KERNEL);
 	if (!msg)
@@ -394,9 +409,9 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf,
 		msg->reserved = 0;
 		memcpy(msg->data, buf + sent, chunk);
 
-		ret = multikernel_send_ipi_data(0, msg,
-				sizeof(*msg) - MKTTY_MAX_DATA + chunk,
-				MKTTY_IPI_TYPE);
+		ret = multikernel_send_ipi_data(parent_id, msg,
+						sizeof(*msg) - MKTTY_MAX_DATA + chunk,
+						MKTTY_IPI_TYPE);
 		if (ret < 0) {
 			kfree(msg);
 			return sent > 0 ? sent : ret;
@@ -449,8 +464,12 @@ static void mktty_spawn_console_write(struct console *con, const char *s,
 {
 	unsigned long flags;
 	size_t chunk;
+	int parent_id;
 
 	spin_lock_irqsave(&mktty_console_lock, flags);
+	parent_id = mktty_spawn_parent_id();
+	if (parent_id < 0)
+		goto out;
 	while (count > 0) {
 		chunk = min_t(size_t, count, MKTTY_MAX_DATA);
 		mktty_console_msg.type = MKTTY_MSG_OUTPUT;
@@ -459,12 +478,13 @@ static void mktty_spawn_console_write(struct console *con, const char *s,
 		mktty_console_msg.reserved = 0;
 		memcpy(mktty_console_msg.data, s, chunk);
 
-		multikernel_send_ipi_data(0, &mktty_console_msg,
-				sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk,
-				MKTTY_IPI_TYPE);
+		multikernel_send_ipi_data(parent_id, &mktty_console_msg,
+					  sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk,
+					  MKTTY_IPI_TYPE);
 		s += chunk;
 		count -= chunk;
 	}
+out:
 	spin_unlock_irqrestore(&mktty_console_lock, flags);
 }
 

include/linux/multikernel.h

@@ -6,7 +6,9 @@
 #define _LINUX_MULTIKERNEL_H
 
 #include <linux/types.h>
-#include <linux/irq_work.h>
+#include <linux/list.h>
+#include <linux/rcupdate.h>
+#include <linux/spinlock.h>
 #include <linux/kobject.h>
 #include <linux/kernfs.h>
 #include <linux/ioport.h>
@@ -88,6 +90,7 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set)
 
 /* Data structure for passing parameters via IPI */
 struct mk_ipi_data {
+	u32 ready;
 	u64 sender_cpu;          /* Physical ID of the CPU that sent this IPI */
 	unsigned int type;      /* User-defined type identifier */
 	size_t data_size;        /* Size of the data */
@@ -96,8 +99,6 @@ struct mk_ipi_data {
 
 /* IPI ring buffer for queuing messages */
 struct mk_ipi_ring {
-	atomic_t head;                          /* Producer index */
-	atomic_t tail;                          /* Consumer index */
 	struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */
 };
 
@@ -109,9 +110,10 @@ struct mk_ipi_ring {
 /* Presence table capacity, in CPUs (see mk_cpu_rank) */
 #define MK_PARKED_MAX 512
 
-/* Shared memory structures - per-instance design */
+/* One duplex link per parent/child pair. Each ring has one kernel producer. */
 struct mk_shared_data {
-	struct mk_ipi_ring ring;  /* IPI message ring buffer */
+	struct mk_ipi_ring to_child;
+	struct mk_ipi_ring to_parent;
 	/*
 	 * Force-halt marker, host-owned. Armed before the host NMIs the
 	 * instance's CPUs and cleared with the rest of this struct when
@@ -133,14 +135,30 @@ struct mk_shared_data {
 	u8 parked[MK_PARKED_MAX];
 };
 
+struct mk_instance;
+
+struct mk_ipi_endpoint {
+	struct mk_ipi_ring *tx;
+	struct mk_ipi_ring *rx;
+	raw_spinlock_t tx_lock;
+	raw_spinlock_t rx_lock;
+	u32 tx_head;
+	u32 rx_tail;
+	mk_phys_cpu_t doorbell_cpu;
+	bool tx_enabled;
+	bool rx_dispatching;
+	bool registered;
+	struct list_head rx_node;
+};
+
 /* Function pointer type for IPI callbacks */
 typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *data, void *ctx);
 
 struct mk_ipi_handler {
 	mk_ipi_callback_t callback;
 	void *context;
 	unsigned int ipi_type;       /* IPI type this handler is registered for */
-	struct mk_ipi_handler *next;
+	struct list_head node;
 };
 
 /**
@@ -156,6 +174,9 @@ struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback,
 /**
  * multikernel_unregister_handler - Unregister a multikernel IPI callback
  * @handler: Handler pointer returned from multikernel_register_handler
+ *
+ * Must be called from process context and not from a multikernel IPI
+ * callback.
  */
 void multikernel_unregister_handler(struct mk_ipi_handler *handler);
 
@@ -166,8 +187,8 @@ void multikernel_unregister_handler(struct mk_ipi_handler *handler);
  * @data_size: Size of data
  * @type: User-defined type identifier
  *
- * This function copies the data to per-CPU storage and sends an IPI
- * to the target CPU.
+ * This function copies the data into the target instance's duplex ring and
+ * sends an IPI to its doorbell CPU.
  *
  * Returns 0 on success, negative error code on failure
  */
@@ -183,8 +204,40 @@ struct mk_instance;
 int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data,
 				 size_t data_size, unsigned long type);
 
-/* Discard everything queued in this kernel's ring (instance re-spawn) */
-void mk_ipi_ring_drop_pending(void);
+#ifdef CONFIG_MULTIKERNEL
+int mk_ipi_endpoint_register(struct mk_instance *instance, bool parent_side,
+			     mk_phys_cpu_t doorbell_cpu);
+void mk_ipi_endpoint_unregister(struct mk_instance *instance);
+int mk_ipi_link_reset(struct mk_instance *instance);
+void mk_ipi_handlers_enable(void);
+struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance);
+#else
+static inline int mk_ipi_endpoint_register(struct mk_instance *instance,
+					   bool parent_side,
+					   mk_phys_cpu_t doorbell_cpu)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline void mk_ipi_endpoint_unregister(struct mk_instance *instance)
+{
+}
+
+static inline int mk_ipi_link_reset(struct mk_instance *instance)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline void mk_ipi_handlers_enable(void)
+{
+}
+
+static inline struct mk_shared_data *
+mk_instance_halt_data(struct mk_instance *instance)
+{
+	return NULL;
+}
+#endif
 
 /*
  * Multikernel Messaging System
@@ -738,6 +791,13 @@ struct mk_instance {
 	struct mk_shared_data *ipi_data; /* IPI shared memory buffer (virtual address) */
 	phys_addr_t ipi_phys;           /* IPI buffer physical address */
 	u32 ipi_pages;                  /* IPI buffer size in pages */
+	/*
+	 * A spawn's separate mapping of its parent's force-halt area.
+	 * Ordinary child records keep this NULL and use @ipi_data.
+	 */
+	struct mk_shared_data *halt_data;
+	struct mk_ipi_endpoint ipi_endpoint;
+	u64 ipi_start_sequence; /* Local start identity for deferred halt work */
 	/*
 	 * Host side only: the page the instance's boot tree is written to at
 	 * each exec. It and the IPI ring are allocated on the first load and
@@ -1157,6 +1217,7 @@ static inline void mk_register_cpus_from_manifest(void)
 static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len)
 {
 }
+
 #endif
 
 /**

kernel/kexec_core.c

@@ -1676,6 +1676,10 @@ int multikernel_kexec_by_id(int mk_id)
 	}
 
 	instance = mk_image->mk_instance;
+	if (instance->state != MK_STATE_LOADED) {
+		rc = -EBUSY;
+		goto unlock;
+	}
 	if (!mk_cpu_set_empty(instance->cpus)) {
 		mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus);
 
@@ -1707,6 +1711,13 @@ int multikernel_kexec_by_id(int mk_id)
 		goto unlock;
 	}
 
+	mk_ipi_endpoint_unregister(instance);
+	rc = mk_ipi_link_reset(instance);
+	if (rc) {
+		pr_err("Failed to reset instance %d IPI link: %d\n", mk_id, rc);
+		goto unlock;
+	}
+
 	/*
 	 * Booting consumes the image: the spawn kernel writes its .data and
 	 * patches its own text, so the copy in instance memory is spent once