FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_ipi.c

     /*      $NetBSD: subr_ipi.c,v 1.10 2022/04/09 23:51:22 riastradh Exp $  */
     
     /*-
      * Copyright (c) 2014 The NetBSD Foundation, Inc.
      * All rights reserved.
      *
      * This code is derived from software contributed to The NetBSD Foundation
      * by Mindaugas Rasiukevicius.
      *
     * Redistribution and use in source and binary forms, with or without
     * modification, are permitted provided that the following conditions
     * are met:
     * 1. Redistributions of source code must retain the above copyright
     *    notice, this list of conditions and the following disclaimer.
     * 2. Redistributions in binary form must reproduce the above copyright
     *    notice, this list of conditions and the following disclaimer in the
     *    documentation and/or other materials provided with the distribution.
     *
     * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     * POSSIBILITY OF SUCH DAMAGE.
     */
    
    /*
     * Inter-processor interrupt (IPI) interface: asynchronous IPIs to
     * invoke functions with a constant argument and synchronous IPIs
     * with the cross-call support.
     */
    
    #include <sys/cdefs.h>
    __KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.10 2022/04/09 23:51:22 riastradh Exp $");
    
    #include <sys/param.h>
    #include <sys/types.h>
    
    #include <sys/atomic.h>
    #include <sys/evcnt.h>
    #include <sys/cpu.h>
    #include <sys/ipi.h>
    #include <sys/intr.h>
    #include <sys/kcpuset.h>
    #include <sys/kmem.h>
    #include <sys/lock.h>
    #include <sys/mutex.h>
    
    /*
     * An array of the IPI handlers used for asynchronous invocation.
     * The lock protects the slot allocation.
     */
    
    typedef struct {
            ipi_func_t      func;
            void *          arg;
    } ipi_intr_t;
    
    static kmutex_t         ipi_mngmt_lock;
    static ipi_intr_t       ipi_intrs[IPI_MAXREG]   __cacheline_aligned;
    
    /*
     * Per-CPU mailbox for IPI messages: it is a single cache line storing
     * up to IPI_MSG_MAX messages.  This interface is built on top of the
     * synchronous IPIs.
     */
    
    #define IPI_MSG_SLOTS   (CACHE_LINE_SIZE / sizeof(ipi_msg_t *))
    #define IPI_MSG_MAX     IPI_MSG_SLOTS
    
    typedef struct {
            ipi_msg_t *     msg[IPI_MSG_SLOTS];
    } ipi_mbox_t;
    
    
    /* Mailboxes for the synchronous IPIs. */
    static ipi_mbox_t *     ipi_mboxes      __read_mostly;
    static struct evcnt     ipi_mboxfull_ev __cacheline_aligned;
    static void             ipi_msg_cpu_handler(void *);
    
    /* Handler for the synchronous IPIs - it must be zero. */
    #define IPI_SYNCH_ID    0
    
    #ifndef MULTIPROCESSOR
    #define cpu_ipi(ci)     KASSERT(ci == NULL)
    #endif
    
    void
    ipi_sysinit(void)
    {
    
            mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE);
            memset(ipi_intrs, 0, sizeof(ipi_intrs));
    
           /*
            * Register the handler for synchronous IPIs.  This mechanism
            * is built on top of the asynchronous interface.  Slot zero is
            * reserved permanently; it is also handy to use zero as a failure
            * for other registers (as it is potentially less error-prone).
            */
           ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler;
   
           evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL,
              "ipi", "full");
   }
   
   void
   ipi_percpu_init(void)
   {
           const size_t len = ncpu * sizeof(ipi_mbox_t);
   
           /* Initialise the per-CPU bit fields. */
           for (u_int i = 0; i < ncpu; i++) {
                   struct cpu_info *ci = cpu_lookup(i);
                   memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend));
           }
   
           /* Allocate per-CPU IPI mailboxes. */
           ipi_mboxes = kmem_zalloc(len, KM_SLEEP);
           KASSERT(ipi_mboxes != NULL);
   }
   
   /*
    * ipi_register: register an asynchronous IPI handler.
    *
    * => Returns IPI ID which is greater than zero; on failure - zero.
    */
   u_int
   ipi_register(ipi_func_t func, void *arg)
   {
           mutex_enter(&ipi_mngmt_lock);
           for (u_int i = 0; i < IPI_MAXREG; i++) {
                   if (ipi_intrs[i].func == NULL) {
                           /* Register the function. */
                           ipi_intrs[i].func = func;
                           ipi_intrs[i].arg = arg;
                           mutex_exit(&ipi_mngmt_lock);
   
                           KASSERT(i != IPI_SYNCH_ID);
                           return i;
                   }
           }
           mutex_exit(&ipi_mngmt_lock);
           printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n");
           return 0;
   }
   
   /*
    * ipi_unregister: release the IPI handler given the ID.
    */
   void
   ipi_unregister(u_int ipi_id)
   {
           ipi_msg_t ipimsg = { .func = __FPTRCAST(ipi_func_t, nullop) };
   
           KASSERT(ipi_id != IPI_SYNCH_ID);
           KASSERT(ipi_id < IPI_MAXREG);
   
           /* Release the slot. */
           mutex_enter(&ipi_mngmt_lock);
           KASSERT(ipi_intrs[ipi_id].func != NULL);
           ipi_intrs[ipi_id].func = NULL;
   
           /* Ensure that there are no IPIs in flight. */
           kpreempt_disable();
           ipi_broadcast(&ipimsg, false);
           ipi_wait(&ipimsg);
           kpreempt_enable();
           mutex_exit(&ipi_mngmt_lock);
   }
   
   /*
    * ipi_mark_pending: internal routine to mark an IPI pending on the
    * specified CPU (which might be curcpu()).
    */
   static bool
   ipi_mark_pending(u_int ipi_id, struct cpu_info *ci)
   {
           const u_int i = ipi_id >> IPI_BITW_SHIFT;
           const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK);
   
           KASSERT(ipi_id < IPI_MAXREG);
           KASSERT(kpreempt_disabled());
   
           /* Mark as pending and return true if not previously marked. */
           if ((atomic_load_acquire(&ci->ci_ipipend[i]) & bitm) == 0) {
   #ifndef __HAVE_ATOMIC_AS_MEMBAR
                   membar_release();
   #endif
                   atomic_or_32(&ci->ci_ipipend[i], bitm);
                   return true;
           }
           return false;
   }
   
   /*
    * ipi_trigger: asynchronously send an IPI to the specified CPU.
    */
   void
   ipi_trigger(u_int ipi_id, struct cpu_info *ci)
   {
   
           KASSERT(curcpu() != ci);
           if (ipi_mark_pending(ipi_id, ci)) {
                   cpu_ipi(ci);
           }
   }
   
   /*
    * ipi_trigger_multi_internal: the guts of ipi_trigger_multi() and
    * ipi_trigger_broadcast().
    */
   static void
   ipi_trigger_multi_internal(u_int ipi_id, const kcpuset_t *target,
       bool skip_self)
   {
           const cpuid_t selfid = cpu_index(curcpu());
           CPU_INFO_ITERATOR cii;
           struct cpu_info *ci;
   
           KASSERT(kpreempt_disabled());
           KASSERT(target != NULL);
   
           for (CPU_INFO_FOREACH(cii, ci)) {
                   const cpuid_t cpuid = cpu_index(ci);
   
                   if (!kcpuset_isset(target, cpuid) || cpuid == selfid) {
                           continue;
                   }
                   ipi_trigger(ipi_id, ci);
           }
           if (!skip_self && kcpuset_isset(target, selfid)) {
                   ipi_mark_pending(ipi_id, curcpu());
                   int s = splhigh();
                   ipi_cpu_handler();
                   splx(s);
           }
   }
   
   /*
    * ipi_trigger_multi: same as ipi_trigger() but sends to the multiple
    * CPUs given the target CPU set.
    */
   void
   ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target)
   {
           ipi_trigger_multi_internal(ipi_id, target, false);
   }
   
   /*
    * ipi_trigger_broadcast: same as ipi_trigger_multi() to kcpuset_attached,
    * optionally skipping the sending CPU.
    */
   void
   ipi_trigger_broadcast(u_int ipi_id, bool skip_self)
   {
           ipi_trigger_multi_internal(ipi_id, kcpuset_attached, skip_self);
   }
   
   /*
    * put_msg: insert message into the mailbox.
    *
    * Caller is responsible for issuing membar_release first.
    */
   static inline void
   put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg)
   {
           int count = SPINLOCK_BACKOFF_MIN;
   again:
           for (u_int i = 0; i < IPI_MSG_MAX; i++) {
                   if (atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) {
                           return;
                   }
           }
   
           /* All slots are full: we have to spin-wait. */
           ipi_mboxfull_ev.ev_count++;
           SPINLOCK_BACKOFF(count);
           goto again;
   }
   
   /*
    * ipi_cpu_handler: the IPI handler.
    */
   void
   ipi_cpu_handler(void)
   {
           struct cpu_info * const ci = curcpu();
   
           /*
            * Handle asynchronous IPIs: inspect per-CPU bit field, extract
            * IPI ID numbers and execute functions in those slots.
            */
           for (u_int i = 0; i < IPI_BITWORDS; i++) {
                   uint32_t pending, bit;
   
                   if (atomic_load_relaxed(&ci->ci_ipipend[i]) == 0) {
                           continue;
                   }
                   pending = atomic_swap_32(&ci->ci_ipipend[i], 0);
   #ifndef __HAVE_ATOMIC_AS_MEMBAR
                   membar_acquire();
   #endif
                   while ((bit = ffs(pending)) != 0) {
                           const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit;
                           ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id];
   
                           pending &= ~(1U << bit);
                           KASSERT(ipi_hdl->func != NULL);
                           ipi_hdl->func(ipi_hdl->arg);
                   }
           }
   }
   
   /*
    * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox,
    * execute the passed functions and acknowledge the messages.
    */
   static void
   ipi_msg_cpu_handler(void *arg __unused)
   {
           const struct cpu_info * const ci = curcpu();
           ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)];
   
           for (u_int i = 0; i < IPI_MSG_MAX; i++) {
                   ipi_msg_t *msg;
   
                   /* Get the message. */
                   if ((msg = atomic_load_acquire(&mbox->msg[i])) == NULL) {
                           continue;
                   }
                   atomic_store_relaxed(&mbox->msg[i], NULL);
   
                   /* Execute the handler. */
                   KASSERT(msg->func);
                   msg->func(msg->arg);
   
                   /* Ack the request. */
   #ifndef __HAVE_ATOMIC_AS_MEMBAR
                   membar_release();
   #endif
                   atomic_dec_uint(&msg->_pending);
           }
   }
   
   /*
    * ipi_unicast: send an IPI to a single CPU.
    *
    * => The CPU must be remote; must not be local.
    * => The caller must ipi_wait() on the message for completion.
    */
   void
   ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci)
   {
           const cpuid_t id = cpu_index(ci);
   
           KASSERT(msg->func != NULL);
           KASSERT(kpreempt_disabled());
           KASSERT(curcpu() != ci);
   
           msg->_pending = 1;
   #ifndef __HAVE_ATOMIC_AS_MEMBAR
           membar_release();
   #endif
   
           put_msg(&ipi_mboxes[id], msg);
           ipi_trigger(IPI_SYNCH_ID, ci);
   }
   
   /*
    * ipi_multicast: send an IPI to each CPU in the specified set.
    *
    * => The caller must ipi_wait() on the message for completion.
    */
   void
   ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target)
   {
           const struct cpu_info * const self = curcpu();
           CPU_INFO_ITERATOR cii;
           struct cpu_info *ci;
           u_int local;
   
           KASSERT(msg->func != NULL);
           KASSERT(kpreempt_disabled());
   
           local = !!kcpuset_isset(target, cpu_index(self));
           msg->_pending = kcpuset_countset(target) - local;
   #ifndef __HAVE_ATOMIC_AS_MEMBAR
           membar_release();
   #endif
   
           for (CPU_INFO_FOREACH(cii, ci)) {
                   cpuid_t id;
   
                   if (__predict_false(ci == self)) {
                           continue;
                   }
                   id = cpu_index(ci);
                   if (!kcpuset_isset(target, id)) {
                           continue;
                   }
                   put_msg(&ipi_mboxes[id], msg);
                   ipi_trigger(IPI_SYNCH_ID, ci);
           }
           if (local) {
                   msg->func(msg->arg);
           }
   }
   
   /*
    * ipi_broadcast: send an IPI to all CPUs.
    *
    * => The caller must ipi_wait() on the message for completion.
    */
   void
   ipi_broadcast(ipi_msg_t *msg, bool skip_self)
   {
           const struct cpu_info * const self = curcpu();
           CPU_INFO_ITERATOR cii;
           struct cpu_info *ci;
   
           KASSERT(msg->func != NULL);
           KASSERT(kpreempt_disabled());
   
           msg->_pending = ncpu - 1;
   #ifndef __HAVE_ATOMIC_AS_MEMBAR
           membar_release();
   #endif
   
           /* Broadcast IPIs for remote CPUs. */
           for (CPU_INFO_FOREACH(cii, ci)) {
                   cpuid_t id;
   
                   if (__predict_false(ci == self)) {
                           continue;
                   }
                   id = cpu_index(ci);
                   put_msg(&ipi_mboxes[id], msg);
                   ipi_trigger(IPI_SYNCH_ID, ci);
           }
   
           if (!skip_self) {
                   /* Finally, execute locally. */
                   msg->func(msg->arg);
           }
   }
   
   /*
    * ipi_wait: spin-wait until the message is processed.
    */
   void
   ipi_wait(ipi_msg_t *msg)
   {
           int count = SPINLOCK_BACKOFF_MIN;
   
           while (atomic_load_acquire(&msg->_pending)) {
                   KASSERT(atomic_load_relaxed(&msg->_pending) < ncpu);
                   SPINLOCK_BACKOFF(count);
           }
   }

Cache object: 8b7d1f4e67db5d0a3f0fb2c39455c4af