The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_xcall.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: subr_xcall.c,v 1.34 2020/12/22 01:57:29 ad Exp $       */
    2 
    3 /*-
    4  * Copyright (c) 2007-2010, 2019 The NetBSD Foundation, Inc.
    5  * All rights reserved.
    6  *
    7  * This code is derived from software contributed to The NetBSD Foundation
    8  * by Andrew Doran and Mindaugas Rasiukevicius.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  *
   19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
   20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
   21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
   23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   29  * POSSIBILITY OF SUCH DAMAGE.
   30  */
   31 
   32 /*
   33  * Cross call support
   34  *
   35  * Background
   36  *
   37  *      Sometimes it is necessary to modify hardware state that is tied
   38  *      directly to individual CPUs (such as a CPU's local timer), and
   39  *      these updates can not be done remotely by another CPU.  The LWP
   40  *      requesting the update may be unable to guarantee that it will be
   41  *      running on the CPU where the update must occur, when the update
   42  *      occurs.
   43  *
   44  *      Additionally, it's sometimes necessary to modify per-CPU software
   45  *      state from a remote CPU.  Where these update operations are so
   46  *      rare or the access to the per-CPU data so frequent that the cost
   47  *      of using locking or atomic operations to provide coherency is
   48  *      prohibitive, another way must be found.
   49  *
   50  *      Cross calls help to solve these types of problem by allowing
   51  *      any LWP in the system to request that an arbitrary function be
   52  *      executed on a specific CPU.
   53  *
   54  * Implementation
   55  *
   56  *      A slow mechanism for making low priority cross calls is
   57  *      provided.  The function to be executed runs on the remote CPU
   58  *      within a bound kthread.  No queueing is provided, and the
   59  *      implementation uses global state.  The function being called may
   60  *      block briefly on locks, but in doing so must be careful to not
   61  *      interfere with other cross calls in the system.  The function is
   62  *      called with thread context and not from a soft interrupt, so it
   63  *      can ensure that it is not interrupting other code running on the
   64  *      CPU, and so has exclusive access to the CPU.  Since this facility
   65  *      is heavyweight, it's expected that it will not be used often.
   66  *
   67  *      Cross calls must not allocate memory, as the pagedaemon uses cross
   68  *      calls (and memory allocation may need to wait on the pagedaemon).
   69  *
   70  *      A low-overhead mechanism for high priority calls (XC_HIGHPRI) is
   71  *      also provided.  The function to be executed runs in software
   72  *      interrupt context at IPL_SOFTSERIAL level, and is expected to
   73  *      be very lightweight, e.g. avoid blocking.
   74  */
   75 
   76 #include <sys/cdefs.h>
   77 __KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.34 2020/12/22 01:57:29 ad Exp $");
   78 
   79 #include <sys/types.h>
   80 #include <sys/param.h>
   81 #include <sys/xcall.h>
   82 #include <sys/mutex.h>
   83 #include <sys/condvar.h>
   84 #include <sys/evcnt.h>
   85 #include <sys/kthread.h>
   86 #include <sys/cpu.h>
   87 #include <sys/atomic.h>
   88 
   89 #ifdef _RUMPKERNEL
   90 #include "rump_private.h"
   91 #endif
   92 
   93 /* Cross-call state box. */
   94 typedef struct {
   95         kmutex_t        xc_lock;
   96         kcondvar_t      xc_busy;
   97         xcfunc_t        xc_func;
   98         void *          xc_arg1;
   99         void *          xc_arg2;
  100         uint64_t        xc_headp;
  101         uint64_t        xc_donep;
  102         unsigned int    xc_ipl;
  103 } xc_state_t;
  104 
  105 /* Bit indicating high (1) or low (0) priority. */
  106 #define XC_PRI_BIT      (1ULL << 63)
  107 
  108 /* Low priority xcall structures. */
  109 static xc_state_t       xc_low_pri      __cacheline_aligned;
  110 
  111 /* High priority xcall structures. */
  112 static xc_state_t       xc_high_pri     __cacheline_aligned;
  113 static void *           xc_sihs[4]      __cacheline_aligned;
  114 
  115 /* Event counters. */
  116 static struct evcnt     xc_unicast_ev   __cacheline_aligned;
  117 static struct evcnt     xc_broadcast_ev __cacheline_aligned;
  118 
  119 static void             xc_init(void);
  120 static void             xc_thread(void *);
  121 
  122 static inline uint64_t  xc_highpri(xcfunc_t, void *, void *, struct cpu_info *,
  123                             unsigned int);
  124 static inline uint64_t  xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *);
  125 
  126 /* The internal form of IPL */
  127 #define XC_IPL_MASK             0xff00
  128 /*
  129  * Assign 0 to XC_IPL_SOFTSERIAL to treat IPL_SOFTSERIAL as the default value
  130  * (just XC_HIGHPRI).
  131  */
  132 #define XC_IPL_SOFTSERIAL       0
  133 #define XC_IPL_SOFTNET          1
  134 #define XC_IPL_SOFTBIO          2
  135 #define XC_IPL_SOFTCLOCK        3
  136 #define XC_IPL_MAX              XC_IPL_SOFTCLOCK
  137 
  138 CTASSERT(XC_IPL_MAX <= __arraycount(xc_sihs));
  139 
  140 /*
  141  * xc_init:
  142  *
  143  *      Initialize low and high priority cross-call structures.
  144  */
  145 static void
  146 xc_init(void)
  147 {
  148         xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri;
  149 
  150         memset(xclo, 0, sizeof(xc_state_t));
  151         mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE);
  152         cv_init(&xclo->xc_busy, "xclocv");
  153 
  154         memset(xchi, 0, sizeof(xc_state_t));
  155         mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
  156         cv_init(&xchi->xc_busy, "xchicv");
  157 
  158         /* Set up a softint for each IPL_SOFT*. */
  159 #define SETUP_SOFTINT(xipl, sipl) do {                                  \
  160                 xc_sihs[(xipl)] = softint_establish( (sipl) | SOFTINT_MPSAFE,\
  161                     xc__highpri_intr, NULL);                            \
  162                 KASSERT(xc_sihs[(xipl)] != NULL);                       \
  163         } while (0)
  164 
  165         SETUP_SOFTINT(XC_IPL_SOFTSERIAL, SOFTINT_SERIAL);
  166         /*
  167          * If a IPL_SOFTXXX have the same value of the previous, we don't use
  168          * the IPL (see xc_encode_ipl).  So we don't need to allocate a softint
  169          * for it.
  170          */
  171 #if IPL_SOFTNET != IPL_SOFTSERIAL
  172         SETUP_SOFTINT(XC_IPL_SOFTNET, SOFTINT_NET);
  173 #endif
  174 #if IPL_SOFTBIO != IPL_SOFTNET
  175         SETUP_SOFTINT(XC_IPL_SOFTBIO, SOFTINT_BIO);
  176 #endif
  177 #if IPL_SOFTCLOCK != IPL_SOFTBIO
  178         SETUP_SOFTINT(XC_IPL_SOFTCLOCK, SOFTINT_CLOCK);
  179 #endif
  180 
  181 #undef SETUP_SOFTINT
  182 
  183         evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL,
  184            "crosscall", "unicast");
  185         evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL,
  186            "crosscall", "broadcast");
  187 }
  188 
  189 /*
  190  * Encode an IPL to a form that can be embedded into flags of xc_broadcast
  191  * or xc_unicast.
  192  */
  193 unsigned int
  194 xc_encode_ipl(int ipl)
  195 {
  196 
  197         switch (ipl) {
  198         case IPL_SOFTSERIAL:
  199                 return __SHIFTIN(XC_IPL_SOFTSERIAL, XC_IPL_MASK);
  200         /* IPL_SOFT* can be the same value (e.g., on sparc or mips). */
  201 #if IPL_SOFTNET != IPL_SOFTSERIAL
  202         case IPL_SOFTNET:
  203                 return __SHIFTIN(XC_IPL_SOFTNET, XC_IPL_MASK);
  204 #endif
  205 #if IPL_SOFTBIO != IPL_SOFTNET
  206         case IPL_SOFTBIO:
  207                 return __SHIFTIN(XC_IPL_SOFTBIO, XC_IPL_MASK);
  208 #endif
  209 #if IPL_SOFTCLOCK != IPL_SOFTBIO
  210         case IPL_SOFTCLOCK:
  211                 return __SHIFTIN(XC_IPL_SOFTCLOCK, XC_IPL_MASK);
  212 #endif
  213         }
  214 
  215         panic("Invalid IPL: %d", ipl);
  216 }
  217 
  218 /*
  219  * Extract an XC_IPL from flags of xc_broadcast or xc_unicast.
  220  */
  221 static inline unsigned int
  222 xc_extract_ipl(unsigned int flags)
  223 {
  224 
  225         return __SHIFTOUT(flags, XC_IPL_MASK);
  226 }
  227 
  228 /*
  229  * xc_init_cpu:
  230  *
  231  *      Initialize the cross-call subsystem.  Called once for each CPU
  232  *      in the system as they are attached.
  233  */
  234 void
  235 xc_init_cpu(struct cpu_info *ci)
  236 {
  237         static bool again = false;
  238         int error __diagused;
  239 
  240         if (!again) {
  241                 /* Autoconfiguration will prevent re-entry. */
  242                 xc_init();
  243                 again = true;
  244         }
  245         cv_init(&ci->ci_data.cpu_xcall, "xcall");
  246         error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread,
  247             NULL, NULL, "xcall/%u", ci->ci_index);
  248         KASSERT(error == 0);
  249 }
  250 
  251 /*
  252  * xc_broadcast:
  253  *
  254  *      Trigger a call on all CPUs in the system.
  255  */
  256 uint64_t
  257 xc_broadcast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2)
  258 {
  259 
  260         KASSERT(!cpu_intr_p() && !cpu_softintr_p());
  261         ASSERT_SLEEPABLE();
  262 
  263         if (__predict_false(!mp_online)) {
  264                 (*func)(arg1, arg2);
  265                 return 0;
  266         }
  267 
  268         if ((flags & XC_HIGHPRI) != 0) {
  269                 int ipl = xc_extract_ipl(flags);
  270                 return xc_highpri(func, arg1, arg2, NULL, ipl);
  271         } else {
  272                 return xc_lowpri(func, arg1, arg2, NULL);
  273         }
  274 }
  275 
  276 static void
  277 xc_nop(void *arg1, void *arg2)
  278 {
  279 
  280         return;
  281 }
  282 
  283 /*
  284  * xc_barrier:
  285  *
  286  *      Broadcast a nop to all CPUs in the system.
  287  */
  288 void
  289 xc_barrier(unsigned int flags)
  290 {
  291         uint64_t where;
  292 
  293         where = xc_broadcast(flags, xc_nop, NULL, NULL);
  294         xc_wait(where);
  295 }
  296 
  297 /*
  298  * xc_unicast:
  299  *
  300  *      Trigger a call on one CPU.
  301  */
  302 uint64_t
  303 xc_unicast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2,
  304            struct cpu_info *ci)
  305 {
  306         int s;
  307 
  308         KASSERT(ci != NULL);
  309         KASSERT(!cpu_intr_p() && !cpu_softintr_p());
  310         ASSERT_SLEEPABLE();
  311 
  312         if (__predict_false(!mp_online)) {
  313                 KASSERT(ci == curcpu());
  314                 s = splsoftserial();
  315                 (*func)(arg1, arg2);
  316                 splx(s);
  317                 return 0;
  318         }
  319 
  320         if ((flags & XC_HIGHPRI) != 0) {
  321                 int ipl = xc_extract_ipl(flags);
  322                 return xc_highpri(func, arg1, arg2, ci, ipl);
  323         } else {
  324                 return xc_lowpri(func, arg1, arg2, ci);
  325         }
  326 }
  327 
  328 /*
  329  * xc_wait:
  330  *
  331  *      Wait for a cross call to complete.
  332  */
  333 void
  334 xc_wait(uint64_t where)
  335 {
  336         xc_state_t *xc;
  337 
  338         KASSERT(!cpu_intr_p() && !cpu_softintr_p());
  339         ASSERT_SLEEPABLE();
  340 
  341         if (__predict_false(!mp_online)) {
  342                 return;
  343         }
  344 
  345         /* Determine whether it is high or low priority cross-call. */
  346         if ((where & XC_PRI_BIT) != 0) {
  347                 xc = &xc_high_pri;
  348                 where &= ~XC_PRI_BIT;
  349         } else {
  350                 xc = &xc_low_pri;
  351         }
  352 
  353 #ifdef __HAVE_ATOMIC64_LOADSTORE
  354         /* Fast path, if already done. */
  355         if (atomic_load_acquire(&xc->xc_donep) >= where) {
  356                 return;
  357         }
  358 #endif
  359 
  360         /* Slow path: block until awoken. */
  361         mutex_enter(&xc->xc_lock);
  362         while (xc->xc_donep < where) {
  363                 cv_wait(&xc->xc_busy, &xc->xc_lock);
  364         }
  365         mutex_exit(&xc->xc_lock);
  366 }
  367 
  368 /*
  369  * xc_lowpri:
  370  *
  371  *      Trigger a low priority call on one or more CPUs.
  372  */
  373 static inline uint64_t
  374 xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci)
  375 {
  376         xc_state_t *xc = &xc_low_pri;
  377         CPU_INFO_ITERATOR cii;
  378         uint64_t where;
  379 
  380         mutex_enter(&xc->xc_lock);
  381         while (xc->xc_headp != xc->xc_donep) {
  382                 cv_wait(&xc->xc_busy, &xc->xc_lock);
  383         }
  384         xc->xc_arg1 = arg1;
  385         xc->xc_arg2 = arg2;
  386         xc->xc_func = func;
  387         if (ci == NULL) {
  388                 xc_broadcast_ev.ev_count++;
  389                 for (CPU_INFO_FOREACH(cii, ci)) {
  390                         if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0)
  391                                 continue;
  392                         xc->xc_headp += 1;
  393                         ci->ci_data.cpu_xcall_pending = true;
  394                         cv_signal(&ci->ci_data.cpu_xcall);
  395                 }
  396         } else {
  397                 xc_unicast_ev.ev_count++;
  398                 xc->xc_headp += 1;
  399                 ci->ci_data.cpu_xcall_pending = true;
  400                 cv_signal(&ci->ci_data.cpu_xcall);
  401         }
  402         KASSERT(xc->xc_donep < xc->xc_headp);
  403         where = xc->xc_headp;
  404         mutex_exit(&xc->xc_lock);
  405 
  406         /* Return a low priority ticket. */
  407         KASSERT((where & XC_PRI_BIT) == 0);
  408         return where;
  409 }
  410 
  411 /*
  412  * xc_thread:
  413  *
  414  *      One thread per-CPU to dispatch low priority calls.
  415  */
  416 static void
  417 xc_thread(void *cookie)
  418 {
  419         struct cpu_info *ci = curcpu();
  420         xc_state_t *xc = &xc_low_pri;
  421         void *arg1, *arg2;
  422         xcfunc_t func;
  423 
  424         mutex_enter(&xc->xc_lock);
  425         for (;;) {
  426                 while (!ci->ci_data.cpu_xcall_pending) {
  427                         if (xc->xc_headp == xc->xc_donep) {
  428                                 cv_broadcast(&xc->xc_busy);
  429                         }
  430                         cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock);
  431                         KASSERT(ci == curcpu());
  432                 }
  433                 ci->ci_data.cpu_xcall_pending = false;
  434                 func = xc->xc_func;
  435                 arg1 = xc->xc_arg1;
  436                 arg2 = xc->xc_arg2;
  437                 mutex_exit(&xc->xc_lock);
  438 
  439                 KASSERT(func != NULL);
  440                 (*func)(arg1, arg2);
  441 
  442                 mutex_enter(&xc->xc_lock);
  443 #ifdef __HAVE_ATOMIC64_LOADSTORE
  444                 atomic_store_release(&xc->xc_donep, xc->xc_donep + 1);
  445 #else
  446                 xc->xc_donep++;
  447 #endif
  448         }
  449         /* NOTREACHED */
  450 }
  451 
  452 /*
  453  * xc_ipi_handler:
  454  *
  455  *      Handler of cross-call IPI.
  456  */
  457 void
  458 xc_ipi_handler(void)
  459 {
  460         xc_state_t *xc = & xc_high_pri;
  461 
  462         KASSERT(xc->xc_ipl < __arraycount(xc_sihs));
  463         KASSERT(xc_sihs[xc->xc_ipl] != NULL);
  464 
  465         /* Executes xc__highpri_intr() via software interrupt. */
  466         softint_schedule(xc_sihs[xc->xc_ipl]);
  467 }
  468 
  469 /*
  470  * xc__highpri_intr:
  471  *
  472  *      A software interrupt handler for high priority calls.
  473  */
  474 void
  475 xc__highpri_intr(void *dummy)
  476 {
  477         xc_state_t *xc = &xc_high_pri;
  478         void *arg1, *arg2;
  479         xcfunc_t func;
  480 
  481         KASSERTMSG(!cpu_intr_p(), "high priority xcall for function %p",
  482             xc->xc_func);
  483         /*
  484          * Lock-less fetch of function and its arguments.
  485          * Safe since it cannot change at this point.
  486          */
  487         func = xc->xc_func;
  488         arg1 = xc->xc_arg1;
  489         arg2 = xc->xc_arg2;
  490 
  491         KASSERT(func != NULL);
  492         (*func)(arg1, arg2);
  493 
  494         /*
  495          * Note the request as done, and if we have reached the head,
  496          * cross-call has been processed - notify waiters, if any.
  497          */
  498         mutex_enter(&xc->xc_lock);
  499         KASSERT(xc->xc_donep < xc->xc_headp);
  500 #ifdef __HAVE_ATOMIC64_LOADSTORE
  501         atomic_store_release(&xc->xc_donep, xc->xc_donep + 1);
  502 #else
  503         xc->xc_donep++;
  504 #endif
  505         if (xc->xc_donep == xc->xc_headp) {
  506                 cv_broadcast(&xc->xc_busy);
  507         }
  508         mutex_exit(&xc->xc_lock);
  509 }
  510 
  511 /*
  512  * xc_highpri:
  513  *
  514  *      Trigger a high priority call on one or more CPUs.
  515  */
  516 static inline uint64_t
  517 xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci,
  518     unsigned int ipl)
  519 {
  520         xc_state_t *xc = &xc_high_pri;
  521         uint64_t where;
  522 
  523         mutex_enter(&xc->xc_lock);
  524         while (xc->xc_headp != xc->xc_donep) {
  525                 cv_wait(&xc->xc_busy, &xc->xc_lock);
  526         }
  527         xc->xc_func = func;
  528         xc->xc_arg1 = arg1;
  529         xc->xc_arg2 = arg2;
  530         xc->xc_headp += (ci ? 1 : ncpu);
  531         xc->xc_ipl = ipl;
  532         where = xc->xc_headp;
  533         mutex_exit(&xc->xc_lock);
  534 
  535         /*
  536          * Send the IPI once lock is released.
  537          * Note: it will handle the local CPU case.
  538          */
  539 
  540 #ifdef _RUMPKERNEL
  541         rump_xc_highpri(ci);
  542 #else
  543 #ifdef MULTIPROCESSOR
  544         kpreempt_disable();
  545         if (curcpu() == ci) {
  546                 /* Unicast: local CPU. */
  547                 xc_ipi_handler();
  548         } else if (ci) {
  549                 /* Unicast: remote CPU. */
  550                 xc_send_ipi(ci);
  551         } else {
  552                 /* Broadcast: all, including local. */
  553                 xc_send_ipi(NULL);
  554                 xc_ipi_handler();
  555         }
  556         kpreempt_enable();
  557 #else
  558         KASSERT(ci == NULL || curcpu() == ci);
  559         xc_ipi_handler();
  560 #endif
  561 #endif
  562 
  563         /* Indicate a high priority ticket. */
  564         return (where | XC_PRI_BIT);
  565 }

Cache object: f0db85774963f673e510e478d503a0a9


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.