subr_interrupt.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*      $NetBSD: subr_interrupt.c,v 1.5 2021/12/10 20:36:04 andvar Exp $        */
    2 
    3 /*
    4  * Copyright (c) 2015 Internet Initiative Japan Inc.
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
   17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
   18  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   19  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
   20  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   26  * POSSIBILITY OF SUCH DAMAGE.
   27  */
   28 
   29 #include <sys/cdefs.h>
   30 __KERNEL_RCSID(0, "$NetBSD: subr_interrupt.c,v 1.5 2021/12/10 20:36:04 andvar Exp $");
   31 
   32 #include <sys/param.h>
   33 #include <sys/systm.h>
   34 #include <sys/kernel.h>
   35 #include <sys/errno.h>
   36 #include <sys/cpu.h>
   37 #include <sys/interrupt.h>
   38 #include <sys/intr.h>
   39 #include <sys/kcpuset.h>
   40 #include <sys/kmem.h>
   41 #include <sys/proc.h>
   42 #include <sys/xcall.h>
   43 #include <sys/sysctl.h>
   44 
   45 #include <sys/conf.h>
   46 #include <sys/intrio.h>
   47 #include <sys/kauth.h>
   48 
   49 #include <machine/limits.h>
   50 
   51 #ifdef INTR_DEBUG
   52 #define DPRINTF(msg) printf msg
   53 #else
   54 #define DPRINTF(msg)
   55 #endif
   56 
   57 static struct intrio_set kintrio_set = { "\0", NULL, 0 };
   58 
   59 #define UNSET_NOINTR_SHIELD     0
   60 #define SET_NOINTR_SHIELD       1
   61 
   62 static void
   63 interrupt_shield_xcall(void *arg1, void *arg2)
   64 {
   65         struct cpu_info *ci;
   66         struct schedstate_percpu *spc;
   67         int s, shield;
   68 
   69         ci = arg1;
   70         shield = (int)(intptr_t)arg2;
   71         spc = &ci->ci_schedstate;
   72 
   73         s = splsched();
   74         if (shield == UNSET_NOINTR_SHIELD)
   75                 spc->spc_flags &= ~SPCF_NOINTR;
   76         else if (shield == SET_NOINTR_SHIELD)
   77                 spc->spc_flags |= SPCF_NOINTR;
   78         splx(s);
   79 }
   80 
   81 /*
   82  * Change SPCF_NOINTR flag of schedstate_percpu->spc_flags.
   83  */
   84 static int
   85 interrupt_shield(u_int cpu_idx, int shield)
   86 {
   87         struct cpu_info *ci;
   88         struct schedstate_percpu *spc;
   89 
   90         KASSERT(mutex_owned(&cpu_lock));
   91 
   92         ci = cpu_lookup(cpu_idx);
   93         if (ci == NULL)
   94                 return EINVAL;
   95 
   96         spc = &ci->ci_schedstate;
   97         if (shield == UNSET_NOINTR_SHIELD) {
   98                 if ((spc->spc_flags & SPCF_NOINTR) == 0)
   99                         return 0;
  100         } else if (shield == SET_NOINTR_SHIELD) {
  101                 if ((spc->spc_flags & SPCF_NOINTR) != 0)
  102                         return 0;
  103         }
  104 
  105         if (ci == curcpu() || !mp_online) {
  106                 interrupt_shield_xcall(ci, (void *)(intptr_t)shield);
  107         } else {
  108                 uint64_t where;
  109                 where = xc_unicast(0, interrupt_shield_xcall, ci,
  110                         (void *)(intptr_t)shield, ci);
  111                 xc_wait(where);
  112         }
  113 
  114         spc->spc_lastmod = time_second;
  115         return 0;
  116 }
  117 
  118 /*
  119  * Move all assigned interrupts from "cpu_idx" to the other cpu as possible.
  120  * The destination cpu is the lowest cpuid of available cpus.
  121  * If there are no available cpus, give up to move interrupts.
  122  */
  123 static int
  124 interrupt_avert_intr(u_int cpu_idx)
  125 {
  126         kcpuset_t *cpuset;
  127         struct intrids_handler *ii_handler;
  128         intrid_t *ids;
  129         int error = 0, i, nids;
  130 
  131         kcpuset_create(&cpuset, true);
  132         kcpuset_set(cpuset, cpu_idx);
  133 
  134         ii_handler = interrupt_construct_intrids(cpuset);
  135         if (ii_handler == NULL) {
  136                 error = EINVAL;
  137                 goto out;
  138         }
  139         nids = ii_handler->iih_nids;
  140         if (nids == 0) {
  141                 error = 0;
  142                 goto destruct_out;
  143         }
  144 
  145         interrupt_get_available(cpuset);
  146         kcpuset_clear(cpuset, cpu_idx);
  147         if (kcpuset_iszero(cpuset)) {
  148                 DPRINTF(("%s: no available cpu\n", __func__));
  149                 error = ENOENT;
  150                 goto destruct_out;
  151         }
  152 
  153         ids = ii_handler->iih_intrids;
  154         for (i = 0; i < nids; i++) {
  155                 error = interrupt_distribute_handler(ids[i], cpuset, NULL);
  156                 if (error)
  157                         break;
  158         }
  159 
  160  destruct_out:
  161         interrupt_destruct_intrids(ii_handler);
  162  out:
  163         kcpuset_destroy(cpuset);
  164         return error;
  165 }
  166 
  167 /*
  168  * Return actual intrio_list_line size.
  169  * intrio_list_line size is variable by ncpu.
  170  */
  171 static size_t
  172 interrupt_intrio_list_line_size(void)
  173 {
  174 
  175         return sizeof(struct intrio_list_line) +
  176                 sizeof(struct intrio_list_line_cpu) * (ncpu - 1);
  177 }
  178 
  179 /*
  180  * Return the size of interrupts list data on success.
  181  * Reterun 0 on failed.
  182  */
  183 static int
  184 interrupt_intrio_list_size(size_t *ilsize)
  185 {
  186         struct intrids_handler *ii_handler;
  187 
  188         *ilsize = 0;
  189 
  190         /* buffer header */
  191         *ilsize += sizeof(struct intrio_list);
  192 
  193         /* il_line body */
  194         ii_handler = interrupt_construct_intrids(kcpuset_running);
  195         if (ii_handler == NULL)
  196                 return EOPNOTSUPP;
  197         *ilsize += interrupt_intrio_list_line_size() * ii_handler->iih_nids;
  198 
  199         interrupt_destruct_intrids(ii_handler);
  200         return 0;
  201 }
  202 
  203 /*
  204  * Set intrctl list data to "il", and return list structure bytes.
  205  * If error occurred, return <0.
  206  * If "data" == NULL, simply return list structure bytes.
  207  */
  208 static int
  209 interrupt_intrio_list(struct intrio_list *il, size_t ilsize)
  210 {
  211         struct intrio_list_line *illine;
  212         kcpuset_t *assigned, *avail;
  213         struct intrids_handler *ii_handler;
  214         intrid_t *ids;
  215         u_int cpu_idx;
  216         int nids, intr_idx, error, line_size;
  217 
  218         illine = (struct intrio_list_line *)
  219             ((char *)il + sizeof(struct intrio_list));
  220         il->il_lineoffset = (off_t)((uintptr_t)illine - (uintptr_t)il);
  221 
  222         kcpuset_create(&avail, true);
  223         interrupt_get_available(avail);
  224         kcpuset_create(&assigned, true);
  225 
  226         ii_handler = interrupt_construct_intrids(kcpuset_running);
  227         if (ii_handler == NULL) {
  228                 DPRINTF(("%s: interrupt_construct_intrids() failed\n",
  229                     __func__));
  230                 error = EOPNOTSUPP;
  231                 goto out;
  232         }
  233 
  234         line_size = interrupt_intrio_list_line_size();
  235         /* ensure interrupts are not added after interrupt_intrio_list_size() */
  236         nids = ii_handler->iih_nids;
  237         ids = ii_handler->iih_intrids;
  238         if (ilsize < sizeof(struct intrio_list) + line_size * nids) {
  239                 DPRINTF(("%s: interrupts are added during execution.\n",
  240                     __func__));
  241                 error = EAGAIN;
  242                 goto destruct_out;
  243         }
  244 
  245         for (intr_idx = 0; intr_idx < nids; intr_idx++) {
  246                 char devname[INTRDEVNAMEBUF];
  247 
  248                 strncpy(illine->ill_intrid, ids[intr_idx], INTRIDBUF);
  249                 interrupt_get_devname(ids[intr_idx], devname, sizeof(devname));
  250                 strncpy(illine->ill_xname, devname, INTRDEVNAMEBUF);
  251 
  252                 interrupt_get_assigned(ids[intr_idx], assigned);
  253                 for (cpu_idx = 0; cpu_idx < ncpu; cpu_idx++) {
  254                         struct intrio_list_line_cpu *illcpu =
  255                             &illine->ill_cpu[cpu_idx];
  256 
  257                         illcpu->illc_assigned =
  258                             kcpuset_isset(assigned, cpu_idx);
  259                         illcpu->illc_count =
  260                             interrupt_get_count(ids[intr_idx], cpu_idx);
  261                 }
  262 
  263                 illine = (struct intrio_list_line *)
  264                     ((char *)illine + line_size);
  265         }
  266 
  267         error = 0;
  268         il->il_version = INTRIO_LIST_VERSION;
  269         il->il_ncpus = ncpu;
  270         il->il_nintrs = nids;
  271         il->il_linesize = line_size;
  272         il->il_bufsize = ilsize;
  273 
  274  destruct_out:
  275         interrupt_destruct_intrids(ii_handler);
  276  out:
  277         kcpuset_destroy(assigned);
  278         kcpuset_destroy(avail);
  279 
  280         return error;
  281 }
  282 
  283 /*
  284  * "intrctl list" entry
  285  */
  286 static int
  287 interrupt_intrio_list_sysctl(SYSCTLFN_ARGS)
  288 {
  289         int error;
  290         void *buf;
  291         size_t ilsize;
  292 
  293         if (oldlenp == NULL)
  294                 return EINVAL;
  295 
  296         if ((error = interrupt_intrio_list_size(&ilsize)) != 0)
  297                 return error;
  298 
  299         /*
  300          * If oldp == NULL, the sysctl(8) caller process want to get the size of
  301          * intrctl list data only.
  302          */
  303         if (oldp == NULL) {
  304                 *oldlenp = ilsize;
  305                 return 0;
  306         }
  307 
  308         /*
  309          * If oldp != NULL, the sysctl(8) caller process want to get both the
  310          * size and the contents of intrctl list data.
  311          */
  312         if (*oldlenp < ilsize)
  313                 return ENOMEM;
  314 
  315         buf = kmem_zalloc(ilsize, KM_SLEEP);
  316         if ((error = interrupt_intrio_list(buf, ilsize)) != 0)
  317                 goto out;
  318 
  319         error = copyout(buf, oldp, ilsize);
  320  out:
  321         kmem_free(buf, ilsize);
  322         return error;
  323 }
  324 
  325 /*
  326  * "intrctl affinity" entry
  327  */
  328 static int
  329 interrupt_set_affinity_sysctl(SYSCTLFN_ARGS)
  330 {
  331         struct sysctlnode node;
  332         struct intrio_set *iset;
  333         cpuset_t *ucpuset;
  334         kcpuset_t *kcpuset;
  335         int error;
  336 
  337         error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_INTR,
  338             KAUTH_REQ_SYSTEM_INTR_AFFINITY, NULL, NULL, NULL);
  339         if (error)
  340                 return EPERM;
  341 
  342         node = *rnode;
  343         iset = (struct intrio_set *)node.sysctl_data;
  344 
  345         error = sysctl_lookup(SYSCTLFN_CALL(&node));
  346         if (error != 0 || newp == NULL)
  347                 return error;
  348 
  349         ucpuset = iset->cpuset;
  350         kcpuset_create(&kcpuset, true);
  351         error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
  352         if (error)
  353                 goto out;
  354         if (kcpuset_iszero(kcpuset)) {
  355                 error = EINVAL;
  356                 goto out;
  357         }
  358 
  359         error = interrupt_distribute_handler(iset->intrid, kcpuset, NULL);
  360 
  361  out:
  362         kcpuset_destroy(kcpuset);
  363         return error;
  364 }
  365 
  366 /*
  367  * "intrctl intr" entry
  368  */
  369 static int
  370 interrupt_intr_sysctl(SYSCTLFN_ARGS)
  371 {
  372         struct sysctlnode node;
  373         struct intrio_set *iset;
  374         cpuset_t *ucpuset;
  375         kcpuset_t *kcpuset;
  376         int error;
  377         u_int cpu_idx;
  378 
  379         error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU,
  380             KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL);
  381         if (error)
  382                 return EPERM;
  383 
  384         node = *rnode;
  385         iset = (struct intrio_set *)node.sysctl_data;
  386 
  387         error = sysctl_lookup(SYSCTLFN_CALL(&node));
  388         if (error != 0 || newp == NULL)
  389                 return error;
  390 
  391         ucpuset = iset->cpuset;
  392         kcpuset_create(&kcpuset, true);
  393         error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
  394         if (error)
  395                 goto out;
  396         if (kcpuset_iszero(kcpuset)) {
  397                 error = EINVAL;
  398                 goto out;
  399         }
  400 
  401         cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */
  402 
  403         mutex_enter(&cpu_lock);
  404         error = interrupt_shield(cpu_idx, UNSET_NOINTR_SHIELD);
  405         mutex_exit(&cpu_lock);
  406 
  407  out:
  408         kcpuset_destroy(kcpuset);
  409         return error;
  410 }
  411 
  412 /*
  413  * "intrctl nointr" entry
  414  */
  415 static int
  416 interrupt_nointr_sysctl(SYSCTLFN_ARGS)
  417 {
  418         struct sysctlnode node;
  419         struct intrio_set *iset;
  420         cpuset_t *ucpuset;
  421         kcpuset_t *kcpuset;
  422         int error;
  423         u_int cpu_idx;
  424 
  425         error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU,
  426             KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL);
  427         if (error)
  428                 return EPERM;
  429 
  430         node = *rnode;
  431         iset = (struct intrio_set *)node.sysctl_data;
  432 
  433         error = sysctl_lookup(SYSCTLFN_CALL(&node));
  434         if (error != 0 || newp == NULL)
  435                 return error;
  436 
  437         ucpuset = iset->cpuset;
  438         kcpuset_create(&kcpuset, true);
  439         error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
  440         if (error)
  441                 goto out;
  442         if (kcpuset_iszero(kcpuset)) {
  443                 error = EINVAL;
  444                 goto out;
  445         }
  446 
  447         cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */
  448 
  449         mutex_enter(&cpu_lock);
  450         error = interrupt_shield(cpu_idx, SET_NOINTR_SHIELD);
  451         mutex_exit(&cpu_lock);
  452         if (error)
  453                 goto out;
  454 
  455         error = interrupt_avert_intr(cpu_idx);
  456 
  457  out:
  458         kcpuset_destroy(kcpuset);
  459         return error;
  460 }
  461 
  462 SYSCTL_SETUP(sysctl_interrupt_setup, "sysctl interrupt setup")
  463 {
  464         const struct sysctlnode *node = NULL;
  465 
  466         sysctl_createv(clog, 0, NULL, &node,
  467                        CTLFLAG_PERMANENT, CTLTYPE_NODE,
  468                        "intr", SYSCTL_DESCR("Interrupt options"),
  469                        NULL, 0, NULL, 0,
  470                        CTL_KERN, CTL_CREATE, CTL_EOL);
  471 
  472         sysctl_createv(clog, 0, &node, NULL,
  473                        CTLFLAG_PERMANENT, CTLTYPE_STRUCT,
  474                        "list", SYSCTL_DESCR("intrctl list"),
  475                        interrupt_intrio_list_sysctl, 0, NULL,
  476                         0, CTL_CREATE, CTL_EOL);
  477 
  478         sysctl_createv(clog, 0, &node, NULL,
  479                        CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
  480                        "affinity", SYSCTL_DESCR("set affinity"),
  481                        interrupt_set_affinity_sysctl, 0, &kintrio_set,
  482                        sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
  483 
  484         sysctl_createv(clog, 0, &node, NULL,
  485                        CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
  486                        "intr", SYSCTL_DESCR("set intr"),
  487                        interrupt_intr_sysctl, 0, &kintrio_set,
  488                        sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
  489 
  490         sysctl_createv(clog, 0, &node, NULL,
  491                        CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
  492                        "nointr", SYSCTL_DESCR("set nointr"),
  493                        interrupt_nointr_sysctl, 0, &kintrio_set,
  494                        sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
  495 }
Cache object: d543d537837ff06d54ebf968ac44399c
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/kern/subr_interrupt.c

FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_interrupt.c
