The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/msi.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2006 Yahoo!, Inc.
    3  * All rights reserved.
    4  * Written by: John Baldwin <jhb@FreeBSD.org>
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 3. Neither the name of the author nor the names of any co-contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  */
   30 
   31 /*
   32  * Support for PCI Message Signalled Interrupts (MSI).  MSI interrupts on
   33  * x86 are basically APIC messages that the northbridge delivers directly
   34  * to the local APICs as if they had come from an I/O APIC.
   35  */
   36 
   37 #include <sys/cdefs.h>
   38 __FBSDID("$FreeBSD$");
   39 
   40 #include <sys/param.h>
   41 #include <sys/bus.h>
   42 #include <sys/kernel.h>
   43 #include <sys/lock.h>
   44 #include <sys/malloc.h>
   45 #include <sys/mutex.h>
   46 #include <sys/sx.h>
   47 #include <sys/systm.h>
   48 #include <machine/apicreg.h>
   49 #include <machine/md_var.h>
   50 #include <machine/frame.h>
   51 #include <machine/intr_machdep.h>
   52 #include <machine/apicvar.h>
   53 #include <dev/pci/pcivar.h>
   54 
   55 /* Fields in address for Intel MSI messages. */
   56 #define MSI_INTEL_ADDR_DEST             0x000ff000
   57 #define MSI_INTEL_ADDR_RH               0x00000008
   58 # define MSI_INTEL_ADDR_RH_ON           0x00000008
   59 # define MSI_INTEL_ADDR_RH_OFF          0x00000000
   60 #define MSI_INTEL_ADDR_DM               0x00000004
   61 # define MSI_INTEL_ADDR_DM_PHYSICAL     0x00000000
   62 # define MSI_INTEL_ADDR_DM_LOGICAL      0x00000004
   63 
   64 /* Fields in data for Intel MSI messages. */
   65 #define MSI_INTEL_DATA_TRGRMOD          IOART_TRGRMOD   /* Trigger mode. */
   66 # define MSI_INTEL_DATA_TRGREDG         IOART_TRGREDG
   67 # define MSI_INTEL_DATA_TRGRLVL         IOART_TRGRLVL
   68 #define MSI_INTEL_DATA_LEVEL            0x00004000      /* Polarity. */
   69 # define MSI_INTEL_DATA_DEASSERT        0x00000000
   70 # define MSI_INTEL_DATA_ASSERT          0x00004000
   71 #define MSI_INTEL_DATA_DELMOD           IOART_DELMOD    /* Delivery mode. */
   72 # define MSI_INTEL_DATA_DELFIXED        IOART_DELFIXED
   73 # define MSI_INTEL_DATA_DELLOPRI        IOART_DELLOPRI
   74 # define MSI_INTEL_DATA_DELSMI          IOART_DELSMI
   75 # define MSI_INTEL_DATA_DELNMI          IOART_DELNMI
   76 # define MSI_INTEL_DATA_DELINIT         IOART_DELINIT
   77 # define MSI_INTEL_DATA_DELEXINT        IOART_DELEXINT
   78 #define MSI_INTEL_DATA_INTVEC           IOART_INTVEC    /* Interrupt vector. */
   79 
   80 /*
   81  * Build Intel MSI message and data values from a source.  AMD64 systems
   82  * seem to be compatible, so we use the same function for both.
   83  */
   84 #define INTEL_ADDR(msi)                                                 \
   85         (MSI_INTEL_ADDR_BASE | (msi)->msi_cpu << 12 |                   \
   86             MSI_INTEL_ADDR_RH_OFF | MSI_INTEL_ADDR_DM_PHYSICAL)
   87 #define INTEL_DATA(msi)                                                 \
   88         (MSI_INTEL_DATA_TRGREDG | MSI_INTEL_DATA_DELFIXED | (msi)->msi_vector)
   89 
   90 static MALLOC_DEFINE(M_MSI, "msi", "PCI MSI");
   91 
   92 /*
   93  * MSI sources are bunched into groups.  This is because MSI forces
   94  * all of the messages to share the address and data registers and
   95  * thus certain properties (such as the local APIC ID target on x86).
   96  * Each group has a 'first' source that contains information global to
   97  * the group.  These fields are marked with (g) below.
   98  *
   99  * Note that local APIC ID is kind of special.  Each message will be
  100  * assigned an ID by the system; however, a group will use the ID from
  101  * the first message.
  102  *
  103  * For MSI-X, each message is isolated.
  104  */
  105 struct msi_intsrc {
  106         struct intsrc msi_intsrc;
  107         device_t msi_dev;               /* Owning device. (g) */
  108         struct msi_intsrc *msi_first;   /* First source in group. */
  109         u_int msi_irq;                  /* IRQ cookie. */
  110         u_int msi_msix;                 /* MSI-X message. */
  111         u_int msi_vector:8;             /* IDT vector. */
  112         u_int msi_cpu:8;                /* Local APIC ID. (g) */
  113         u_int msi_count:8;              /* Messages in this group. (g) */
  114 };
  115 
  116 static void     msi_create_source(void);
  117 static void     msi_enable_source(struct intsrc *isrc);
  118 static void     msi_disable_source(struct intsrc *isrc, int eoi);
  119 static void     msi_eoi_source(struct intsrc *isrc);
  120 static void     msi_enable_intr(struct intsrc *isrc);
  121 static void     msi_disable_intr(struct intsrc *isrc);
  122 static int      msi_vector(struct intsrc *isrc);
  123 static int      msi_source_pending(struct intsrc *isrc);
  124 static int      msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
  125                     enum intr_polarity pol);
  126 static void     msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
  127 
  128 struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
  129                        msi_enable_intr, msi_disable_intr, msi_vector,
  130                        msi_source_pending, NULL, NULL, msi_config_intr,
  131                        msi_assign_cpu };
  132 
  133 static int msi_enabled;
  134 static int msi_last_irq;
  135 static struct mtx msi_lock;
  136 
  137 static void
  138 msi_enable_source(struct intsrc *isrc)
  139 {
  140 }
  141 
  142 static void
  143 msi_disable_source(struct intsrc *isrc, int eoi)
  144 {
  145 
  146         if (eoi == PIC_EOI)
  147                 lapic_eoi();
  148 }
  149 
  150 static void
  151 msi_eoi_source(struct intsrc *isrc)
  152 {
  153 
  154         lapic_eoi();
  155 }
  156 
  157 static void
  158 msi_enable_intr(struct intsrc *isrc)
  159 {
  160         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  161 
  162         apic_enable_vector(msi->msi_vector);
  163 }
  164 
  165 static void
  166 msi_disable_intr(struct intsrc *isrc)
  167 {
  168         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  169 
  170         apic_disable_vector(msi->msi_vector);
  171 }
  172 
  173 static int
  174 msi_vector(struct intsrc *isrc)
  175 {
  176         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  177 
  178         return (msi->msi_irq);
  179 }
  180 
  181 static int
  182 msi_source_pending(struct intsrc *isrc)
  183 {
  184 
  185         return (0);
  186 }
  187 
  188 static int
  189 msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
  190     enum intr_polarity pol)
  191 {
  192 
  193         return (ENODEV);
  194 }
  195 
  196 static void
  197 msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
  198 {
  199         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  200 
  201         msi->msi_cpu = apic_id;
  202         if (bootverbose)
  203                 printf("msi: Assigning %s IRQ %d to local APIC %u\n",
  204                     msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
  205                     msi->msi_cpu);      
  206         pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
  207 }
  208 
  209 void
  210 msi_init(void)
  211 {
  212 
  213         /* Check if we have a supported CPU. */
  214         if (!(strcmp(cpu_vendor, "GenuineIntel") == 0 ||
  215               strcmp(cpu_vendor, "AuthenticAMD") == 0))
  216                 return;
  217 
  218         msi_enabled = 1;
  219         intr_register_pic(&msi_pic);
  220         mtx_init(&msi_lock, "msi", NULL, MTX_DEF);
  221 }
  222 
  223 void
  224 msi_create_source(void)
  225 {
  226         struct msi_intsrc *msi;
  227         u_int irq;
  228 
  229         mtx_lock(&msi_lock);
  230         if (msi_last_irq >= NUM_MSI_INTS) {
  231                 mtx_unlock(&msi_lock);
  232                 return;
  233         }
  234         irq = msi_last_irq + FIRST_MSI_INT;
  235         msi_last_irq++;
  236         mtx_unlock(&msi_lock);
  237 
  238         msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);      
  239         msi->msi_intsrc.is_pic = &msi_pic;
  240         msi->msi_irq = irq;
  241         intr_register_source(&msi->msi_intsrc);
  242         nexus_add_irq(irq);
  243 }
  244 
  245 /*
  246  * Try to allocate 'count' interrupt sources with contiguous IDT values.  If
  247  * we allocate any new sources, then their IRQ values will be at the end of
  248  * the irqs[] array, with *newirq being the index of the first new IRQ value
  249  * and *newcount being the number of new IRQ values added.
  250  */
  251 int
  252 msi_alloc(device_t dev, int count, int maxcount, int *irqs)
  253 {
  254         struct msi_intsrc *msi, *fsrc;
  255         int cnt, i, vector;
  256 
  257         if (!msi_enabled)
  258                 return (ENXIO);
  259 
  260 again:
  261         mtx_lock(&msi_lock);
  262 
  263         /* Try to find 'count' free IRQs. */
  264         cnt = 0;
  265         for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
  266                 msi = (struct msi_intsrc *)intr_lookup_source(i);
  267 
  268                 /* End of allocated sources, so break. */
  269                 if (msi == NULL)
  270                         break;
  271 
  272                 /* If this is a free one, save its IRQ in the array. */
  273                 if (msi->msi_dev == NULL) {
  274                         irqs[cnt] = i;
  275                         cnt++;
  276                         if (cnt == count)
  277                                 break;
  278                 }
  279         }
  280 
  281         /* Do we need to create some new sources? */
  282         if (cnt < count) {
  283                 /* If we would exceed the max, give up. */
  284                 if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) {
  285                         mtx_unlock(&msi_lock);
  286                         return (ENXIO);
  287                 }
  288                 mtx_unlock(&msi_lock);
  289 
  290                 /* We need count - cnt more sources. */
  291                 while (cnt < count) {
  292                         msi_create_source();
  293                         cnt++;
  294                 }
  295                 goto again;
  296         }
  297 
  298         /* Ok, we now have the IRQs allocated. */
  299         KASSERT(cnt == count, ("count mismatch"));
  300 
  301         /* Allocate 'count' IDT vectors. */
  302         vector = apic_alloc_vectors(irqs, count, maxcount);
  303         if (vector == 0) {
  304                 mtx_unlock(&msi_lock);
  305                 return (ENOSPC);
  306         }
  307 
  308         /* Assign IDT vectors and make these messages owned by 'dev'. */
  309         fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
  310         for (i = 0; i < count; i++) {
  311                 msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
  312                 msi->msi_dev = dev;
  313                 msi->msi_vector = vector + i;
  314                 if (bootverbose)
  315                         printf("msi: routing MSI IRQ %d to vector %u\n",
  316                             msi->msi_irq, msi->msi_vector);
  317                 msi->msi_first = fsrc;
  318                 KASSERT(msi->msi_intsrc.is_handlers == 0,
  319                     ("dead MSI has handlers"));
  320         }
  321         fsrc->msi_count = count;
  322         mtx_unlock(&msi_lock);
  323 
  324         return (0);
  325 }
  326 
  327 int
  328 msi_release(int *irqs, int count)
  329 {
  330         struct msi_intsrc *msi, *first;
  331         int i;
  332 
  333         mtx_lock(&msi_lock);
  334         first = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
  335         if (first == NULL) {
  336                 mtx_unlock(&msi_lock);
  337                 return (ENOENT);
  338         }
  339 
  340         /* Make sure this isn't an MSI-X message. */
  341         if (first->msi_msix) {
  342                 mtx_unlock(&msi_lock);
  343                 return (EINVAL);
  344         }
  345 
  346         /* Make sure this message is allocated to a group. */
  347         if (first->msi_first == NULL) {
  348                 mtx_unlock(&msi_lock);
  349                 return (ENXIO);
  350         }
  351 
  352         /*
  353          * Make sure this is the start of a group and that we are releasing
  354          * the entire group.
  355          */
  356         if (first->msi_first != first || first->msi_count != count) {
  357                 mtx_unlock(&msi_lock);
  358                 return (EINVAL);
  359         }
  360         KASSERT(first->msi_dev != NULL, ("unowned group"));
  361 
  362         /* Clear all the extra messages in the group. */
  363         for (i = 1; i < count; i++) {
  364                 msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
  365                 KASSERT(msi->msi_first == first, ("message not in group"));
  366                 KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
  367                 msi->msi_first = NULL;
  368                 msi->msi_dev = NULL;
  369                 apic_free_vector(msi->msi_vector, msi->msi_irq);
  370                 msi->msi_vector = 0;
  371         }
  372 
  373         /* Clear out the first message. */
  374         first->msi_first = NULL;
  375         first->msi_dev = NULL;
  376         apic_free_vector(first->msi_vector, first->msi_irq);
  377         first->msi_vector = 0;
  378         first->msi_count = 0;
  379 
  380         mtx_unlock(&msi_lock);
  381         return (0);
  382 }
  383 
  384 int
  385 msi_map(int irq, uint64_t *addr, uint32_t *data)
  386 {
  387         struct msi_intsrc *msi;
  388 
  389         mtx_lock(&msi_lock);
  390         msi = (struct msi_intsrc *)intr_lookup_source(irq);
  391         if (msi == NULL) {
  392                 mtx_unlock(&msi_lock);
  393                 return (ENOENT);
  394         }
  395 
  396         /* Make sure this message is allocated to a device. */
  397         if (msi->msi_dev == NULL) {
  398                 mtx_unlock(&msi_lock);
  399                 return (ENXIO);
  400         }
  401 
  402         /*
  403          * If this message isn't an MSI-X message, make sure it's part
  404          * of a group, and switch to the first message in the
  405          * group.
  406          */
  407         if (!msi->msi_msix) {
  408                 if (msi->msi_first == NULL) {
  409                         mtx_unlock(&msi_lock);
  410                         return (ENXIO);
  411                 }
  412                 msi = msi->msi_first;
  413         }
  414 
  415         *addr = INTEL_ADDR(msi);
  416         *data = INTEL_DATA(msi);
  417         mtx_unlock(&msi_lock);
  418         return (0);
  419 }
  420 
  421 int
  422 msix_alloc(device_t dev, int *irq)
  423 {
  424         struct msi_intsrc *msi;
  425         int i, vector;
  426 
  427         if (!msi_enabled)
  428                 return (ENXIO);
  429 
  430 again:
  431         mtx_lock(&msi_lock);
  432 
  433         /* Find a free IRQ. */
  434         for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
  435                 msi = (struct msi_intsrc *)intr_lookup_source(i);
  436 
  437                 /* End of allocated sources, so break. */
  438                 if (msi == NULL)
  439                         break;
  440 
  441                 /* Stop at the first free source. */
  442                 if (msi->msi_dev == NULL)
  443                         break;
  444         }
  445 
  446         /* Do we need to create a new source? */
  447         if (msi == NULL) {
  448                 /* If we would exceed the max, give up. */
  449                 if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) {
  450                         mtx_unlock(&msi_lock);
  451                         return (ENXIO);
  452                 }
  453                 mtx_unlock(&msi_lock);
  454 
  455                 /* Create a new source. */
  456                 msi_create_source();
  457                 goto again;
  458         }
  459 
  460         /* Allocate an IDT vector. */
  461         vector = apic_alloc_vector(i);
  462         if (bootverbose)
  463                 printf("msi: routing MSI-X IRQ %d to vector %u\n", msi->msi_irq,
  464                     vector);
  465 
  466         /* Setup source. */
  467         msi->msi_dev = dev;
  468         msi->msi_vector = vector;
  469         msi->msi_msix = 1;
  470 
  471         KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
  472         mtx_unlock(&msi_lock);
  473 
  474         *irq = i;
  475         return (0);
  476 }
  477 
  478 int
  479 msix_release(int irq)
  480 {
  481         struct msi_intsrc *msi;
  482 
  483         mtx_lock(&msi_lock);
  484         msi = (struct msi_intsrc *)intr_lookup_source(irq);
  485         if (msi == NULL) {
  486                 mtx_unlock(&msi_lock);
  487                 return (ENOENT);
  488         }
  489 
  490         /* Make sure this is an MSI-X message. */
  491         if (!msi->msi_msix) {
  492                 mtx_unlock(&msi_lock);
  493                 return (EINVAL);
  494         }
  495 
  496         KASSERT(msi->msi_dev != NULL, ("unowned message"));
  497 
  498         /* Clear out the message. */
  499         msi->msi_dev = NULL;
  500         apic_free_vector(msi->msi_vector, msi->msi_irq);
  501         msi->msi_vector = 0;
  502         msi->msi_msix = 0;
  503 
  504         mtx_unlock(&msi_lock);
  505         return (0);
  506 }

Cache object: 8b30e1af9499004f04c4edf3af3e332b


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.