The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/msi.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2006 Yahoo!, Inc.
    3  * All rights reserved.
    4  * Written by: John Baldwin <jhb@FreeBSD.org>
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 3. Neither the name of the author nor the names of any co-contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  */
   30 
   31 /*
   32  * Support for PCI Message Signalled Interrupts (MSI).  MSI interrupts on
   33  * x86 are basically APIC messages that the northbridge delivers directly
   34  * to the local APICs as if they had come from an I/O APIC.
   35  */
   36 
   37 #include <sys/cdefs.h>
   38 __FBSDID("$FreeBSD: releng/6.4/sys/amd64/amd64/msi.c 173199 2007-10-30 18:01:44Z jhb $");
   39 
   40 #include <sys/param.h>
   41 #include <sys/bus.h>
   42 #include <sys/kernel.h>
   43 #include <sys/lock.h>
   44 #include <sys/malloc.h>
   45 #include <sys/mutex.h>
   46 #include <sys/sx.h>
   47 #include <sys/systm.h>
   48 #include <machine/apicreg.h>
   49 #include <machine/md_var.h>
   50 #include <machine/frame.h>
   51 #include <machine/intr_machdep.h>
   52 #include <machine/apicvar.h>
   53 #include <dev/pci/pcivar.h>
   54 
   55 /* Fields in address for Intel MSI messages. */
   56 #define MSI_INTEL_ADDR_DEST             0x000ff000
   57 #define MSI_INTEL_ADDR_RH               0x00000008
   58 # define MSI_INTEL_ADDR_RH_ON           0x00000008
   59 # define MSI_INTEL_ADDR_RH_OFF          0x00000000
   60 #define MSI_INTEL_ADDR_DM               0x00000004
   61 # define MSI_INTEL_ADDR_DM_PHYSICAL     0x00000000
   62 # define MSI_INTEL_ADDR_DM_LOGICAL      0x00000004
   63 
   64 /* Fields in data for Intel MSI messages. */
   65 #define MSI_INTEL_DATA_TRGRMOD          IOART_TRGRMOD   /* Trigger mode. */
   66 # define MSI_INTEL_DATA_TRGREDG         IOART_TRGREDG
   67 # define MSI_INTEL_DATA_TRGRLVL         IOART_TRGRLVL
   68 #define MSI_INTEL_DATA_LEVEL            0x00004000      /* Polarity. */
   69 # define MSI_INTEL_DATA_DEASSERT        0x00000000
   70 # define MSI_INTEL_DATA_ASSERT          0x00004000
   71 #define MSI_INTEL_DATA_DELMOD           IOART_DELMOD    /* Delivery mode. */
   72 # define MSI_INTEL_DATA_DELFIXED        IOART_DELFIXED
   73 # define MSI_INTEL_DATA_DELLOPRI        IOART_DELLOPRI
   74 # define MSI_INTEL_DATA_DELSMI          IOART_DELSMI
   75 # define MSI_INTEL_DATA_DELNMI          IOART_DELNMI
   76 # define MSI_INTEL_DATA_DELINIT         IOART_DELINIT
   77 # define MSI_INTEL_DATA_DELEXINT        IOART_DELEXINT
   78 #define MSI_INTEL_DATA_INTVEC           IOART_INTVEC    /* Interrupt vector. */
   79 
   80 /*
   81  * Build Intel MSI message and data values from a source.  AMD64 systems
   82  * seem to be compatible, so we use the same function for both.
   83  */
   84 #define INTEL_ADDR(msi)                                                 \
   85         (MSI_INTEL_ADDR_BASE | (msi)->msi_cpu << 12 |                   \
   86             MSI_INTEL_ADDR_RH_OFF | MSI_INTEL_ADDR_DM_PHYSICAL)
   87 #define INTEL_DATA(msi)                                                 \
   88         (MSI_INTEL_DATA_TRGREDG | MSI_INTEL_DATA_DELFIXED | (msi)->msi_vector)
   89 
   90 static MALLOC_DEFINE(M_MSI, "msi", "PCI MSI");
   91 
   92 /*
   93  * MSI sources are bunched into groups.  This is because MSI forces
   94  * all of the messages to share the address and data registers and
   95  * thus certain properties (such as the local APIC ID target on x86).
   96  * Each group has a 'first' source that contains information global to
   97  * the group.  These fields are marked with (g) below.
   98  *
   99  * Note that local APIC ID is kind of special.  Each message will be
  100  * assigned an ID by the system; however, a group will use the ID from
  101  * the first message.
  102  *
  103  * For MSI-X, each message is isolated.
  104  */
  105 struct msi_intsrc {
  106         struct intsrc msi_intsrc;
  107         device_t msi_dev;               /* Owning device. (g) */
  108         struct msi_intsrc *msi_first;   /* First source in group. */
  109         u_int msi_irq;                  /* IRQ cookie. */
  110         u_int msi_msix;                 /* MSI-X message. */
  111         u_int msi_vector:8;             /* IDT vector. */
  112         u_int msi_cpu:8;                /* Local APIC ID. (g) */
  113         u_int msi_count:8;              /* Messages in this group. (g) */
  114 };
  115 
  116 static void     msi_create_source(void);
  117 static void     msi_enable_source(struct intsrc *isrc);
  118 static void     msi_disable_source(struct intsrc *isrc, int eoi);
  119 static void     msi_eoi_source(struct intsrc *isrc);
  120 static void     msi_enable_intr(struct intsrc *isrc);
  121 static int      msi_vector(struct intsrc *isrc);
  122 static int      msi_source_pending(struct intsrc *isrc);
  123 static int      msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
  124                     enum intr_polarity pol);
  125 static void     msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
  126 
  127 struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
  128                        msi_enable_intr, msi_vector, msi_source_pending,
  129                        NULL, NULL, msi_config_intr, msi_assign_cpu };
  130 
  131 static int msi_enabled;
  132 static int msi_last_irq;
  133 static struct mtx msi_lock;
  134 
  135 static void
  136 msi_enable_source(struct intsrc *isrc)
  137 {
  138 }
  139 
  140 static void
  141 msi_disable_source(struct intsrc *isrc, int eoi)
  142 {
  143 
  144         if (eoi == PIC_EOI)
  145                 lapic_eoi();
  146 }
  147 
  148 static void
  149 msi_eoi_source(struct intsrc *isrc)
  150 {
  151 
  152         lapic_eoi();
  153 }
  154 
  155 static void
  156 msi_enable_intr(struct intsrc *isrc)
  157 {
  158         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  159 
  160         apic_enable_vector(msi->msi_vector);
  161 }
  162 
  163 static int
  164 msi_vector(struct intsrc *isrc)
  165 {
  166         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  167 
  168         return (msi->msi_irq);
  169 }
  170 
  171 static int
  172 msi_source_pending(struct intsrc *isrc)
  173 {
  174 
  175         return (0);
  176 }
  177 
  178 static int
  179 msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
  180     enum intr_polarity pol)
  181 {
  182 
  183         return (ENODEV);
  184 }
  185 
  186 static void
  187 msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
  188 {
  189         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  190 
  191         msi->msi_cpu = apic_id;
  192         if (bootverbose)
  193                 printf("msi: Assigning %s IRQ %d to local APIC %u\n",
  194                     msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
  195                     msi->msi_cpu);      
  196         if (isrc->is_enabled)
  197                 pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
  198 }
  199 
  200 void
  201 msi_init(void)
  202 {
  203 
  204         /* Check if we have a supported CPU. */
  205         if (!(strcmp(cpu_vendor, "GenuineIntel") == 0 ||
  206               strcmp(cpu_vendor, "AuthenticAMD") == 0))
  207                 return;
  208 
  209         msi_enabled = 1;
  210         intr_register_pic(&msi_pic);
  211         mtx_init(&msi_lock, "msi", NULL, MTX_DEF);
  212 }
  213 
  214 void
  215 msi_create_source(void)
  216 {
  217         struct msi_intsrc *msi;
  218         u_int irq;
  219 
  220         mtx_lock(&msi_lock);
  221         if (msi_last_irq >= NUM_MSI_INTS) {
  222                 mtx_unlock(&msi_lock);
  223                 return;
  224         }
  225         irq = msi_last_irq + FIRST_MSI_INT;
  226         msi_last_irq++;
  227         mtx_unlock(&msi_lock);
  228 
  229         msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);      
  230         msi->msi_intsrc.is_pic = &msi_pic;
  231         msi->msi_irq = irq;
  232         intr_register_source(&msi->msi_intsrc);
  233         nexus_add_irq(irq);
  234 }
  235 
  236 /*
  237  * Try to allocate 'count' interrupt sources with contiguous IDT values.  If
  238  * we allocate any new sources, then their IRQ values will be at the end of
  239  * the irqs[] array, with *newirq being the index of the first new IRQ value
  240  * and *newcount being the number of new IRQ values added.
  241  */
  242 int
  243 msi_alloc(device_t dev, int count, int maxcount, int *irqs)
  244 {
  245         struct msi_intsrc *msi, *fsrc;
  246         int cnt, i, vector;
  247 
  248         if (!msi_enabled)
  249                 return (ENXIO);
  250 
  251 again:
  252         mtx_lock(&msi_lock);
  253 
  254         /* Try to find 'count' free IRQs. */
  255         cnt = 0;
  256         for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
  257                 msi = (struct msi_intsrc *)intr_lookup_source(i);
  258 
  259                 /* End of allocated sources, so break. */
  260                 if (msi == NULL)
  261                         break;
  262 
  263                 /* If this is a free one, save its IRQ in the array. */
  264                 if (msi->msi_dev == NULL) {
  265                         irqs[cnt] = i;
  266                         cnt++;
  267                         if (cnt == count)
  268                                 break;
  269                 }
  270         }
  271 
  272         /* Do we need to create some new sources? */
  273         if (cnt < count) {
  274                 /* If we would exceed the max, give up. */
  275                 if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) {
  276                         mtx_unlock(&msi_lock);
  277                         return (ENXIO);
  278                 }
  279                 mtx_unlock(&msi_lock);
  280 
  281                 /* We need count - cnt more sources. */
  282                 while (cnt < count) {
  283                         msi_create_source();
  284                         cnt++;
  285                 }
  286                 goto again;
  287         }
  288 
  289         /* Ok, we now have the IRQs allocated. */
  290         KASSERT(cnt == count, ("count mismatch"));
  291 
  292         /* Allocate 'count' IDT vectors. */
  293         vector = apic_alloc_vectors(irqs, count, maxcount);
  294         if (vector == 0) {
  295                 mtx_unlock(&msi_lock);
  296                 return (ENOSPC);
  297         }
  298 
  299         /* Assign IDT vectors and make these messages owned by 'dev'. */
  300         fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
  301         for (i = 0; i < count; i++) {
  302                 msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
  303                 msi->msi_dev = dev;
  304                 msi->msi_vector = vector + i;
  305                 if (bootverbose)
  306                         printf("msi: routing MSI IRQ %d to vector %u\n",
  307                             msi->msi_irq, msi->msi_vector);
  308                 msi->msi_first = fsrc;
  309 
  310                 /* XXX: Somewhat gross. */
  311                 msi->msi_intsrc.is_enabled = 0;
  312         }
  313         fsrc->msi_count = count;
  314         mtx_unlock(&msi_lock);
  315 
  316         return (0);
  317 }
  318 
  319 int
  320 msi_release(int *irqs, int count)
  321 {
  322         struct msi_intsrc *msi, *first;
  323         int i;
  324 
  325         mtx_lock(&msi_lock);
  326         first = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
  327         if (first == NULL) {
  328                 mtx_unlock(&msi_lock);
  329                 return (ENOENT);
  330         }
  331 
  332         /* Make sure this isn't an MSI-X message. */
  333         if (first->msi_msix) {
  334                 mtx_unlock(&msi_lock);
  335                 return (EINVAL);
  336         }
  337 
  338         /* Make sure this message is allocated to a group. */
  339         if (first->msi_first == NULL) {
  340                 mtx_unlock(&msi_lock);
  341                 return (ENXIO);
  342         }
  343 
  344         /*
  345          * Make sure this is the start of a group and that we are releasing
  346          * the entire group.
  347          */
  348         if (first->msi_first != first || first->msi_count != count) {
  349                 mtx_unlock(&msi_lock);
  350                 return (EINVAL);
  351         }
  352         KASSERT(first->msi_dev != NULL, ("unowned group"));
  353 
  354         /* Clear all the extra messages in the group. */
  355         for (i = 1; i < count; i++) {
  356                 msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
  357                 KASSERT(msi->msi_first == first, ("message not in group"));
  358                 KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
  359                 msi->msi_first = NULL;
  360                 msi->msi_dev = NULL;
  361                 apic_free_vector(msi->msi_vector, msi->msi_irq);
  362                 msi->msi_vector = 0;
  363         }
  364 
  365         /* Clear out the first message. */
  366         first->msi_first = NULL;
  367         first->msi_dev = NULL;
  368         apic_free_vector(first->msi_vector, first->msi_irq);
  369         first->msi_vector = 0;
  370         first->msi_count = 0;
  371 
  372         mtx_unlock(&msi_lock);
  373         return (0);
  374 }
  375 
  376 int
  377 msi_map(int irq, uint64_t *addr, uint32_t *data)
  378 {
  379         struct msi_intsrc *msi;
  380 
  381         mtx_lock(&msi_lock);
  382         msi = (struct msi_intsrc *)intr_lookup_source(irq);
  383         if (msi == NULL) {
  384                 mtx_unlock(&msi_lock);
  385                 return (ENOENT);
  386         }
  387 
  388         /* Make sure this message is allocated to a device. */
  389         if (msi->msi_dev == NULL) {
  390                 mtx_unlock(&msi_lock);
  391                 return (ENXIO);
  392         }
  393 
  394         /*
  395          * If this message isn't an MSI-X message, make sure it's part
  396          * of a group, and switch to the first message in the
  397          * group.
  398          */
  399         if (!msi->msi_msix) {
  400                 if (msi->msi_first == NULL) {
  401                         mtx_unlock(&msi_lock);
  402                         return (ENXIO);
  403                 }
  404                 msi = msi->msi_first;
  405         }
  406 
  407         *addr = INTEL_ADDR(msi);
  408         *data = INTEL_DATA(msi);
  409         mtx_unlock(&msi_lock);
  410         return (0);
  411 }
  412 
  413 int
  414 msix_alloc(device_t dev, int *irq)
  415 {
  416         struct msi_intsrc *msi;
  417         int i, vector;
  418 
  419         if (!msi_enabled)
  420                 return (ENXIO);
  421 
  422 again:
  423         mtx_lock(&msi_lock);
  424 
  425         /* Find a free IRQ. */
  426         for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
  427                 msi = (struct msi_intsrc *)intr_lookup_source(i);
  428 
  429                 /* End of allocated sources, so break. */
  430                 if (msi == NULL)
  431                         break;
  432 
  433                 /* Stop at the first free source. */
  434                 if (msi->msi_dev == NULL)
  435                         break;
  436         }
  437 
  438         /* Do we need to create a new source? */
  439         if (msi == NULL) {
  440                 /* If we would exceed the max, give up. */
  441                 if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) {
  442                         mtx_unlock(&msi_lock);
  443                         return (ENXIO);
  444                 }
  445                 mtx_unlock(&msi_lock);
  446 
  447                 /* Create a new source. */
  448                 msi_create_source();
  449                 goto again;
  450         }
  451 
  452         /* Allocate an IDT vector. */
  453         vector = apic_alloc_vector(i);
  454         if (bootverbose)
  455                 printf("msi: routing MSI-X IRQ %d to vector %u\n", msi->msi_irq,
  456                     vector);
  457 
  458         /* Setup source. */
  459         msi->msi_dev = dev;
  460         msi->msi_vector = vector;
  461         msi->msi_msix = 1;
  462 
  463         /* XXX: Somewhat gross. */
  464         msi->msi_intsrc.is_enabled = 0;
  465         mtx_unlock(&msi_lock);
  466 
  467         *irq = i;
  468         return (0);
  469 }
  470 
  471 int
  472 msix_release(int irq)
  473 {
  474         struct msi_intsrc *msi;
  475 
  476         mtx_lock(&msi_lock);
  477         msi = (struct msi_intsrc *)intr_lookup_source(irq);
  478         if (msi == NULL) {
  479                 mtx_unlock(&msi_lock);
  480                 return (ENOENT);
  481         }
  482 
  483         /* Make sure this is an MSI-X message. */
  484         if (!msi->msi_msix) {
  485                 mtx_unlock(&msi_lock);
  486                 return (EINVAL);
  487         }
  488 
  489         KASSERT(msi->msi_dev != NULL, ("unowned message"));
  490 
  491         /* Clear out the message. */
  492         msi->msi_dev = NULL;
  493         apic_free_vector(msi->msi_vector, msi->msi_irq);
  494         msi->msi_vector = 0;
  495         msi->msi_msix = 0;
  496 
  497         mtx_unlock(&msi_lock);
  498         return (0);
  499 }

Cache object: c49232109ef9ea20b1d17cacc55a8fbc


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.