The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/msi.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2006 Yahoo!, Inc.
    3  * All rights reserved.
    4  * Written by: John Baldwin <jhb@FreeBSD.org>
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 3. Neither the name of the author nor the names of any co-contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  */
   30 
   31 /*
   32  * Support for PCI Message Signalled Interrupts (MSI).  MSI interrupts on
   33  * x86 are basically APIC messages that the northbridge delivers directly
   34  * to the local APICs as if they had come from an I/O APIC.
   35  */
   36 
   37 #include <sys/cdefs.h>
   38 __FBSDID("$FreeBSD: releng/7.4/sys/amd64/amd64/msi.c 204480 2010-02-28 21:21:29Z avg $");
   39 
   40 #include <sys/param.h>
   41 #include <sys/bus.h>
   42 #include <sys/kernel.h>
   43 #include <sys/lock.h>
   44 #include <sys/malloc.h>
   45 #include <sys/mutex.h>
   46 #include <sys/sx.h>
   47 #include <sys/systm.h>
   48 #include <machine/apicreg.h>
   49 #include <machine/cputypes.h>
   50 #include <machine/md_var.h>
   51 #include <machine/frame.h>
   52 #include <machine/intr_machdep.h>
   53 #include <machine/apicvar.h>
   54 #include <machine/specialreg.h>
   55 #include <dev/pci/pcivar.h>
   56 
   57 /* Fields in address for Intel MSI messages. */
   58 #define MSI_INTEL_ADDR_DEST             0x000ff000
   59 #define MSI_INTEL_ADDR_RH               0x00000008
   60 # define MSI_INTEL_ADDR_RH_ON           0x00000008
   61 # define MSI_INTEL_ADDR_RH_OFF          0x00000000
   62 #define MSI_INTEL_ADDR_DM               0x00000004
   63 # define MSI_INTEL_ADDR_DM_PHYSICAL     0x00000000
   64 # define MSI_INTEL_ADDR_DM_LOGICAL      0x00000004
   65 
   66 /* Fields in data for Intel MSI messages. */
   67 #define MSI_INTEL_DATA_TRGRMOD          IOART_TRGRMOD   /* Trigger mode. */
   68 # define MSI_INTEL_DATA_TRGREDG         IOART_TRGREDG
   69 # define MSI_INTEL_DATA_TRGRLVL         IOART_TRGRLVL
   70 #define MSI_INTEL_DATA_LEVEL            0x00004000      /* Polarity. */
   71 # define MSI_INTEL_DATA_DEASSERT        0x00000000
   72 # define MSI_INTEL_DATA_ASSERT          0x00004000
   73 #define MSI_INTEL_DATA_DELMOD           IOART_DELMOD    /* Delivery mode. */
   74 # define MSI_INTEL_DATA_DELFIXED        IOART_DELFIXED
   75 # define MSI_INTEL_DATA_DELLOPRI        IOART_DELLOPRI
   76 # define MSI_INTEL_DATA_DELSMI          IOART_DELSMI
   77 # define MSI_INTEL_DATA_DELNMI          IOART_DELNMI
   78 # define MSI_INTEL_DATA_DELINIT         IOART_DELINIT
   79 # define MSI_INTEL_DATA_DELEXINT        IOART_DELEXINT
   80 #define MSI_INTEL_DATA_INTVEC           IOART_INTVEC    /* Interrupt vector. */
   81 
   82 /*
   83  * Build Intel MSI message and data values from a source.  AMD64 systems
   84  * seem to be compatible, so we use the same function for both.
   85  */
   86 #define INTEL_ADDR(msi)                                                 \
   87         (MSI_INTEL_ADDR_BASE | (msi)->msi_cpu << 12 |                   \
   88             MSI_INTEL_ADDR_RH_OFF | MSI_INTEL_ADDR_DM_PHYSICAL)
   89 #define INTEL_DATA(msi)                                                 \
   90         (MSI_INTEL_DATA_TRGREDG | MSI_INTEL_DATA_DELFIXED | (msi)->msi_vector)
   91 
   92 static MALLOC_DEFINE(M_MSI, "msi", "PCI MSI");
   93 
   94 /*
   95  * MSI sources are bunched into groups.  This is because MSI forces
   96  * all of the messages to share the address and data registers and
   97  * thus certain properties (such as the local APIC ID target on x86).
   98  * Each group has a 'first' source that contains information global to
   99  * the group.  These fields are marked with (g) below.
  100  *
  101  * Note that local APIC ID is kind of special.  Each message will be
  102  * assigned an ID by the system; however, a group will use the ID from
  103  * the first message.
  104  *
  105  * For MSI-X, each message is isolated.
  106  */
  107 struct msi_intsrc {
  108         struct intsrc msi_intsrc;
  109         device_t msi_dev;               /* Owning device. (g) */
  110         struct msi_intsrc *msi_first;   /* First source in group. */
  111         u_int msi_irq;                  /* IRQ cookie. */
  112         u_int msi_msix;                 /* MSI-X message. */
  113         u_int msi_vector:8;             /* IDT vector. */
  114         u_int msi_cpu:8;                /* Local APIC ID. (g) */
  115         u_int msi_count:8;              /* Messages in this group. (g) */
  116 };
  117 
  118 static void     msi_create_source(void);
  119 static void     msi_enable_source(struct intsrc *isrc);
  120 static void     msi_disable_source(struct intsrc *isrc, int eoi);
  121 static void     msi_eoi_source(struct intsrc *isrc);
  122 static void     msi_enable_intr(struct intsrc *isrc);
  123 static void     msi_disable_intr(struct intsrc *isrc);
  124 static int      msi_vector(struct intsrc *isrc);
  125 static int      msi_source_pending(struct intsrc *isrc);
  126 static int      msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
  127                     enum intr_polarity pol);
  128 static void     msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
  129 
  130 struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
  131                        msi_enable_intr, msi_disable_intr, msi_vector,
  132                        msi_source_pending, NULL, NULL, msi_config_intr,
  133                        msi_assign_cpu };
  134 
  135 static int msi_enabled;
  136 static int msi_last_irq;
  137 static struct mtx msi_lock;
  138 
  139 static void
  140 msi_enable_source(struct intsrc *isrc)
  141 {
  142 }
  143 
  144 static void
  145 msi_disable_source(struct intsrc *isrc, int eoi)
  146 {
  147 
  148         if (eoi == PIC_EOI)
  149                 lapic_eoi();
  150 }
  151 
  152 static void
  153 msi_eoi_source(struct intsrc *isrc)
  154 {
  155 
  156         lapic_eoi();
  157 }
  158 
  159 static void
  160 msi_enable_intr(struct intsrc *isrc)
  161 {
  162         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  163 
  164         apic_enable_vector(msi->msi_vector);
  165 }
  166 
  167 static void
  168 msi_disable_intr(struct intsrc *isrc)
  169 {
  170         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  171 
  172         apic_disable_vector(msi->msi_vector);
  173 }
  174 
  175 static int
  176 msi_vector(struct intsrc *isrc)
  177 {
  178         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  179 
  180         return (msi->msi_irq);
  181 }
  182 
  183 static int
  184 msi_source_pending(struct intsrc *isrc)
  185 {
  186 
  187         return (0);
  188 }
  189 
  190 static int
  191 msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
  192     enum intr_polarity pol)
  193 {
  194 
  195         return (ENODEV);
  196 }
  197 
  198 static void
  199 msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
  200 {
  201         struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
  202 
  203         msi->msi_cpu = apic_id;
  204         if (bootverbose)
  205                 printf("msi: Assigning %s IRQ %d to local APIC %u\n",
  206                     msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
  207                     msi->msi_cpu);      
  208         pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
  209 }
  210 
  211 void
  212 msi_init(void)
  213 {
  214 
  215         /* Check if we have a supported CPU. */
  216         switch (cpu_vendor_id) {
  217         case CPU_VENDOR_INTEL:
  218         case CPU_VENDOR_AMD:
  219                 break;
  220         case CPU_VENDOR_CENTAUR:
  221                 if (CPUID_TO_FAMILY(cpu_id) == 0x6 &&
  222                     CPUID_TO_MODEL(cpu_id) >= 0xf)
  223                         break;
  224                 /* FALLTHROUGH */
  225         default:
  226                 return;
  227         }
  228 
  229         msi_enabled = 1;
  230         intr_register_pic(&msi_pic);
  231         mtx_init(&msi_lock, "msi", NULL, MTX_DEF);
  232 }
  233 
  234 static void
  235 msi_create_source(void)
  236 {
  237         struct msi_intsrc *msi;
  238         u_int irq;
  239 
  240         mtx_lock(&msi_lock);
  241         if (msi_last_irq >= NUM_MSI_INTS) {
  242                 mtx_unlock(&msi_lock);
  243                 return;
  244         }
  245         irq = msi_last_irq + FIRST_MSI_INT;
  246         msi_last_irq++;
  247         mtx_unlock(&msi_lock);
  248 
  249         msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);      
  250         msi->msi_intsrc.is_pic = &msi_pic;
  251         msi->msi_irq = irq;
  252         intr_register_source(&msi->msi_intsrc);
  253         nexus_add_irq(irq);
  254 }
  255 
  256 /*
  257  * Try to allocate 'count' interrupt sources with contiguous IDT values.  If
  258  * we allocate any new sources, then their IRQ values will be at the end of
  259  * the irqs[] array, with *newirq being the index of the first new IRQ value
  260  * and *newcount being the number of new IRQ values added.
  261  */
  262 int
  263 msi_alloc(device_t dev, int count, int maxcount, int *irqs)
  264 {
  265         struct msi_intsrc *msi, *fsrc;
  266         int cnt, i, vector;
  267 
  268         if (!msi_enabled)
  269                 return (ENXIO);
  270 
  271 again:
  272         mtx_lock(&msi_lock);
  273 
  274         /* Try to find 'count' free IRQs. */
  275         cnt = 0;
  276         for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
  277                 msi = (struct msi_intsrc *)intr_lookup_source(i);
  278 
  279                 /* End of allocated sources, so break. */
  280                 if (msi == NULL)
  281                         break;
  282 
  283                 /* If this is a free one, save its IRQ in the array. */
  284                 if (msi->msi_dev == NULL) {
  285                         irqs[cnt] = i;
  286                         cnt++;
  287                         if (cnt == count)
  288                                 break;
  289                 }
  290         }
  291 
  292         /* Do we need to create some new sources? */
  293         if (cnt < count) {
  294                 /* If we would exceed the max, give up. */
  295                 if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) {
  296                         mtx_unlock(&msi_lock);
  297                         return (ENXIO);
  298                 }
  299                 mtx_unlock(&msi_lock);
  300 
  301                 /* We need count - cnt more sources. */
  302                 while (cnt < count) {
  303                         msi_create_source();
  304                         cnt++;
  305                 }
  306                 goto again;
  307         }
  308 
  309         /* Ok, we now have the IRQs allocated. */
  310         KASSERT(cnt == count, ("count mismatch"));
  311 
  312         /* Allocate 'count' IDT vectors. */
  313         vector = apic_alloc_vectors(irqs, count, maxcount);
  314         if (vector == 0) {
  315                 mtx_unlock(&msi_lock);
  316                 return (ENOSPC);
  317         }
  318 
  319         /* Assign IDT vectors and make these messages owned by 'dev'. */
  320         fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
  321         for (i = 0; i < count; i++) {
  322                 msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
  323                 msi->msi_dev = dev;
  324                 msi->msi_cpu = PCPU_GET(apic_id);
  325                 msi->msi_vector = vector + i;
  326                 if (bootverbose)
  327                         printf("msi: routing MSI IRQ %d to vector %u\n",
  328                             msi->msi_irq, msi->msi_vector);
  329                 msi->msi_first = fsrc;
  330                 KASSERT(msi->msi_intsrc.is_handlers == 0,
  331                     ("dead MSI has handlers"));
  332         }
  333         fsrc->msi_count = count;
  334         mtx_unlock(&msi_lock);
  335 
  336         return (0);
  337 }
  338 
  339 int
  340 msi_release(int *irqs, int count)
  341 {
  342         struct msi_intsrc *msi, *first;
  343         int i;
  344 
  345         mtx_lock(&msi_lock);
  346         first = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
  347         if (first == NULL) {
  348                 mtx_unlock(&msi_lock);
  349                 return (ENOENT);
  350         }
  351 
  352         /* Make sure this isn't an MSI-X message. */
  353         if (first->msi_msix) {
  354                 mtx_unlock(&msi_lock);
  355                 return (EINVAL);
  356         }
  357 
  358         /* Make sure this message is allocated to a group. */
  359         if (first->msi_first == NULL) {
  360                 mtx_unlock(&msi_lock);
  361                 return (ENXIO);
  362         }
  363 
  364         /*
  365          * Make sure this is the start of a group and that we are releasing
  366          * the entire group.
  367          */
  368         if (first->msi_first != first || first->msi_count != count) {
  369                 mtx_unlock(&msi_lock);
  370                 return (EINVAL);
  371         }
  372         KASSERT(first->msi_dev != NULL, ("unowned group"));
  373 
  374         /* Clear all the extra messages in the group. */
  375         for (i = 1; i < count; i++) {
  376                 msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
  377                 KASSERT(msi->msi_first == first, ("message not in group"));
  378                 KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
  379                 msi->msi_first = NULL;
  380                 msi->msi_dev = NULL;
  381                 apic_free_vector(msi->msi_vector, msi->msi_irq);
  382                 msi->msi_vector = 0;
  383         }
  384 
  385         /* Clear out the first message. */
  386         first->msi_first = NULL;
  387         first->msi_dev = NULL;
  388         apic_free_vector(first->msi_vector, first->msi_irq);
  389         first->msi_vector = 0;
  390         first->msi_count = 0;
  391 
  392         mtx_unlock(&msi_lock);
  393         return (0);
  394 }
  395 
  396 int
  397 msi_map(int irq, uint64_t *addr, uint32_t *data)
  398 {
  399         struct msi_intsrc *msi;
  400 
  401         mtx_lock(&msi_lock);
  402         msi = (struct msi_intsrc *)intr_lookup_source(irq);
  403         if (msi == NULL) {
  404                 mtx_unlock(&msi_lock);
  405                 return (ENOENT);
  406         }
  407 
  408         /* Make sure this message is allocated to a device. */
  409         if (msi->msi_dev == NULL) {
  410                 mtx_unlock(&msi_lock);
  411                 return (ENXIO);
  412         }
  413 
  414         /*
  415          * If this message isn't an MSI-X message, make sure it's part
  416          * of a group, and switch to the first message in the
  417          * group.
  418          */
  419         if (!msi->msi_msix) {
  420                 if (msi->msi_first == NULL) {
  421                         mtx_unlock(&msi_lock);
  422                         return (ENXIO);
  423                 }
  424                 msi = msi->msi_first;
  425         }
  426 
  427         *addr = INTEL_ADDR(msi);
  428         *data = INTEL_DATA(msi);
  429         mtx_unlock(&msi_lock);
  430         return (0);
  431 }
  432 
  433 int
  434 msix_alloc(device_t dev, int *irq)
  435 {
  436         struct msi_intsrc *msi;
  437         int i, vector;
  438 
  439         if (!msi_enabled)
  440                 return (ENXIO);
  441 
  442 again:
  443         mtx_lock(&msi_lock);
  444 
  445         /* Find a free IRQ. */
  446         for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
  447                 msi = (struct msi_intsrc *)intr_lookup_source(i);
  448 
  449                 /* End of allocated sources, so break. */
  450                 if (msi == NULL)
  451                         break;
  452 
  453                 /* Stop at the first free source. */
  454                 if (msi->msi_dev == NULL)
  455                         break;
  456         }
  457 
  458         /* Do we need to create a new source? */
  459         if (msi == NULL) {
  460                 /* If we would exceed the max, give up. */
  461                 if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) {
  462                         mtx_unlock(&msi_lock);
  463                         return (ENXIO);
  464                 }
  465                 mtx_unlock(&msi_lock);
  466 
  467                 /* Create a new source. */
  468                 msi_create_source();
  469                 goto again;
  470         }
  471 
  472         /* Allocate an IDT vector. */
  473         vector = apic_alloc_vector(i);
  474         if (bootverbose)
  475                 printf("msi: routing MSI-X IRQ %d to vector %u\n", msi->msi_irq,
  476                     vector);
  477 
  478         /* Setup source. */
  479         msi->msi_dev = dev;
  480         msi->msi_vector = vector;
  481         msi->msi_cpu = PCPU_GET(apic_id);
  482         msi->msi_msix = 1;
  483 
  484         KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
  485         mtx_unlock(&msi_lock);
  486 
  487         *irq = i;
  488         return (0);
  489 }
  490 
  491 int
  492 msix_release(int irq)
  493 {
  494         struct msi_intsrc *msi;
  495 
  496         mtx_lock(&msi_lock);
  497         msi = (struct msi_intsrc *)intr_lookup_source(irq);
  498         if (msi == NULL) {
  499                 mtx_unlock(&msi_lock);
  500                 return (ENOENT);
  501         }
  502 
  503         /* Make sure this is an MSI-X message. */
  504         if (!msi->msi_msix) {
  505                 mtx_unlock(&msi_lock);
  506                 return (EINVAL);
  507         }
  508 
  509         KASSERT(msi->msi_dev != NULL, ("unowned message"));
  510 
  511         /* Clear out the message. */
  512         msi->msi_dev = NULL;
  513         apic_free_vector(msi->msi_vector, msi->msi_irq);
  514         msi->msi_vector = 0;
  515         msi->msi_msix = 0;
  516 
  517         mtx_unlock(&msi_lock);
  518         return (0);
  519 }

Cache object: 2f02f0eacaf9e8802d81c542426ee025


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.