The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/apic_vector.S

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1989, 1990 William F. Jolitz.
    3  * Copyright (c) 1990 The Regents of the University of California.
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 4. Neither the name of the University nor the names of its contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  *      from: vector.s, 386BSD 0.1 unknown origin
   31  * $FreeBSD: releng/10.0/sys/amd64/amd64/apic_vector.S 255217 2013-09-04 23:31:29Z kib $
   32  */
   33 
   34 /*
   35  * Interrupt entry points for external interrupts triggered by I/O APICs
   36  * as well as IPI handlers.
   37  */
   38 
   39 #include "opt_smp.h"
   40 
   41 #include <machine/asmacros.h>
   42 #include <x86/apicreg.h>
   43 
   44 #include "assym.s"
   45 
   46 #ifdef SMP
   47 #define LK      lock ;
   48 #else
   49 #define LK
   50 #endif
   51 
   52 /*
   53  * I/O Interrupt Entry Point.  Rather than having one entry point for
   54  * each interrupt source, we use one entry point for each 32-bit word
   55  * in the ISR.  The handler determines the highest bit set in the ISR,
   56  * translates that into a vector, and passes the vector to the
   57  * lapic_handle_intr() function.
   58  */
   59 #define ISR_VEC(index, vec_name)                                        \
   60         .text ;                                                         \
   61         SUPERALIGN_TEXT ;                                               \
   62 IDTVEC(vec_name) ;                                                      \
   63         PUSH_FRAME ;                                                    \
   64         FAKE_MCOUNT(TF_RIP(%rsp)) ;                                     \
   65         movq    lapic, %rdx ;   /* pointer to local APIC */             \
   66         movl    LA_ISR + 16 * (index)(%rdx), %eax ;     /* load ISR */  \
   67         bsrl    %eax, %eax ;    /* index of highest set bit in ISR */   \
   68         jz      1f ;                                                    \
   69         addl    $(32 * index),%eax ;                                    \
   70         movq    %rsp, %rsi      ;                                       \
   71         movl    %eax, %edi ;    /* pass the IRQ */                      \
   72         call    lapic_handle_intr ;                                     \
   73 1: ;                                                                    \
   74         MEXITCOUNT ;                                                    \
   75         jmp     doreti
   76 
   77 /*
   78  * Handle "spurious INTerrupts".
   79  * Notes:
   80  *  This is different than the "spurious INTerrupt" generated by an
   81  *   8259 PIC for missing INTs.  See the APIC documentation for details.
   82  *  This routine should NOT do an 'EOI' cycle.
   83  */
   84         .text
   85         SUPERALIGN_TEXT
   86 IDTVEC(spuriousint)
   87 
   88         /* No EOI cycle used here */
   89 
   90         jmp     doreti_iret
   91 
   92         ISR_VEC(1, apic_isr1)
   93         ISR_VEC(2, apic_isr2)
   94         ISR_VEC(3, apic_isr3)
   95         ISR_VEC(4, apic_isr4)
   96         ISR_VEC(5, apic_isr5)
   97         ISR_VEC(6, apic_isr6)
   98         ISR_VEC(7, apic_isr7)
   99 
  100 /*
  101  * Local APIC periodic timer handler.
  102  */
  103         .text
  104         SUPERALIGN_TEXT
  105 IDTVEC(timerint)
  106         PUSH_FRAME
  107         FAKE_MCOUNT(TF_RIP(%rsp))
  108         movq    %rsp, %rdi
  109         call    lapic_handle_timer
  110         MEXITCOUNT
  111         jmp     doreti
  112 
  113 /*
  114  * Local APIC CMCI handler.
  115  */
  116         .text
  117         SUPERALIGN_TEXT
  118 IDTVEC(cmcint)
  119         PUSH_FRAME
  120         FAKE_MCOUNT(TF_RIP(%rsp))
  121         call    lapic_handle_cmc
  122         MEXITCOUNT
  123         jmp     doreti
  124 
  125 /*
  126  * Local APIC error interrupt handler.
  127  */
  128         .text
  129         SUPERALIGN_TEXT
  130 IDTVEC(errorint)
  131         PUSH_FRAME
  132         FAKE_MCOUNT(TF_RIP(%rsp))
  133         call    lapic_handle_error
  134         MEXITCOUNT
  135         jmp     doreti
  136 
  137 #ifdef XENHVM
  138 /*
  139  * Xen event channel upcall interrupt handler.
  140  * Only used when the hypervisor supports direct vector callbacks.
  141  */
  142         .text
  143         SUPERALIGN_TEXT
  144 IDTVEC(xen_intr_upcall)
  145         PUSH_FRAME
  146         FAKE_MCOUNT(TF_RIP(%rsp))
  147         movq    %rsp, %rdi
  148         call    xen_intr_handle_upcall
  149         MEXITCOUNT
  150         jmp     doreti
  151 #endif
  152 
  153 #ifdef SMP
  154 /*
  155  * Global address space TLB shootdown.
  156  */
  157         .text
  158 
  159 #define NAKE_INTR_CS    24
  160 
  161         SUPERALIGN_TEXT
  162 global_invltlb:
  163         movq    %cr4,%rax
  164         andq    $~0x80,%rax     /* PGE */
  165         movq    %rax,%cr4
  166         orq     $0x80,%rax
  167         movq    %rax,%cr4
  168 invltlb_ret_clear_pm_save:
  169         movq    smp_tlb_pmap,%rdx
  170         testq   %rdx,%rdx
  171         jz      invltlb_ret_rdx
  172         testb   $SEL_RPL_MASK,NAKE_INTR_CS(%rsp)
  173         jz      1f
  174         swapgs
  175 1:
  176         movl    PCPU(CPUID),%eax
  177         jz      2f
  178         swapgs
  179 2:
  180         LK btcl %eax,PM_SAVE(%rdx)
  181         SUPERALIGN_TEXT
  182 invltlb_ret_rdx:
  183         popq    %rdx
  184 invltlb_ret_rax:
  185         movq    lapic, %rax
  186         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
  187         LK incl smp_tlb_wait
  188         popq    %rax
  189         jmp     doreti_iret
  190 
  191         SUPERALIGN_TEXT
  192 IDTVEC(invltlb_pcid)
  193 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
  194         PUSH_FRAME
  195         movl    PCPU(CPUID), %eax
  196 #ifdef COUNT_XINVLTLB_HITS
  197         incl    xhits_gbl(,%rax,4)
  198 #endif
  199 #ifdef COUNT_IPIS
  200         movq    ipi_invltlb_counts(,%rax,8),%rax
  201         incq    (%rax)
  202 #endif
  203         POP_FRAME
  204 #endif
  205 
  206         pushq   %rax
  207         pushq   %rdx
  208 
  209         movq    %cr3,%rax
  210 
  211         movq    $smp_tlb_invpcid,%rdx
  212         cmpl    $0,(%rdx)
  213         je      global_invltlb
  214         cmpl    $-1,(%rdx)
  215         je      global_invltlb
  216 
  217         /*
  218          * Only invalidate TLB for entries with current PCID.
  219          */
  220         cmpl    $0,invpcid_works
  221         je      1f
  222         /* Use invpcid if available. */
  223         movl    $1,%eax /* INVPCID_CTX */
  224         /* invpcid (%rdx),%rax */
  225         .byte 0x66,0x0f,0x38,0x82,0x02
  226         jmp     invltlb_ret_clear_pm_save
  227 1:
  228         /* Otherwise reload %cr3 twice. */
  229         movq    pcid_cr3,%rdx
  230         cmpq    %rax,%rdx
  231         je      2f
  232         movq    %rdx,%cr3       /* Invalidate, bit 63 is zero. */
  233         btsq    $63,%rax
  234 2:
  235         movq    %rax,%cr3
  236         jmp     invltlb_ret_clear_pm_save
  237 
  238         SUPERALIGN_TEXT
  239 IDTVEC(invltlb)
  240 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
  241         PUSH_FRAME
  242         movl    PCPU(CPUID), %eax
  243 #ifdef COUNT_XINVLTLB_HITS
  244         incl    xhits_gbl(,%rax,4)
  245 #endif
  246 #ifdef COUNT_IPIS
  247         movq    ipi_invltlb_counts(,%rax,8),%rax
  248         incq    (%rax)
  249 #endif
  250         POP_FRAME
  251 #endif
  252 
  253         pushq   %rax
  254         movq    %cr3, %rax              /* invalidate the TLB */
  255         movq    %rax, %cr3
  256         jmp     invltlb_ret_rax
  257 
  258 /*
  259  * Single page TLB shootdown
  260  */
  261         .text
  262         SUPERALIGN_TEXT
  263 IDTVEC(invlpg_pcid)
  264 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
  265         PUSH_FRAME
  266         movl    PCPU(CPUID), %eax
  267 #ifdef COUNT_XINVLTLB_HITS
  268         incl    xhits_pg(,%rax,4)
  269 #endif
  270 #ifdef COUNT_IPIS
  271         movq    ipi_invlpg_counts(,%rax,8),%rax
  272         incq    (%rax)
  273 #endif
  274         POP_FRAME
  275 #endif
  276 
  277         pushq   %rax
  278         pushq   %rdx
  279         movq    $smp_tlb_invpcid,%rdx
  280         cmpl    $0,invpcid_works
  281         jne     2f
  282 
  283         /* kernel pmap - use invlpg to invalidate global mapping */
  284         cmpl    $0,(%rdx)
  285         je      3f
  286         cmpl    $-1,(%rdx)
  287         je      global_invltlb
  288 
  289         /*
  290          * PCID supported, but INVPCID is not.
  291          * Temporarily switch to the target address space and do INVLPG.
  292          */
  293         pushq   %rcx
  294         movq    %cr3,%rcx
  295         movq    pcid_cr3,%rax
  296         cmp     %rcx,%rax
  297         je      1f
  298         btsq    $63,%rax
  299         movq    %rax,%cr3
  300 1:      movq    8(%rdx),%rax
  301         invlpg  (%rax)
  302         btsq    $63,%rcx
  303         movq    %rcx,%cr3
  304         popq    %rcx
  305         jmp     invltlb_ret_rdx
  306 
  307         /*
  308          * Invalidate the TLB entry using INVPCID_ADDR.
  309          */
  310 2:
  311         xorl    %eax,%eax
  312 /*      invpcid (%rdx),%rax */
  313         .byte   0x66,0x0f,0x38,0x82,0x02
  314         jmp     invltlb_ret_rdx
  315 
  316         /*
  317          * PCID is not supported or kernel pmap.
  318          * Invalidate single page using INVLPG.
  319          */
  320 3:
  321         movq    8(%rdx),%rax
  322         invlpg  (%rax)
  323         jmp     invltlb_ret_rdx
  324 
  325         SUPERALIGN_TEXT
  326 IDTVEC(invlpg)
  327 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
  328         PUSH_FRAME
  329         movl    PCPU(CPUID), %eax
  330 #ifdef COUNT_XINVLTLB_HITS
  331         incl    xhits_pg(,%rax,4)
  332 #endif
  333 #ifdef COUNT_IPIS
  334         movq    ipi_invlpg_counts(,%rax,8),%rax
  335         incq    (%rax)
  336 #endif
  337         POP_FRAME
  338 #endif
  339 
  340         pushq   %rax
  341         movq    smp_tlb_invpcid+8,%rax
  342         invlpg  (%rax)                  /* invalidate single page */
  343         jmp     invltlb_ret_rax
  344 
  345 /*
  346  * Page range TLB shootdown.
  347  */
  348         .text
  349         SUPERALIGN_TEXT
  350 IDTVEC(invlrng)
  351 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
  352         PUSH_FRAME
  353         movl    PCPU(CPUID), %eax
  354 #ifdef COUNT_XINVLTLB_HITS
  355         incl    xhits_rng(,%rax,4)
  356 #endif
  357 #ifdef COUNT_IPIS
  358         movq    ipi_invlrng_counts(,%rax,8),%rax
  359         incq    (%rax)
  360 #endif
  361         POP_FRAME
  362 #endif
  363 
  364         pushq   %rax
  365         pushq   %rdx
  366         movq    $smp_tlb_invpcid,%rdx
  367         cmpl    $0,pmap_pcid_enabled
  368         je      invlrng_single_page
  369 
  370         /* kernel pmap - use invlpg to invalidate global mapping */
  371         cmpl    $0,(%rdx)
  372         je      invlrng_single_page
  373         cmpl    $-1,(%rdx)
  374         je      global_invltlb
  375         cmpl    $0,invpcid_works
  376         jne     invlrng_invpcid
  377 
  378         pushq   %rcx
  379         movq    %cr3,%rcx
  380         movq    pcid_cr3,%rax
  381         cmpq    %rcx,%rax
  382         je      1f
  383         btsq    $63,%rax
  384         movq    %rax,%cr3
  385 1:
  386         movq    8(%rdx),%rdx
  387         movq    smp_tlb_addr2,%rax
  388 2:
  389         invlpg  (%rdx)
  390         addq    $PAGE_SIZE,%rdx
  391         cmpq    %rax,%rdx
  392         jb      2b
  393         btsq    $63,%rcx
  394         movq    %rcx,%cr3
  395         popq    %rcx
  396         jmp     invltlb_ret_rdx
  397 
  398 invlrng_invpcid:
  399         pushq   %rcx
  400         subq    $16,%rsp
  401         movq    (%rdx),%rcx
  402         movq    %rcx,(%rsp)
  403         movq    8(%rdx),%rax
  404         movq    %rax,8(%rsp)
  405         movq    smp_tlb_addr2,%rcx
  406         subq    %rax,%rcx
  407         shrq    $PAGE_SHIFT,%rcx
  408 1:
  409 //      invpcid (%rdx),%rax
  410         .byte   0x66,0x0f,0x38,0x82,0x02
  411         addq    $PAGE_SIZE,8(%rsp)
  412         dec     %rcx
  413         jne     1b
  414         addq    $16,%rsp
  415         popq    %rcx
  416         jmp     invltlb_ret_rdx
  417 
  418 invlrng_single_page:
  419         movq    8(%rdx),%rdx
  420         movq    smp_tlb_addr2,%rax
  421 1:      invlpg  (%rdx)                  /* invalidate single page */
  422         addq    $PAGE_SIZE,%rdx
  423         cmpq    %rax,%rdx
  424         jb      1b
  425         jmp     invltlb_ret_rdx
  426 
  427 /*
  428  * Invalidate cache.
  429  */
  430         .text
  431         SUPERALIGN_TEXT
  432 IDTVEC(invlcache)
  433 #ifdef COUNT_IPIS
  434         PUSH_FRAME
  435         movl    PCPU(CPUID), %eax
  436         movq    ipi_invlcache_counts(,%rax,8),%rax
  437         incq    (%rax)
  438         POP_FRAME
  439 #endif
  440 
  441         pushq   %rax
  442         wbinvd
  443         jmp     invltlb_ret_rax
  444 
  445 /*
  446  * Handler for IPIs sent via the per-cpu IPI bitmap.
  447  */
  448         .text
  449         SUPERALIGN_TEXT
  450 IDTVEC(ipi_intr_bitmap_handler)         
  451         PUSH_FRAME
  452 
  453         movq    lapic, %rdx
  454         movl    $0, LA_EOI(%rdx)        /* End Of Interrupt to APIC */
  455         
  456         FAKE_MCOUNT(TF_RIP(%rsp))
  457 
  458         call    ipi_bitmap_handler
  459         MEXITCOUNT
  460         jmp     doreti
  461 
  462 /*
  463  * Executed by a CPU when it receives an IPI_STOP from another CPU.
  464  */
  465         .text
  466         SUPERALIGN_TEXT
  467 IDTVEC(cpustop)
  468         PUSH_FRAME
  469 
  470         movq    lapic, %rax
  471         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
  472 
  473         call    cpustop_handler
  474         jmp     doreti
  475 
  476 /*
  477  * Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
  478  */
  479         .text
  480         SUPERALIGN_TEXT
  481 IDTVEC(cpususpend)
  482         PUSH_FRAME
  483 
  484         call    cpususpend_handler
  485         movq    lapic, %rax
  486         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
  487         jmp     doreti
  488 
  489 /*
  490  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
  491  *
  492  * - Calls the generic rendezvous action function.
  493  */
  494         .text
  495         SUPERALIGN_TEXT
  496 IDTVEC(rendezvous)
  497         PUSH_FRAME
  498 #ifdef COUNT_IPIS
  499         movl    PCPU(CPUID), %eax
  500         movq    ipi_rendezvous_counts(,%rax,8), %rax
  501         incq    (%rax)
  502 #endif
  503         call    smp_rendezvous_action
  504         movq    lapic, %rax
  505         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
  506         jmp     doreti
  507 #endif /* SMP */

Cache object: f858836e9c84f0a58ff64e303ecb78b4


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.