The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/isa/apic_vector.s

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  *      from: vector.s, 386BSD 0.1 unknown origin
    3  * $FreeBSD$
    4  */
    5 
    6 
    7 #include <machine/apic.h>
    8 #include <machine/smp.h>
    9 
   10 #include "i386/isa/intr_machdep.h"
   11 
   12 
   13 #ifdef FAST_SIMPLELOCK
   14 
   15 #define GET_FAST_INTR_LOCK                                              \
   16         pushl   $_fast_intr_lock ;              /* address of lock */   \
   17         call    _s_lock ;                       /* MP-safe */           \
   18         addl    $4,%esp
   19 
   20 #define REL_FAST_INTR_LOCK                                              \
   21         pushl   $_fast_intr_lock ;              /* address of lock */   \
   22         call    _s_unlock ;                     /* MP-safe */           \
   23         addl    $4,%esp
   24 
   25 #else /* FAST_SIMPLELOCK */
   26 
   27 #define GET_FAST_INTR_LOCK                                              \
   28         call    _get_isrlock
   29 
   30 #define REL_FAST_INTR_LOCK                                              \
   31         pushl   $_mp_lock ;     /* GIANT_LOCK */                        \
   32         call    _MPrellock ;                                            \
   33         add     $4, %esp
   34 
   35 #endif /* FAST_SIMPLELOCK */
   36 
   37 /* convert an absolute IRQ# into a bitmask */
   38 #define IRQ_BIT(irq_num)        (1 << (irq_num))
   39 
   40 /* make an index into the IO APIC from the IRQ# */
   41 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
   42 
   43 
   44 /*
   45  * Macros for interrupt interrupt entry, call to handler, and exit.
   46  */
   47 
   48 #ifdef FAST_WITHOUTCPL
   49 
   50 /*
   51  */
   52 #define FAST_INTR(irq_num, vec_name)                                    \
   53         .text ;                                                         \
   54         SUPERALIGN_TEXT ;                                               \
   55 IDTVEC(vec_name) ;                                                      \
   56         pushl   %eax ;          /* save only call-used registers */     \
   57         pushl   %ecx ;                                                  \
   58         pushl   %edx ;                                                  \
   59         pushl   %ds ;                                                   \
   60         MAYBE_PUSHL_ES ;                                                \
   61         movl    $KDSEL,%eax ;                                           \
   62         movl    %ax,%ds ;                                               \
   63         MAYBE_MOVW_AX_ES ;                                              \
   64         FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ;                      \
   65         pushl   _intr_unit + (irq_num) * 4 ;                            \
   66         GET_FAST_INTR_LOCK ;                                            \
   67         call    *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
   68         REL_FAST_INTR_LOCK ;                                            \
   69         addl    $4, %esp ;                                              \
   70         movl    $0, lapic_eoi ;                                         \
   71         lock ;                                                          \
   72         incl    _cnt+V_INTR ;   /* book-keeping can wait */             \
   73         movl    _intr_countp + (irq_num) * 4, %eax ;                    \
   74         lock ;                                                          \
   75         incl    (%eax) ;                                                \
   76         MEXITCOUNT ;                                                    \
   77         MAYBE_POPL_ES ;                                                 \
   78         popl    %ds ;                                                   \
   79         popl    %edx ;                                                  \
   80         popl    %ecx ;                                                  \
   81         popl    %eax ;                                                  \
   82         iret
   83 
   84 #else /* FAST_WITHOUTCPL */
   85 
   86 #define FAST_INTR(irq_num, vec_name)                                    \
   87         .text ;                                                         \
   88         SUPERALIGN_TEXT ;                                               \
   89 IDTVEC(vec_name) ;                                                      \
   90         pushl   %eax ;          /* save only call-used registers */     \
   91         pushl   %ecx ;                                                  \
   92         pushl   %edx ;                                                  \
   93         pushl   %ds ;                                                   \
   94         MAYBE_PUSHL_ES ;                                                \
   95         movl    $KDSEL, %eax ;                                          \
   96         movl    %ax, %ds ;                                              \
   97         MAYBE_MOVW_AX_ES ;                                              \
   98         FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ;                      \
   99         GET_FAST_INTR_LOCK ;                                            \
  100         pushl   _intr_unit + (irq_num) * 4 ;                            \
  101         call    *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
  102         addl    $4, %esp ;                                              \
  103         movl    $0, lapic_eoi ;                                         \
  104         lock ;                                                          \
  105         incl    _cnt+V_INTR ;   /* book-keeping can wait */             \
  106         movl    _intr_countp + (irq_num) * 4,%eax ;                     \
  107         lock ;                                                          \
  108         incl    (%eax) ;                                                \
  109         movl    _cpl, %eax ;    /* unmasking pending HWIs or SWIs? */   \
  110         notl    %eax ;                                                  \
  111         andl    _ipending, %eax ;                                       \
  112         jne     2f ;            /* yes, maybe handle them */            \
  113 1: ;                                                                    \
  114         MEXITCOUNT ;                                                    \
  115         REL_FAST_INTR_LOCK ;                                            \
  116         MAYBE_POPL_ES ;                                                 \
  117         popl    %ds ;                                                   \
  118         popl    %edx ;                                                  \
  119         popl    %ecx ;                                                  \
  120         popl    %eax ;                                                  \
  121         iret ;                                                          \
  122 ;                                                                       \
  123         ALIGN_TEXT ;                                                    \
  124 2: ;                                                                    \
  125         cmpb    $3, _intr_nesting_level ;       /* enough stack? */     \
  126         jae     1b ;            /* no, return */                        \
  127         movl    _cpl, %eax ;                                            \
  128         /* XXX next line is probably unnecessary now. */                \
  129         movl    $HWI_MASK|SWI_MASK, _cpl ;      /* limit nesting ... */ \
  130         lock ;                                                          \
  131         incb    _intr_nesting_level ;   /* ... really limit it ... */   \
  132         sti ;                   /* to do this as early as possible */   \
  133         MAYBE_POPL_ES ;         /* discard most of thin frame ... */    \
  134         popl    %ecx ;          /* ... original %ds ... */              \
  135         popl    %edx ;                                                  \
  136         xchgl   %eax, 4(%esp) ; /* orig %eax; save cpl */               \
  137         pushal ;                /* build fat frame (grrr) ... */        \
  138         pushl   %ecx ;          /* ... actually %ds ... */              \
  139         pushl   %es ;                                                   \
  140         movl    $KDSEL, %eax ;                                          \
  141         movl    %ax, %es ;                                              \
  142         movl    (2+8+0)*4(%esp), %ecx ; /* %ecx from thin frame ... */  \
  143         movl    %ecx, (2+6)*4(%esp) ;   /* ... to fat frame ... */      \
  144         movl    (2+8+1)*4(%esp), %eax ; /* ... cpl from thin frame */   \
  145         pushl   %eax ;                                                  \
  146         subl    $4, %esp ;      /* junk for unit number */              \
  147         MEXITCOUNT ;                                                    \
  148         jmp     _doreti
  149 
  150 #endif /** FAST_WITHOUTCPL */
  151 
  152 
  153 /*
  154  * 
  155  */
  156 #define PUSH_FRAME                                                      \
  157         pushl   $0 ;            /* dummy error code */                  \
  158         pushl   $0 ;            /* dummy trap type */                   \
  159         pushal ;                                                        \
  160         pushl   %ds ;           /* save data and extra segments ... */  \
  161         pushl   %es
  162 
  163 #define POP_FRAME                                                       \
  164         popl    %es ;                                                   \
  165         popl    %ds ;                                                   \
  166         popal ;                                                         \
  167         addl    $4+4,%esp
  168 
  169 #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
  170 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
  171         
  172 #define MASK_IRQ(irq_num)                                               \
  173         IMASK_LOCK ;                            /* into critical reg */ \
  174         testl   $IRQ_BIT(irq_num), _apic_imen ;                         \
  175         jne     7f ;                    /* masked, don't mask */        \
  176         orl     $IRQ_BIT(irq_num), _apic_imen ; /* set the mask bit */  \
  177         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
  178         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  179         movl    %eax, (%ecx) ;                  /* write the index */   \
  180         movl    IOAPIC_WINDOW(%ecx), %eax ;     /* current value */     \
  181         orl     $IOART_INTMASK, %eax ;          /* set the mask */      \
  182         movl    %eax, IOAPIC_WINDOW(%ecx) ;     /* new value */         \
  183 7: ;                                            /* already masked */    \
  184         IMASK_UNLOCK
  185 /*
  186  * Test to see whether we are handling an edge or level triggered INT.
  187  *  Level-triggered INTs must still be masked as we don't clear the source,
  188  *  and the EOI cycle would cause redundant INTs to occur.
  189  */
  190 #define MASK_LEVEL_IRQ(irq_num)                                         \
  191         testl   $IRQ_BIT(irq_num), _apic_pin_trigger ;                  \
  192         jz      9f ;                            /* edge, don't mask */  \
  193         MASK_IRQ(irq_num) ;                                             \
  194 9:
  195 
  196 
  197 #ifdef APIC_INTR_REORDER
  198 #define EOI_IRQ(irq_num)                                                \
  199         movl    _apic_isrbit_location + 8 * (irq_num), %eax ;           \
  200         movl    (%eax), %eax ;                                          \
  201         testl   _apic_isrbit_location + 4 + 8 * (irq_num), %eax ;       \
  202         jz      9f ;                            /* not active */        \
  203         movl    $0, lapic_eoi ;                                         \
  204         APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ;        \
  205 9:
  206 
  207 #else
  208 #define EOI_IRQ(irq_num)                                                \
  209         testl   $IRQ_BIT(irq_num), lapic_isr1;                          \
  210         jz      9f      ;                       /* not active */        \
  211         movl    $0, lapic_eoi;                                          \
  212         APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ;        \
  213 9:
  214 #endif
  215         
  216         
  217 /*
  218  * Test to see if the source is currntly masked, clear if so.
  219  */
  220 #define UNMASK_IRQ(irq_num)                                     \
  221         IMASK_LOCK ;                            /* into critical reg */ \
  222         testl   $IRQ_BIT(irq_num), _apic_imen ;                         \
  223         je      7f ;                    /* bit clear, not masked */     \
  224         andl    $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */    \
  225         movl    IOAPICADDR(irq_num),%ecx ;      /* ioapic addr */       \
  226         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  227         movl    %eax,(%ecx) ;                   /* write the index */   \
  228         movl    IOAPIC_WINDOW(%ecx),%eax ;      /* current value */     \
  229         andl    $~IOART_INTMASK,%eax ;          /* clear the mask */    \
  230         movl    %eax,IOAPIC_WINDOW(%ecx) ;      /* new value */         \
  231 7: ;                                                                    \
  232         IMASK_UNLOCK
  233 
  234 #ifdef INTR_SIMPLELOCK
  235 #define ENLOCK
  236 #define DELOCK
  237 #define LATELOCK call   _get_isrlock
  238 #else
  239 #define ENLOCK \
  240         ISR_TRYLOCK ;           /* XXX this is going away... */         \
  241         testl   %eax, %eax ;                    /* did we get it? */    \
  242         jz      3f
  243 #define DELOCK  ISR_RELLOCK
  244 #define LATELOCK
  245 #endif
  246 
  247 #ifdef APIC_INTR_DIAGNOSTIC
  248 #ifdef APIC_INTR_DIAGNOSTIC_IRQ
  249 log_intr_event:
  250         pushf
  251         cli
  252         pushl   $CNAME(apic_itrace_debuglock)
  253         call    _s_lock_np
  254         addl    $4, %esp
  255         movl    CNAME(apic_itrace_debugbuffer_idx), %ecx
  256         andl    $32767, %ecx
  257         movl    _cpuid, %eax
  258         shll    $8,     %eax
  259         orl     8(%esp), %eax
  260         movw    %ax,    CNAME(apic_itrace_debugbuffer)(,%ecx,2)
  261         incl    %ecx
  262         andl    $32767, %ecx
  263         movl    %ecx,   CNAME(apic_itrace_debugbuffer_idx)
  264         pushl   $CNAME(apic_itrace_debuglock)
  265         call    _s_unlock_np
  266         addl    $4, %esp
  267         popf
  268         ret
  269         
  270 
  271 #define APIC_ITRACE(name, irq_num, id)                                  \
  272         lock ;                                  /* MP-safe */           \
  273         incl    CNAME(name) + (irq_num) * 4 ;                           \
  274         pushl   %eax ;                                                  \
  275         pushl   %ecx ;                                                  \
  276         pushl   %edx ;                                                  \
  277         movl    $(irq_num), %eax ;                                      \
  278         cmpl    $APIC_INTR_DIAGNOSTIC_IRQ, %eax ;                       \
  279         jne     7f ;                                                    \
  280         pushl   $id ;                                                   \
  281         call    log_intr_event ;                                        \
  282         addl    $4, %esp ;                                              \
  283 7: ;                                                                    \
  284         popl    %edx ;                                                  \
  285         popl    %ecx ;                                                  \
  286         popl    %eax
  287 #else
  288 #define APIC_ITRACE(name, irq_num, id)                                  \
  289         lock ;                                  /* MP-safe */           \
  290         incl    CNAME(name) + (irq_num) * 4
  291 #endif
  292 
  293 #define APIC_ITRACE_ENTER 1
  294 #define APIC_ITRACE_EOI 2
  295 #define APIC_ITRACE_TRYISRLOCK 3
  296 #define APIC_ITRACE_GOTISRLOCK 4
  297 #define APIC_ITRACE_ENTER2 5
  298 #define APIC_ITRACE_LEAVE 6
  299 #define APIC_ITRACE_UNMASK 7
  300 #define APIC_ITRACE_ACTIVE 8
  301 #define APIC_ITRACE_MASKED 9
  302 #define APIC_ITRACE_NOISRLOCK 10
  303 #define APIC_ITRACE_MASKED2 11
  304 #define APIC_ITRACE_SPLZ 12
  305 #define APIC_ITRACE_DORETI 13   
  306         
  307 #else   
  308 #define APIC_ITRACE(name, irq_num, id)
  309 #endif
  310                 
  311 #ifdef CPL_AND_CML
  312 
  313 #define INTR(irq_num, vec_name)                                         \
  314         .text ;                                                         \
  315         SUPERALIGN_TEXT ;                                               \
  316 /* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */  \
  317 IDTVEC(vec_name) ;                                                      \
  318         PUSH_FRAME ;                                                    \
  319         movl    $KDSEL, %eax ;  /* reload with kernel's data segment */ \
  320         movl    %ax, %ds ;                                              \
  321         movl    %ax, %es ;                                              \
  322 ;                                                                       \
  323         APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ;    \
  324         lock ;                                  /* MP-safe */           \
  325         btsl    $(irq_num), iactive ;           /* lazy masking */      \
  326         jc      1f ;                            /* already active */    \
  327 ;                                                                       \
  328         MASK_LEVEL_IRQ(irq_num) ;                                       \
  329         EOI_IRQ(irq_num) ;                                              \
  330 0: ;                                                                    \
  331         APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
  332         ENLOCK ;                                                        \
  333 ;                                                                       \
  334         APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
  335         AVCPL_LOCK ;                            /* MP-safe */           \
  336         testl   $IRQ_BIT(irq_num), _cpl ;                               \
  337         jne     2f ;                            /* this INT masked */   \
  338         testl   $IRQ_BIT(irq_num), _cml ;                               \
  339         jne     2f ;                            /* this INT masked */   \
  340         orl     $IRQ_BIT(irq_num), _cil ;                               \
  341         AVCPL_UNLOCK ;                                                  \
  342 ;                                                                       \
  343         incb    _intr_nesting_level ;                                   \
  344 ;                                                                       \
  345   /* entry point used by doreti_unpend for HWIs. */                     \
  346 __CONCAT(Xresume,irq_num): ;                                            \
  347         FAKE_MCOUNT(12*4(%esp)) ;               /* XXX avoid dbl cnt */ \
  348         lock ;  incl    _cnt+V_INTR ;           /* tally interrupts */  \
  349         movl    _intr_countp + (irq_num) * 4, %eax ;                    \
  350         lock ;  incl    (%eax) ;                                        \
  351 ;                                                                       \
  352         AVCPL_LOCK ;                            /* MP-safe */           \
  353         movl    _cml, %eax ;                                            \
  354         pushl   %eax ;                                                  \
  355         orl     _intr_mask + (irq_num) * 4, %eax ;                      \
  356         movl    %eax, _cml ;                                            \
  357         AVCPL_UNLOCK ;                                                  \
  358 ;                                                                       \
  359         pushl   _intr_unit + (irq_num) * 4 ;                            \
  360         incl    _inside_intr ;                                          \
  361         APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ;  \
  362         sti ;                                                           \
  363         call    *_intr_handler + (irq_num) * 4 ;                        \
  364         cli ;                                                           \
  365         APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ;    \
  366         decl    _inside_intr ;                                          \
  367 ;                                                                       \
  368         lock ;  andl $~IRQ_BIT(irq_num), iactive ;                      \
  369         lock ;  andl $~IRQ_BIT(irq_num), _cil ;                         \
  370         UNMASK_IRQ(irq_num) ;                                           \
  371         APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ;  \
  372         sti ;                           /* doreti repeats cli/sti */    \
  373         MEXITCOUNT ;                                                    \
  374         LATELOCK ;                                                      \
  375         jmp     _doreti ;                                               \
  376 ;                                                                       \
  377         ALIGN_TEXT ;                                                    \
  378 1: ;                                            /* active */            \
  379         APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ;  \
  380         MASK_IRQ(irq_num) ;                                             \
  381         EOI_IRQ(irq_num) ;                                              \
  382         AVCPL_LOCK ;                            /* MP-safe */           \
  383         orl     $IRQ_BIT(irq_num), _ipending ;                          \
  384         AVCPL_UNLOCK ;                                                  \
  385         lock ;                                                          \
  386         btsl    $(irq_num), iactive ;           /* still active */      \
  387         jnc     0b ;                            /* retry */             \
  388         POP_FRAME ;                                                     \
  389         iret ;                                                          \
  390 ;                                                                       \
  391         ALIGN_TEXT ;                                                    \
  392 2: ;                                            /* masked by cpl|cml */ \
  393         APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ;  \
  394         orl     $IRQ_BIT(irq_num), _ipending ;                          \
  395         AVCPL_UNLOCK ;                                                  \
  396         DELOCK ;                /* XXX this is going away... */         \
  397         POP_FRAME ;                                                     \
  398         iret ;                                                          \
  399         ALIGN_TEXT ;                                                    \
  400 3: ;                    /* other cpu has isr lock */                    \
  401         APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
  402         AVCPL_LOCK ;                            /* MP-safe */           \
  403         orl     $IRQ_BIT(irq_num), _ipending ;                          \
  404         testl   $IRQ_BIT(irq_num), _cpl ;                               \
  405         jne     4f ;                            /* this INT masked */   \
  406         testl   $IRQ_BIT(irq_num), _cml ;                               \
  407         jne     4f ;                            /* this INT masked */   \
  408         orl     $IRQ_BIT(irq_num), _cil ;                               \
  409         AVCPL_UNLOCK ;                                                  \
  410         call    forward_irq ;   /* forward irq to lock holder */        \
  411         POP_FRAME ;                             /* and return */        \
  412         iret ;                                                          \
  413         ALIGN_TEXT ;                                                    \
  414 4: ;                                            /* blocked */           \
  415         APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
  416         AVCPL_UNLOCK ;                                                  \
  417         POP_FRAME ;                             /* and return */        \
  418         iret
  419 
  420 #else /* CPL_AND_CML */
  421 
  422 
  423 #define INTR(irq_num, vec_name)                                         \
  424         .text ;                                                         \
  425         SUPERALIGN_TEXT ;                                               \
  426 /* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */  \
  427 IDTVEC(vec_name) ;                                                      \
  428         PUSH_FRAME ;                                                    \
  429         movl    $KDSEL, %eax ;  /* reload with kernel's data segment */ \
  430         movl    %ax, %ds ;                                              \
  431         movl    %ax, %es ;                                              \
  432 ;                                                                       \
  433         APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ;    \
  434         lock ;                                  /* MP-safe */           \
  435         btsl    $(irq_num), iactive ;           /* lazy masking */      \
  436         jc      1f ;                            /* already active */    \
  437 ;                                                                       \
  438         MASK_LEVEL_IRQ(irq_num) ;                                       \
  439         EOI_IRQ(irq_num) ;                                              \
  440 0: ;                                                                    \
  441         APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
  442         ISR_TRYLOCK ;           /* XXX this is going away... */         \
  443         testl   %eax, %eax ;                    /* did we get it? */    \
  444         jz      3f ;                            /* no */                \
  445 ;                                                                       \
  446         APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
  447         AVCPL_LOCK ;                            /* MP-safe */           \
  448         testl   $IRQ_BIT(irq_num), _cpl ;                               \
  449         jne     2f ;                            /* this INT masked */   \
  450         AVCPL_UNLOCK ;                                                  \
  451 ;                                                                       \
  452         incb    _intr_nesting_level ;                                   \
  453 ;                                                                       \
  454   /* entry point used by doreti_unpend for HWIs. */                     \
  455 __CONCAT(Xresume,irq_num): ;                                            \
  456         FAKE_MCOUNT(12*4(%esp)) ;               /* XXX avoid dbl cnt */ \
  457         lock ;  incl    _cnt+V_INTR ;           /* tally interrupts */  \
  458         movl    _intr_countp + (irq_num) * 4, %eax ;                    \
  459         lock ;  incl    (%eax) ;                                        \
  460 ;                                                                       \
  461         AVCPL_LOCK ;                            /* MP-safe */           \
  462         movl    _cpl, %eax ;                                            \
  463         pushl   %eax ;                                                  \
  464         orl     _intr_mask + (irq_num) * 4, %eax ;                      \
  465         movl    %eax, _cpl ;                                            \
  466         andl    $~IRQ_BIT(irq_num), _ipending ;                         \
  467         AVCPL_UNLOCK ;                                                  \
  468 ;                                                                       \
  469         pushl   _intr_unit + (irq_num) * 4 ;                            \
  470         APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ;  \
  471         sti ;                                                           \
  472         call    *_intr_handler + (irq_num) * 4 ;                        \
  473         cli ;                                                           \
  474         APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ;    \
  475 ;                                                                       \
  476         lock ;  andl    $~IRQ_BIT(irq_num), iactive ;                   \
  477         UNMASK_IRQ(irq_num) ;                                           \
  478         APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ;  \
  479         sti ;                           /* doreti repeats cli/sti */    \
  480         MEXITCOUNT ;                                                    \
  481         jmp     _doreti ;                                               \
  482 ;                                                                       \
  483         ALIGN_TEXT ;                                                    \
  484 1: ;                                            /* active  */           \
  485         APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ;  \
  486         MASK_IRQ(irq_num) ;                                             \
  487         EOI_IRQ(irq_num) ;                                              \
  488         AVCPL_LOCK ;                            /* MP-safe */           \
  489         orl     $IRQ_BIT(irq_num), _ipending ;                          \
  490         AVCPL_UNLOCK ;                                                  \
  491         lock ;                                                          \
  492         btsl    $(irq_num), iactive ;           /* still active */      \
  493         jnc     0b ;                            /* retry */             \
  494         POP_FRAME ;                                                     \
  495         iret ;          /* XXX:  iactive bit might be 0 now */          \
  496         ALIGN_TEXT ;                                                    \
  497 2: ;                            /* masked by cpl, leave iactive set */  \
  498         APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ;  \
  499         orl     $IRQ_BIT(irq_num), _ipending ;                          \
  500         AVCPL_UNLOCK ;                                                  \
  501         ISR_RELLOCK ;           /* XXX this is going away... */         \
  502         POP_FRAME ;                                                     \
  503         iret ;                                                          \
  504         ALIGN_TEXT ;                                                    \
  505 3: ;                    /* other cpu has isr lock */                    \
  506         APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
  507         AVCPL_LOCK ;                            /* MP-safe */           \
  508         orl     $IRQ_BIT(irq_num), _ipending ;                          \
  509         testl   $IRQ_BIT(irq_num), _cpl ;                               \
  510         jne     4f ;                            /* this INT masked */   \
  511         AVCPL_UNLOCK ;                                                  \
  512         call    forward_irq ;    /* forward irq to lock holder */       \
  513         POP_FRAME ;                             /* and return */        \
  514         iret ;                                                          \
  515         ALIGN_TEXT ;                                                    \
  516 4: ;                                            /* blocked */           \
  517         APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
  518         AVCPL_UNLOCK ;                                                  \
  519         POP_FRAME ;                             /* and return */        \
  520         iret
  521 
  522 #endif /* CPL_AND_CML */
  523 
  524 
  525 /*
  526  * Handle "spurious INTerrupts".
  527  * Notes:
  528  *  This is different than the "spurious INTerrupt" generated by an
  529  *   8259 PIC for missing INTs.  See the APIC documentation for details.
  530  *  This routine should NOT do an 'EOI' cycle.
  531  */
  532         .text
  533         SUPERALIGN_TEXT
  534         .globl _Xspuriousint
  535 _Xspuriousint:
  536 
  537         /* No EOI cycle used here */
  538 
  539         iret
  540 
  541 
  542 /*
  543  * Handle TLB shootdowns.
  544  */
  545         .text
  546         SUPERALIGN_TEXT
  547         .globl  _Xinvltlb
  548 _Xinvltlb:
  549         pushl   %eax
  550 
  551 #ifdef COUNT_XINVLTLB_HITS
  552         ss
  553         movl    _cpuid, %eax
  554         ss
  555         incl    _xhits(,%eax,4)
  556 #endif /* COUNT_XINVLTLB_HITS */
  557 
  558         movl    %cr3, %eax              /* invalidate the TLB */
  559         movl    %eax, %cr3
  560 
  561         ss                              /* stack segment, avoid %ds load */
  562         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
  563 
  564         popl    %eax
  565         iret
  566 
  567 
  568 #ifdef BETTER_CLOCK
  569 
  570 /*
  571  * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
  572  *
  573  *  - Stores current cpu state in checkstate_cpustate[cpuid]
  574  *      0 == user, 1 == sys, 2 == intr
  575  *  - Stores current process in checkstate_curproc[cpuid]
  576  *
  577  *  - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
  578  *
  579  * stack: 0 -> ds, 4 -> ebx, 8 -> eax, 12 -> eip, 16 -> cs, 20 -> eflags
  580  */
  581 
  582         .text
  583         SUPERALIGN_TEXT
  584         .globl _Xcpucheckstate
  585         .globl _checkstate_cpustate
  586         .globl _checkstate_curproc
  587         .globl _checkstate_pc
  588 _Xcpucheckstate:
  589         pushl   %eax
  590         pushl   %ebx            
  591         pushl   %ds                     /* save current data segment */
  592 
  593         movl    $KDSEL, %eax
  594         movl    %ax, %ds                /* use KERNEL data segment */
  595 
  596         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
  597 
  598         movl    $0, %ebx                
  599         movl    16(%esp), %eax  
  600         andl    $3, %eax
  601         cmpl    $3, %eax
  602         je      1f
  603 #ifdef VM86
  604         testl   $PSL_VM, 20(%esp)
  605         jne     1f
  606 #endif
  607         incl    %ebx                    /* system or interrupt */
  608 #ifdef CPL_AND_CML      
  609         cmpl    $0, _inside_intr
  610         je      1f
  611         incl    %ebx                    /* interrupt */
  612 #endif
  613 1:      
  614         movl    _cpuid, %eax
  615         movl    %ebx, _checkstate_cpustate(,%eax,4)
  616         movl    _curproc, %ebx
  617         movl    %ebx, _checkstate_curproc(,%eax,4)
  618         movl    12(%esp), %ebx
  619         movl    %ebx, _checkstate_pc(,%eax,4)
  620 
  621         lock                            /* checkstate_probed_cpus |= (1<<id) */
  622         btsl    %eax, _checkstate_probed_cpus
  623 
  624         popl    %ds                     /* restore previous data segment */
  625         popl    %ebx
  626         popl    %eax
  627         iret
  628 
  629 #endif /* BETTER_CLOCK */
  630 
  631 /*
  632  * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
  633  *
  634  *  - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
  635  *
  636  *  - We need a better method of triggering asts on other cpus.
  637  */
  638 
  639         .text
  640         SUPERALIGN_TEXT
  641         .globl _Xcpuast
  642 _Xcpuast:
  643         PUSH_FRAME
  644         movl    $KDSEL, %eax
  645         movl    %ax, %ds                /* use KERNEL data segment */
  646         movl    %ax, %es
  647 
  648         movl    _cpuid, %eax
  649         lock                            /* checkstate_need_ast &= ~(1<<id) */
  650         btrl    %eax, _checkstate_need_ast
  651         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
  652 
  653         lock
  654         btsl    %eax, _checkstate_pending_ast
  655         jc      1f
  656 
  657         FAKE_MCOUNT(12*4(%esp))
  658 
  659         /* 
  660          * Giant locks do not come cheap.
  661          * A lot of cycles are going to be wasted here.
  662          */
  663         call    _get_isrlock
  664 
  665         AVCPL_LOCK
  666 #ifdef CPL_AND_CML
  667         movl    _cml, %eax
  668 #else
  669         movl    _cpl, %eax
  670 #endif
  671         pushl   %eax
  672         orl     $SWI_AST_PENDING, _ipending
  673         AVCPL_UNLOCK
  674         lock
  675         incb    _intr_nesting_level
  676         sti
  677         
  678         pushl   $0
  679         
  680         movl    _cpuid, %eax
  681         lock    
  682         btrl    %eax, _checkstate_pending_ast
  683         lock    
  684         btrl    %eax, CNAME(resched_cpus)
  685         jz      2f
  686         movl    $1, CNAME(want_resched)
  687         lock
  688         incl    CNAME(want_resched_cnt)
  689 2:              
  690         lock
  691         incl    CNAME(cpuast_cnt)
  692         MEXITCOUNT
  693         jmp     _doreti
  694 1:
  695         /* We are already in the process of delivering an ast for this CPU */
  696         POP_FRAME
  697         iret                    
  698 
  699 
  700 /*
  701  *       Executed by a CPU when it receives an XFORWARD_IRQ IPI.
  702  */
  703 
  704         .text
  705         SUPERALIGN_TEXT
  706         .globl _Xforward_irq
  707 _Xforward_irq:
  708         PUSH_FRAME
  709         movl    $KDSEL, %eax
  710         movl    %ax, %ds                /* use KERNEL data segment */
  711         movl    %ax, %es
  712 
  713         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
  714 
  715         FAKE_MCOUNT(12*4(%esp))
  716 
  717         ISR_TRYLOCK
  718         testl   %eax,%eax               /* Did we get the lock ? */
  719         jz  1f                          /* No */
  720 
  721         lock
  722         incl    CNAME(forward_irq_hitcnt)
  723         cmpb    $4, _intr_nesting_level
  724         jae     2f
  725         
  726         AVCPL_LOCK
  727 #ifdef CPL_AND_CML
  728         movl    _cml, %eax
  729 #else
  730         movl    _cpl, %eax
  731 #endif
  732         pushl   %eax
  733         AVCPL_UNLOCK
  734         lock
  735         incb    _intr_nesting_level
  736         sti
  737         
  738         pushl   $0
  739 
  740         MEXITCOUNT
  741         jmp     _doreti                 /* Handle forwarded interrupt */
  742 1:
  743         lock
  744         incl    CNAME(forward_irq_misscnt)
  745         call    forward_irq     /* Oops, we've lost the isr lock */
  746         MEXITCOUNT
  747         POP_FRAME
  748         iret
  749 2:
  750         lock
  751         incl    CNAME(forward_irq_toodeepcnt)
  752 3:      
  753         ISR_RELLOCK
  754         MEXITCOUNT
  755         POP_FRAME
  756         iret
  757 
  758 /*
  759  * 
  760  */
  761 forward_irq:
  762         MCOUNT
  763         cmpl    $0,_invltlb_ok
  764         jz      4f
  765 
  766         cmpl    $0, CNAME(forward_irq_enabled)
  767         jz      4f
  768 
  769         movl    _mp_lock,%eax
  770         cmpl    $FREE_LOCK,%eax
  771         jne     1f
  772         movl    $0, %eax                /* Pick CPU #0 if noone has lock */
  773 1:
  774         shrl    $24,%eax
  775         movl    _cpu_num_to_apic_id(,%eax,4),%ecx
  776         shll    $24,%ecx
  777         movl    lapic_icr_hi, %eax
  778         andl    $~APIC_ID_MASK, %eax
  779         orl     %ecx, %eax
  780         movl    %eax, lapic_icr_hi
  781 
  782 2:
  783         movl    lapic_icr_lo, %eax
  784         andl    $APIC_DELSTAT_MASK,%eax
  785         jnz     2b
  786         movl    lapic_icr_lo, %eax
  787         andl    $APIC_RESV2_MASK, %eax
  788         orl     $(APIC_DEST_DESTFLD|APIC_DELMODE_FIXED|XFORWARD_IRQ_OFFSET), %eax
  789         movl    %eax, lapic_icr_lo
  790 3:
  791         movl    lapic_icr_lo, %eax
  792         andl    $APIC_DELSTAT_MASK,%eax
  793         jnz     3b
  794 4:              
  795         ret
  796         
  797 /*
  798  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
  799  *
  800  *  - Signals its receipt.
  801  *  - Waits for permission to restart.
  802  *  - Signals its restart.
  803  */
  804 
  805         .text
  806         SUPERALIGN_TEXT
  807         .globl _Xcpustop
  808 _Xcpustop:
  809         pushl   %ebp
  810         movl    %esp, %ebp
  811         pushl   %eax
  812         pushl   %ecx
  813         pushl   %edx
  814         pushl   %ds                     /* save current data segment */
  815         pushl   %es             
  816 
  817         movl    $KDSEL, %eax
  818         movl    %ax, %ds                /* use KERNEL data segment */
  819 
  820         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
  821 
  822         movl    _cpuid, %eax
  823         imull   $PCB_SIZE, %eax
  824         leal    CNAME(stoppcbs)(%eax), %eax
  825         pushl   %eax
  826         call    CNAME(savectx)          /* Save process context */
  827         addl    $4, %esp
  828         
  829                 
  830         movl    _cpuid, %eax
  831 
  832         lock
  833         btsl    %eax, _stopped_cpus     /* stopped_cpus |= (1<<id) */
  834 1:
  835         btl     %eax, _started_cpus     /* while (!(started_cpus & (1<<id))) */
  836         jnc     1b
  837 
  838         lock
  839         btrl    %eax, _started_cpus     /* started_cpus &= ~(1<<id) */
  840         lock
  841         btrl    %eax, _stopped_cpus     /* stopped_cpus &= ~(1<<id) */
  842 
  843         test    %eax, %eax
  844         jnz     2f
  845 
  846         movl    CNAME(cpustop_restartfunc), %eax
  847         test    %eax, %eax
  848         jz      2f
  849         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
  850 
  851         call    %eax
  852 2:
  853         popl    %es
  854         popl    %ds                     /* restore previous data segment */
  855         popl    %edx
  856         popl    %ecx
  857         popl    %eax
  858         movl    %ebp, %esp
  859         popl    %ebp
  860         iret
  861 
  862 
  863 MCOUNT_LABEL(bintr)
  864         FAST_INTR(0,fastintr0)
  865         FAST_INTR(1,fastintr1)
  866         FAST_INTR(2,fastintr2)
  867         FAST_INTR(3,fastintr3)
  868         FAST_INTR(4,fastintr4)
  869         FAST_INTR(5,fastintr5)
  870         FAST_INTR(6,fastintr6)
  871         FAST_INTR(7,fastintr7)
  872         FAST_INTR(8,fastintr8)
  873         FAST_INTR(9,fastintr9)
  874         FAST_INTR(10,fastintr10)
  875         FAST_INTR(11,fastintr11)
  876         FAST_INTR(12,fastintr12)
  877         FAST_INTR(13,fastintr13)
  878         FAST_INTR(14,fastintr14)
  879         FAST_INTR(15,fastintr15)
  880         FAST_INTR(16,fastintr16)
  881         FAST_INTR(17,fastintr17)
  882         FAST_INTR(18,fastintr18)
  883         FAST_INTR(19,fastintr19)
  884         FAST_INTR(20,fastintr20)
  885         FAST_INTR(21,fastintr21)
  886         FAST_INTR(22,fastintr22)
  887         FAST_INTR(23,fastintr23)
  888         INTR(0,intr0)
  889         INTR(1,intr1)
  890         INTR(2,intr2)
  891         INTR(3,intr3)
  892         INTR(4,intr4)
  893         INTR(5,intr5)
  894         INTR(6,intr6)
  895         INTR(7,intr7)
  896         INTR(8,intr8)
  897         INTR(9,intr9)
  898         INTR(10,intr10)
  899         INTR(11,intr11)
  900         INTR(12,intr12)
  901         INTR(13,intr13)
  902         INTR(14,intr14)
  903         INTR(15,intr15)
  904         INTR(16,intr16)
  905         INTR(17,intr17)
  906         INTR(18,intr18)
  907         INTR(19,intr19)
  908         INTR(20,intr20)
  909         INTR(21,intr21)
  910         INTR(22,intr22)
  911         INTR(23,intr23)
  912 MCOUNT_LABEL(eintr)
  913 
  914 /*
  915  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
  916  *
  917  * - Calls the generic rendezvous action function.
  918  */
  919         .text
  920         SUPERALIGN_TEXT
  921         .globl  _Xrendezvous
  922 _Xrendezvous:
  923         PUSH_FRAME
  924         movl    $KDSEL, %eax
  925         movl    %ax, %ds                /* use KERNEL data segment */
  926         movl    %ax, %es
  927 
  928         call    _smp_rendezvous_action
  929 
  930         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
  931         POP_FRAME
  932         iret
  933 
  934         
  935         .data
  936 /*
  937  * Addresses of interrupt handlers.
  938  *  XresumeNN: Resumption addresses for HWIs.
  939  */
  940         .globl _ihandlers
  941 _ihandlers:
  942 /*
  943  * used by:
  944  *  ipl.s:      doreti_unpend
  945  */
  946         .long   Xresume0,  Xresume1,  Xresume2,  Xresume3 
  947         .long   Xresume4,  Xresume5,  Xresume6,  Xresume7
  948         .long   Xresume8,  Xresume9,  Xresume10, Xresume11
  949         .long   Xresume12, Xresume13, Xresume14, Xresume15 
  950         .long   Xresume16, Xresume17, Xresume18, Xresume19
  951         .long   Xresume20, Xresume21, Xresume22, Xresume23
  952 /*
  953  * used by:
  954  *  ipl.s:      doreti_unpend
  955  *  apic_ipl.s: splz_unpend
  956  */
  957         .long   _swi_null, swi_net, _swi_null, _swi_null
  958         .long   _swi_vm, _swi_null, _softclock, swi_ast
  959 
  960 imasks:                         /* masks for interrupt handlers */
  961         .space  NHWI*4          /* padding; HWI masks are elsewhere */
  962 
  963         .long   SWI_TTY_MASK, SWI_NET_MASK, SWI_CAMNET_MASK, SWI_CAMBIO_MASK
  964         .long   SWI_VM_MASK, 0, SWI_CLOCK_MASK, SWI_AST_MASK
  965 
  966 /* active flag for lazy masking */
  967 iactive:
  968         .long   0
  969 
  970 #ifdef COUNT_XINVLTLB_HITS
  971         .globl  _xhits
  972 _xhits:
  973         .space  (NCPU * 4), 0
  974 #endif /* COUNT_XINVLTLB_HITS */
  975 
  976 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
  977         .globl _stopped_cpus, _started_cpus
  978 _stopped_cpus:
  979         .long   0
  980 _started_cpus:
  981         .long   0
  982 
  983 #ifdef BETTER_CLOCK
  984         .globl _checkstate_probed_cpus
  985 _checkstate_probed_cpus:
  986         .long   0       
  987 #endif /* BETTER_CLOCK */
  988         .globl _checkstate_need_ast
  989 _checkstate_need_ast:
  990         .long   0
  991 _checkstate_pending_ast:
  992         .long   0
  993         .globl CNAME(forward_irq_misscnt)
  994         .globl CNAME(forward_irq_toodeepcnt)
  995         .globl CNAME(forward_irq_hitcnt)
  996         .globl CNAME(resched_cpus)
  997         .globl CNAME(want_resched_cnt)
  998         .globl CNAME(cpuast_cnt)
  999         .globl CNAME(cpustop_restartfunc)
 1000 CNAME(forward_irq_misscnt):     
 1001         .long 0
 1002 CNAME(forward_irq_hitcnt):      
 1003         .long 0
 1004 CNAME(forward_irq_toodeepcnt):
 1005         .long 0
 1006 CNAME(resched_cpus):
 1007         .long 0
 1008 CNAME(want_resched_cnt):
 1009         .long 0
 1010 CNAME(cpuast_cnt):
 1011         .long 0
 1012 CNAME(cpustop_restartfunc):
 1013         .long 0
 1014                 
 1015 
 1016 
 1017         .globl  _apic_pin_trigger
 1018 _apic_pin_trigger:
 1019         .long   0
 1020 
 1021 
 1022 /*
 1023  * Interrupt counters and names.  The format of these and the label names
 1024  * must agree with what vmstat expects.  The tables are indexed by device
 1025  * ids so that we don't have to move the names around as devices are
 1026  * attached.
 1027  */
 1028 #include "vector.h"
 1029         .globl  _intrcnt, _eintrcnt
 1030 _intrcnt:
 1031         .space  (NR_DEVICES + ICU_LEN) * 4
 1032 _eintrcnt:
 1033 
 1034         .globl  _intrnames, _eintrnames
 1035 _intrnames:
 1036         .ascii  DEVICE_NAMES
 1037         .asciz  "stray irq0"
 1038         .asciz  "stray irq1"
 1039         .asciz  "stray irq2"
 1040         .asciz  "stray irq3"
 1041         .asciz  "stray irq4"
 1042         .asciz  "stray irq5"
 1043         .asciz  "stray irq6"
 1044         .asciz  "stray irq7"
 1045         .asciz  "stray irq8"
 1046         .asciz  "stray irq9"
 1047         .asciz  "stray irq10"
 1048         .asciz  "stray irq11"
 1049         .asciz  "stray irq12"
 1050         .asciz  "stray irq13"
 1051         .asciz  "stray irq14"
 1052         .asciz  "stray irq15"
 1053         .asciz  "stray irq16"
 1054         .asciz  "stray irq17"
 1055         .asciz  "stray irq18"
 1056         .asciz  "stray irq19"
 1057         .asciz  "stray irq20"
 1058         .asciz  "stray irq21"
 1059         .asciz  "stray irq22"
 1060         .asciz  "stray irq23"
 1061 _eintrnames:
 1062 
 1063         .text

Cache object: cc9a69bf3c48b03953ac20f765a90fc8


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.