1 /*
2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD$
4 */
5
6
7 #include <machine/apic.h>
8 #include <machine/smp.h>
9
10 #include "i386/isa/intr_machdep.h"
11
12
13 #ifdef FAST_SIMPLELOCK
14
15 #define GET_FAST_INTR_LOCK \
16 pushl $_fast_intr_lock ; /* address of lock */ \
17 call _s_lock ; /* MP-safe */ \
18 addl $4,%esp
19
20 #define REL_FAST_INTR_LOCK \
21 pushl $_fast_intr_lock ; /* address of lock */ \
22 call _s_unlock ; /* MP-safe */ \
23 addl $4,%esp
24
25 #else /* FAST_SIMPLELOCK */
26
27 #define GET_FAST_INTR_LOCK \
28 call _get_isrlock
29
30 #define REL_FAST_INTR_LOCK \
31 pushl $_mp_lock ; /* GIANT_LOCK */ \
32 call _MPrellock ; \
33 add $4, %esp
34
35 #endif /* FAST_SIMPLELOCK */
36
37 /* convert an absolute IRQ# into a bitmask */
38 #define IRQ_BIT(irq_num) (1 << (irq_num))
39
40 /* make an index into the IO APIC from the IRQ# */
41 #define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2))
42
43
44 /*
45 * Macros for interrupt interrupt entry, call to handler, and exit.
46 */
47
48 #ifdef FAST_WITHOUTCPL
49
50 /*
51 */
52 #define FAST_INTR(irq_num, vec_name) \
53 .text ; \
54 SUPERALIGN_TEXT ; \
55 IDTVEC(vec_name) ; \
56 pushl %eax ; /* save only call-used registers */ \
57 pushl %ecx ; \
58 pushl %edx ; \
59 pushl %ds ; \
60 MAYBE_PUSHL_ES ; \
61 movl $KDSEL,%eax ; \
62 movl %ax,%ds ; \
63 MAYBE_MOVW_AX_ES ; \
64 FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \
65 pushl _intr_unit + (irq_num) * 4 ; \
66 GET_FAST_INTR_LOCK ; \
67 call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
68 REL_FAST_INTR_LOCK ; \
69 addl $4, %esp ; \
70 movl $0, lapic_eoi ; \
71 lock ; \
72 incl _cnt+V_INTR ; /* book-keeping can wait */ \
73 movl _intr_countp + (irq_num) * 4, %eax ; \
74 lock ; \
75 incl (%eax) ; \
76 MEXITCOUNT ; \
77 MAYBE_POPL_ES ; \
78 popl %ds ; \
79 popl %edx ; \
80 popl %ecx ; \
81 popl %eax ; \
82 iret
83
84 #else /* FAST_WITHOUTCPL */
85
86 #define FAST_INTR(irq_num, vec_name) \
87 .text ; \
88 SUPERALIGN_TEXT ; \
89 IDTVEC(vec_name) ; \
90 pushl %eax ; /* save only call-used registers */ \
91 pushl %ecx ; \
92 pushl %edx ; \
93 pushl %ds ; \
94 MAYBE_PUSHL_ES ; \
95 movl $KDSEL, %eax ; \
96 movl %ax, %ds ; \
97 MAYBE_MOVW_AX_ES ; \
98 FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \
99 GET_FAST_INTR_LOCK ; \
100 pushl _intr_unit + (irq_num) * 4 ; \
101 call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
102 addl $4, %esp ; \
103 movl $0, lapic_eoi ; \
104 lock ; \
105 incl _cnt+V_INTR ; /* book-keeping can wait */ \
106 movl _intr_countp + (irq_num) * 4,%eax ; \
107 lock ; \
108 incl (%eax) ; \
109 movl _cpl, %eax ; /* unmasking pending HWIs or SWIs? */ \
110 notl %eax ; \
111 andl _ipending, %eax ; \
112 jne 2f ; /* yes, maybe handle them */ \
113 1: ; \
114 MEXITCOUNT ; \
115 REL_FAST_INTR_LOCK ; \
116 MAYBE_POPL_ES ; \
117 popl %ds ; \
118 popl %edx ; \
119 popl %ecx ; \
120 popl %eax ; \
121 iret ; \
122 ; \
123 ALIGN_TEXT ; \
124 2: ; \
125 cmpb $3, _intr_nesting_level ; /* enough stack? */ \
126 jae 1b ; /* no, return */ \
127 movl _cpl, %eax ; \
128 /* XXX next line is probably unnecessary now. */ \
129 movl $HWI_MASK|SWI_MASK, _cpl ; /* limit nesting ... */ \
130 lock ; \
131 incb _intr_nesting_level ; /* ... really limit it ... */ \
132 sti ; /* to do this as early as possible */ \
133 MAYBE_POPL_ES ; /* discard most of thin frame ... */ \
134 popl %ecx ; /* ... original %ds ... */ \
135 popl %edx ; \
136 xchgl %eax, 4(%esp) ; /* orig %eax; save cpl */ \
137 pushal ; /* build fat frame (grrr) ... */ \
138 pushl %ecx ; /* ... actually %ds ... */ \
139 pushl %es ; \
140 movl $KDSEL, %eax ; \
141 movl %ax, %es ; \
142 movl (2+8+0)*4(%esp), %ecx ; /* %ecx from thin frame ... */ \
143 movl %ecx, (2+6)*4(%esp) ; /* ... to fat frame ... */ \
144 movl (2+8+1)*4(%esp), %eax ; /* ... cpl from thin frame */ \
145 pushl %eax ; \
146 subl $4, %esp ; /* junk for unit number */ \
147 MEXITCOUNT ; \
148 jmp _doreti
149
150 #endif /** FAST_WITHOUTCPL */
151
152
153 /*
154 *
155 */
156 #define PUSH_FRAME \
157 pushl $0 ; /* dummy error code */ \
158 pushl $0 ; /* dummy trap type */ \
159 pushal ; \
160 pushl %ds ; /* save data and extra segments ... */ \
161 pushl %es
162
163 #define POP_FRAME \
164 popl %es ; \
165 popl %ds ; \
166 popal ; \
167 addl $4+4,%esp
168
169 #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
170 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
171
172 #define MASK_IRQ(irq_num) \
173 IMASK_LOCK ; /* into critical reg */ \
174 testl $IRQ_BIT(irq_num), _apic_imen ; \
175 jne 7f ; /* masked, don't mask */ \
176 orl $IRQ_BIT(irq_num), _apic_imen ; /* set the mask bit */ \
177 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
178 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
179 movl %eax, (%ecx) ; /* write the index */ \
180 movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \
181 orl $IOART_INTMASK, %eax ; /* set the mask */ \
182 movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \
183 7: ; /* already masked */ \
184 IMASK_UNLOCK
185 /*
186 * Test to see whether we are handling an edge or level triggered INT.
187 * Level-triggered INTs must still be masked as we don't clear the source,
188 * and the EOI cycle would cause redundant INTs to occur.
189 */
190 #define MASK_LEVEL_IRQ(irq_num) \
191 testl $IRQ_BIT(irq_num), _apic_pin_trigger ; \
192 jz 9f ; /* edge, don't mask */ \
193 MASK_IRQ(irq_num) ; \
194 9:
195
196
197 #ifdef APIC_INTR_REORDER
198 #define EOI_IRQ(irq_num) \
199 movl _apic_isrbit_location + 8 * (irq_num), %eax ; \
200 movl (%eax), %eax ; \
201 testl _apic_isrbit_location + 4 + 8 * (irq_num), %eax ; \
202 jz 9f ; /* not active */ \
203 movl $0, lapic_eoi ; \
204 APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ; \
205 9:
206
207 #else
208 #define EOI_IRQ(irq_num) \
209 testl $IRQ_BIT(irq_num), lapic_isr1; \
210 jz 9f ; /* not active */ \
211 movl $0, lapic_eoi; \
212 APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ; \
213 9:
214 #endif
215
216
217 /*
218 * Test to see if the source is currntly masked, clear if so.
219 */
220 #define UNMASK_IRQ(irq_num) \
221 IMASK_LOCK ; /* into critical reg */ \
222 testl $IRQ_BIT(irq_num), _apic_imen ; \
223 je 7f ; /* bit clear, not masked */ \
224 andl $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */ \
225 movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
226 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
227 movl %eax,(%ecx) ; /* write the index */ \
228 movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \
229 andl $~IOART_INTMASK,%eax ; /* clear the mask */ \
230 movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ \
231 7: ; \
232 IMASK_UNLOCK
233
234 #ifdef INTR_SIMPLELOCK
235 #define ENLOCK
236 #define DELOCK
237 #define LATELOCK call _get_isrlock
238 #else
239 #define ENLOCK \
240 ISR_TRYLOCK ; /* XXX this is going away... */ \
241 testl %eax, %eax ; /* did we get it? */ \
242 jz 3f
243 #define DELOCK ISR_RELLOCK
244 #define LATELOCK
245 #endif
246
247 #ifdef APIC_INTR_DIAGNOSTIC
248 #ifdef APIC_INTR_DIAGNOSTIC_IRQ
249 log_intr_event:
250 pushf
251 cli
252 pushl $CNAME(apic_itrace_debuglock)
253 call _s_lock_np
254 addl $4, %esp
255 movl CNAME(apic_itrace_debugbuffer_idx), %ecx
256 andl $32767, %ecx
257 movl _cpuid, %eax
258 shll $8, %eax
259 orl 8(%esp), %eax
260 movw %ax, CNAME(apic_itrace_debugbuffer)(,%ecx,2)
261 incl %ecx
262 andl $32767, %ecx
263 movl %ecx, CNAME(apic_itrace_debugbuffer_idx)
264 pushl $CNAME(apic_itrace_debuglock)
265 call _s_unlock_np
266 addl $4, %esp
267 popf
268 ret
269
270
271 #define APIC_ITRACE(name, irq_num, id) \
272 lock ; /* MP-safe */ \
273 incl CNAME(name) + (irq_num) * 4 ; \
274 pushl %eax ; \
275 pushl %ecx ; \
276 pushl %edx ; \
277 movl $(irq_num), %eax ; \
278 cmpl $APIC_INTR_DIAGNOSTIC_IRQ, %eax ; \
279 jne 7f ; \
280 pushl $id ; \
281 call log_intr_event ; \
282 addl $4, %esp ; \
283 7: ; \
284 popl %edx ; \
285 popl %ecx ; \
286 popl %eax
287 #else
288 #define APIC_ITRACE(name, irq_num, id) \
289 lock ; /* MP-safe */ \
290 incl CNAME(name) + (irq_num) * 4
291 #endif
292
293 #define APIC_ITRACE_ENTER 1
294 #define APIC_ITRACE_EOI 2
295 #define APIC_ITRACE_TRYISRLOCK 3
296 #define APIC_ITRACE_GOTISRLOCK 4
297 #define APIC_ITRACE_ENTER2 5
298 #define APIC_ITRACE_LEAVE 6
299 #define APIC_ITRACE_UNMASK 7
300 #define APIC_ITRACE_ACTIVE 8
301 #define APIC_ITRACE_MASKED 9
302 #define APIC_ITRACE_NOISRLOCK 10
303 #define APIC_ITRACE_MASKED2 11
304 #define APIC_ITRACE_SPLZ 12
305 #define APIC_ITRACE_DORETI 13
306
307 #else
308 #define APIC_ITRACE(name, irq_num, id)
309 #endif
310
311 #ifdef CPL_AND_CML
312
313 #define INTR(irq_num, vec_name) \
314 .text ; \
315 SUPERALIGN_TEXT ; \
316 /* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \
317 IDTVEC(vec_name) ; \
318 PUSH_FRAME ; \
319 movl $KDSEL, %eax ; /* reload with kernel's data segment */ \
320 movl %ax, %ds ; \
321 movl %ax, %es ; \
322 ; \
323 APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \
324 lock ; /* MP-safe */ \
325 btsl $(irq_num), iactive ; /* lazy masking */ \
326 jc 1f ; /* already active */ \
327 ; \
328 MASK_LEVEL_IRQ(irq_num) ; \
329 EOI_IRQ(irq_num) ; \
330 0: ; \
331 APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
332 ENLOCK ; \
333 ; \
334 APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
335 AVCPL_LOCK ; /* MP-safe */ \
336 testl $IRQ_BIT(irq_num), _cpl ; \
337 jne 2f ; /* this INT masked */ \
338 testl $IRQ_BIT(irq_num), _cml ; \
339 jne 2f ; /* this INT masked */ \
340 orl $IRQ_BIT(irq_num), _cil ; \
341 AVCPL_UNLOCK ; \
342 ; \
343 incb _intr_nesting_level ; \
344 ; \
345 /* entry point used by doreti_unpend for HWIs. */ \
346 __CONCAT(Xresume,irq_num): ; \
347 FAKE_MCOUNT(12*4(%esp)) ; /* XXX avoid dbl cnt */ \
348 lock ; incl _cnt+V_INTR ; /* tally interrupts */ \
349 movl _intr_countp + (irq_num) * 4, %eax ; \
350 lock ; incl (%eax) ; \
351 ; \
352 AVCPL_LOCK ; /* MP-safe */ \
353 movl _cml, %eax ; \
354 pushl %eax ; \
355 orl _intr_mask + (irq_num) * 4, %eax ; \
356 movl %eax, _cml ; \
357 AVCPL_UNLOCK ; \
358 ; \
359 pushl _intr_unit + (irq_num) * 4 ; \
360 incl _inside_intr ; \
361 APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
362 sti ; \
363 call *_intr_handler + (irq_num) * 4 ; \
364 cli ; \
365 APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \
366 decl _inside_intr ; \
367 ; \
368 lock ; andl $~IRQ_BIT(irq_num), iactive ; \
369 lock ; andl $~IRQ_BIT(irq_num), _cil ; \
370 UNMASK_IRQ(irq_num) ; \
371 APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ; \
372 sti ; /* doreti repeats cli/sti */ \
373 MEXITCOUNT ; \
374 LATELOCK ; \
375 jmp _doreti ; \
376 ; \
377 ALIGN_TEXT ; \
378 1: ; /* active */ \
379 APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
380 MASK_IRQ(irq_num) ; \
381 EOI_IRQ(irq_num) ; \
382 AVCPL_LOCK ; /* MP-safe */ \
383 orl $IRQ_BIT(irq_num), _ipending ; \
384 AVCPL_UNLOCK ; \
385 lock ; \
386 btsl $(irq_num), iactive ; /* still active */ \
387 jnc 0b ; /* retry */ \
388 POP_FRAME ; \
389 iret ; \
390 ; \
391 ALIGN_TEXT ; \
392 2: ; /* masked by cpl|cml */ \
393 APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
394 orl $IRQ_BIT(irq_num), _ipending ; \
395 AVCPL_UNLOCK ; \
396 DELOCK ; /* XXX this is going away... */ \
397 POP_FRAME ; \
398 iret ; \
399 ALIGN_TEXT ; \
400 3: ; /* other cpu has isr lock */ \
401 APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
402 AVCPL_LOCK ; /* MP-safe */ \
403 orl $IRQ_BIT(irq_num), _ipending ; \
404 testl $IRQ_BIT(irq_num), _cpl ; \
405 jne 4f ; /* this INT masked */ \
406 testl $IRQ_BIT(irq_num), _cml ; \
407 jne 4f ; /* this INT masked */ \
408 orl $IRQ_BIT(irq_num), _cil ; \
409 AVCPL_UNLOCK ; \
410 call forward_irq ; /* forward irq to lock holder */ \
411 POP_FRAME ; /* and return */ \
412 iret ; \
413 ALIGN_TEXT ; \
414 4: ; /* blocked */ \
415 APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
416 AVCPL_UNLOCK ; \
417 POP_FRAME ; /* and return */ \
418 iret
419
420 #else /* CPL_AND_CML */
421
422
423 #define INTR(irq_num, vec_name) \
424 .text ; \
425 SUPERALIGN_TEXT ; \
426 /* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \
427 IDTVEC(vec_name) ; \
428 PUSH_FRAME ; \
429 movl $KDSEL, %eax ; /* reload with kernel's data segment */ \
430 movl %ax, %ds ; \
431 movl %ax, %es ; \
432 ; \
433 APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \
434 lock ; /* MP-safe */ \
435 btsl $(irq_num), iactive ; /* lazy masking */ \
436 jc 1f ; /* already active */ \
437 ; \
438 MASK_LEVEL_IRQ(irq_num) ; \
439 EOI_IRQ(irq_num) ; \
440 0: ; \
441 APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
442 ISR_TRYLOCK ; /* XXX this is going away... */ \
443 testl %eax, %eax ; /* did we get it? */ \
444 jz 3f ; /* no */ \
445 ; \
446 APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
447 AVCPL_LOCK ; /* MP-safe */ \
448 testl $IRQ_BIT(irq_num), _cpl ; \
449 jne 2f ; /* this INT masked */ \
450 AVCPL_UNLOCK ; \
451 ; \
452 incb _intr_nesting_level ; \
453 ; \
454 /* entry point used by doreti_unpend for HWIs. */ \
455 __CONCAT(Xresume,irq_num): ; \
456 FAKE_MCOUNT(12*4(%esp)) ; /* XXX avoid dbl cnt */ \
457 lock ; incl _cnt+V_INTR ; /* tally interrupts */ \
458 movl _intr_countp + (irq_num) * 4, %eax ; \
459 lock ; incl (%eax) ; \
460 ; \
461 AVCPL_LOCK ; /* MP-safe */ \
462 movl _cpl, %eax ; \
463 pushl %eax ; \
464 orl _intr_mask + (irq_num) * 4, %eax ; \
465 movl %eax, _cpl ; \
466 andl $~IRQ_BIT(irq_num), _ipending ; \
467 AVCPL_UNLOCK ; \
468 ; \
469 pushl _intr_unit + (irq_num) * 4 ; \
470 APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
471 sti ; \
472 call *_intr_handler + (irq_num) * 4 ; \
473 cli ; \
474 APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \
475 ; \
476 lock ; andl $~IRQ_BIT(irq_num), iactive ; \
477 UNMASK_IRQ(irq_num) ; \
478 APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ; \
479 sti ; /* doreti repeats cli/sti */ \
480 MEXITCOUNT ; \
481 jmp _doreti ; \
482 ; \
483 ALIGN_TEXT ; \
484 1: ; /* active */ \
485 APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
486 MASK_IRQ(irq_num) ; \
487 EOI_IRQ(irq_num) ; \
488 AVCPL_LOCK ; /* MP-safe */ \
489 orl $IRQ_BIT(irq_num), _ipending ; \
490 AVCPL_UNLOCK ; \
491 lock ; \
492 btsl $(irq_num), iactive ; /* still active */ \
493 jnc 0b ; /* retry */ \
494 POP_FRAME ; \
495 iret ; /* XXX: iactive bit might be 0 now */ \
496 ALIGN_TEXT ; \
497 2: ; /* masked by cpl, leave iactive set */ \
498 APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
499 orl $IRQ_BIT(irq_num), _ipending ; \
500 AVCPL_UNLOCK ; \
501 ISR_RELLOCK ; /* XXX this is going away... */ \
502 POP_FRAME ; \
503 iret ; \
504 ALIGN_TEXT ; \
505 3: ; /* other cpu has isr lock */ \
506 APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
507 AVCPL_LOCK ; /* MP-safe */ \
508 orl $IRQ_BIT(irq_num), _ipending ; \
509 testl $IRQ_BIT(irq_num), _cpl ; \
510 jne 4f ; /* this INT masked */ \
511 AVCPL_UNLOCK ; \
512 call forward_irq ; /* forward irq to lock holder */ \
513 POP_FRAME ; /* and return */ \
514 iret ; \
515 ALIGN_TEXT ; \
516 4: ; /* blocked */ \
517 APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
518 AVCPL_UNLOCK ; \
519 POP_FRAME ; /* and return */ \
520 iret
521
522 #endif /* CPL_AND_CML */
523
524
525 /*
526 * Handle "spurious INTerrupts".
527 * Notes:
528 * This is different than the "spurious INTerrupt" generated by an
529 * 8259 PIC for missing INTs. See the APIC documentation for details.
530 * This routine should NOT do an 'EOI' cycle.
531 */
532 .text
533 SUPERALIGN_TEXT
534 .globl _Xspuriousint
535 _Xspuriousint:
536
537 /* No EOI cycle used here */
538
539 iret
540
541
542 /*
543 * Handle TLB shootdowns.
544 */
545 .text
546 SUPERALIGN_TEXT
547 .globl _Xinvltlb
548 _Xinvltlb:
549 pushl %eax
550
551 #ifdef COUNT_XINVLTLB_HITS
552 ss
553 movl _cpuid, %eax
554 ss
555 incl _xhits(,%eax,4)
556 #endif /* COUNT_XINVLTLB_HITS */
557
558 movl %cr3, %eax /* invalidate the TLB */
559 movl %eax, %cr3
560
561 ss /* stack segment, avoid %ds load */
562 movl $0, lapic_eoi /* End Of Interrupt to APIC */
563
564 popl %eax
565 iret
566
567
568 #ifdef BETTER_CLOCK
569
570 /*
571 * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
572 *
573 * - Stores current cpu state in checkstate_cpustate[cpuid]
574 * 0 == user, 1 == sys, 2 == intr
575 * - Stores current process in checkstate_curproc[cpuid]
576 *
577 * - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
578 *
579 * stack: 0 -> ds, 4 -> ebx, 8 -> eax, 12 -> eip, 16 -> cs, 20 -> eflags
580 */
581
582 .text
583 SUPERALIGN_TEXT
584 .globl _Xcpucheckstate
585 .globl _checkstate_cpustate
586 .globl _checkstate_curproc
587 .globl _checkstate_pc
588 _Xcpucheckstate:
589 pushl %eax
590 pushl %ebx
591 pushl %ds /* save current data segment */
592
593 movl $KDSEL, %eax
594 movl %ax, %ds /* use KERNEL data segment */
595
596 movl $0, lapic_eoi /* End Of Interrupt to APIC */
597
598 movl $0, %ebx
599 movl 16(%esp), %eax
600 andl $3, %eax
601 cmpl $3, %eax
602 je 1f
603 #ifdef VM86
604 testl $PSL_VM, 20(%esp)
605 jne 1f
606 #endif
607 incl %ebx /* system or interrupt */
608 #ifdef CPL_AND_CML
609 cmpl $0, _inside_intr
610 je 1f
611 incl %ebx /* interrupt */
612 #endif
613 1:
614 movl _cpuid, %eax
615 movl %ebx, _checkstate_cpustate(,%eax,4)
616 movl _curproc, %ebx
617 movl %ebx, _checkstate_curproc(,%eax,4)
618 movl 12(%esp), %ebx
619 movl %ebx, _checkstate_pc(,%eax,4)
620
621 lock /* checkstate_probed_cpus |= (1<<id) */
622 btsl %eax, _checkstate_probed_cpus
623
624 popl %ds /* restore previous data segment */
625 popl %ebx
626 popl %eax
627 iret
628
629 #endif /* BETTER_CLOCK */
630
631 /*
632 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
633 *
634 * - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
635 *
636 * - We need a better method of triggering asts on other cpus.
637 */
638
639 .text
640 SUPERALIGN_TEXT
641 .globl _Xcpuast
642 _Xcpuast:
643 PUSH_FRAME
644 movl $KDSEL, %eax
645 movl %ax, %ds /* use KERNEL data segment */
646 movl %ax, %es
647
648 movl _cpuid, %eax
649 lock /* checkstate_need_ast &= ~(1<<id) */
650 btrl %eax, _checkstate_need_ast
651 movl $0, lapic_eoi /* End Of Interrupt to APIC */
652
653 lock
654 btsl %eax, _checkstate_pending_ast
655 jc 1f
656
657 FAKE_MCOUNT(12*4(%esp))
658
659 /*
660 * Giant locks do not come cheap.
661 * A lot of cycles are going to be wasted here.
662 */
663 call _get_isrlock
664
665 AVCPL_LOCK
666 #ifdef CPL_AND_CML
667 movl _cml, %eax
668 #else
669 movl _cpl, %eax
670 #endif
671 pushl %eax
672 orl $SWI_AST_PENDING, _ipending
673 AVCPL_UNLOCK
674 lock
675 incb _intr_nesting_level
676 sti
677
678 pushl $0
679
680 movl _cpuid, %eax
681 lock
682 btrl %eax, _checkstate_pending_ast
683 lock
684 btrl %eax, CNAME(resched_cpus)
685 jz 2f
686 movl $1, CNAME(want_resched)
687 lock
688 incl CNAME(want_resched_cnt)
689 2:
690 lock
691 incl CNAME(cpuast_cnt)
692 MEXITCOUNT
693 jmp _doreti
694 1:
695 /* We are already in the process of delivering an ast for this CPU */
696 POP_FRAME
697 iret
698
699
700 /*
701 * Executed by a CPU when it receives an XFORWARD_IRQ IPI.
702 */
703
704 .text
705 SUPERALIGN_TEXT
706 .globl _Xforward_irq
707 _Xforward_irq:
708 PUSH_FRAME
709 movl $KDSEL, %eax
710 movl %ax, %ds /* use KERNEL data segment */
711 movl %ax, %es
712
713 movl $0, lapic_eoi /* End Of Interrupt to APIC */
714
715 FAKE_MCOUNT(12*4(%esp))
716
717 ISR_TRYLOCK
718 testl %eax,%eax /* Did we get the lock ? */
719 jz 1f /* No */
720
721 lock
722 incl CNAME(forward_irq_hitcnt)
723 cmpb $4, _intr_nesting_level
724 jae 2f
725
726 AVCPL_LOCK
727 #ifdef CPL_AND_CML
728 movl _cml, %eax
729 #else
730 movl _cpl, %eax
731 #endif
732 pushl %eax
733 AVCPL_UNLOCK
734 lock
735 incb _intr_nesting_level
736 sti
737
738 pushl $0
739
740 MEXITCOUNT
741 jmp _doreti /* Handle forwarded interrupt */
742 1:
743 lock
744 incl CNAME(forward_irq_misscnt)
745 call forward_irq /* Oops, we've lost the isr lock */
746 MEXITCOUNT
747 POP_FRAME
748 iret
749 2:
750 lock
751 incl CNAME(forward_irq_toodeepcnt)
752 3:
753 ISR_RELLOCK
754 MEXITCOUNT
755 POP_FRAME
756 iret
757
758 /*
759 *
760 */
761 forward_irq:
762 MCOUNT
763 cmpl $0,_invltlb_ok
764 jz 4f
765
766 cmpl $0, CNAME(forward_irq_enabled)
767 jz 4f
768
769 movl _mp_lock,%eax
770 cmpl $FREE_LOCK,%eax
771 jne 1f
772 movl $0, %eax /* Pick CPU #0 if noone has lock */
773 1:
774 shrl $24,%eax
775 movl _cpu_num_to_apic_id(,%eax,4),%ecx
776 shll $24,%ecx
777 movl lapic_icr_hi, %eax
778 andl $~APIC_ID_MASK, %eax
779 orl %ecx, %eax
780 movl %eax, lapic_icr_hi
781
782 2:
783 movl lapic_icr_lo, %eax
784 andl $APIC_DELSTAT_MASK,%eax
785 jnz 2b
786 movl lapic_icr_lo, %eax
787 andl $APIC_RESV2_MASK, %eax
788 orl $(APIC_DEST_DESTFLD|APIC_DELMODE_FIXED|XFORWARD_IRQ_OFFSET), %eax
789 movl %eax, lapic_icr_lo
790 3:
791 movl lapic_icr_lo, %eax
792 andl $APIC_DELSTAT_MASK,%eax
793 jnz 3b
794 4:
795 ret
796
797 /*
798 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
799 *
800 * - Signals its receipt.
801 * - Waits for permission to restart.
802 * - Signals its restart.
803 */
804
805 .text
806 SUPERALIGN_TEXT
807 .globl _Xcpustop
808 _Xcpustop:
809 pushl %ebp
810 movl %esp, %ebp
811 pushl %eax
812 pushl %ecx
813 pushl %edx
814 pushl %ds /* save current data segment */
815 pushl %es
816
817 movl $KDSEL, %eax
818 movl %ax, %ds /* use KERNEL data segment */
819
820 movl $0, lapic_eoi /* End Of Interrupt to APIC */
821
822 movl _cpuid, %eax
823 imull $PCB_SIZE, %eax
824 leal CNAME(stoppcbs)(%eax), %eax
825 pushl %eax
826 call CNAME(savectx) /* Save process context */
827 addl $4, %esp
828
829
830 movl _cpuid, %eax
831
832 lock
833 btsl %eax, _stopped_cpus /* stopped_cpus |= (1<<id) */
834 1:
835 btl %eax, _started_cpus /* while (!(started_cpus & (1<<id))) */
836 jnc 1b
837
838 lock
839 btrl %eax, _started_cpus /* started_cpus &= ~(1<<id) */
840 lock
841 btrl %eax, _stopped_cpus /* stopped_cpus &= ~(1<<id) */
842
843 test %eax, %eax
844 jnz 2f
845
846 movl CNAME(cpustop_restartfunc), %eax
847 test %eax, %eax
848 jz 2f
849 movl $0, CNAME(cpustop_restartfunc) /* One-shot */
850
851 call %eax
852 2:
853 popl %es
854 popl %ds /* restore previous data segment */
855 popl %edx
856 popl %ecx
857 popl %eax
858 movl %ebp, %esp
859 popl %ebp
860 iret
861
862
863 MCOUNT_LABEL(bintr)
864 FAST_INTR(0,fastintr0)
865 FAST_INTR(1,fastintr1)
866 FAST_INTR(2,fastintr2)
867 FAST_INTR(3,fastintr3)
868 FAST_INTR(4,fastintr4)
869 FAST_INTR(5,fastintr5)
870 FAST_INTR(6,fastintr6)
871 FAST_INTR(7,fastintr7)
872 FAST_INTR(8,fastintr8)
873 FAST_INTR(9,fastintr9)
874 FAST_INTR(10,fastintr10)
875 FAST_INTR(11,fastintr11)
876 FAST_INTR(12,fastintr12)
877 FAST_INTR(13,fastintr13)
878 FAST_INTR(14,fastintr14)
879 FAST_INTR(15,fastintr15)
880 FAST_INTR(16,fastintr16)
881 FAST_INTR(17,fastintr17)
882 FAST_INTR(18,fastintr18)
883 FAST_INTR(19,fastintr19)
884 FAST_INTR(20,fastintr20)
885 FAST_INTR(21,fastintr21)
886 FAST_INTR(22,fastintr22)
887 FAST_INTR(23,fastintr23)
888 INTR(0,intr0)
889 INTR(1,intr1)
890 INTR(2,intr2)
891 INTR(3,intr3)
892 INTR(4,intr4)
893 INTR(5,intr5)
894 INTR(6,intr6)
895 INTR(7,intr7)
896 INTR(8,intr8)
897 INTR(9,intr9)
898 INTR(10,intr10)
899 INTR(11,intr11)
900 INTR(12,intr12)
901 INTR(13,intr13)
902 INTR(14,intr14)
903 INTR(15,intr15)
904 INTR(16,intr16)
905 INTR(17,intr17)
906 INTR(18,intr18)
907 INTR(19,intr19)
908 INTR(20,intr20)
909 INTR(21,intr21)
910 INTR(22,intr22)
911 INTR(23,intr23)
912 MCOUNT_LABEL(eintr)
913
914 /*
915 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
916 *
917 * - Calls the generic rendezvous action function.
918 */
919 .text
920 SUPERALIGN_TEXT
921 .globl _Xrendezvous
922 _Xrendezvous:
923 PUSH_FRAME
924 movl $KDSEL, %eax
925 movl %ax, %ds /* use KERNEL data segment */
926 movl %ax, %es
927
928 call _smp_rendezvous_action
929
930 movl $0, lapic_eoi /* End Of Interrupt to APIC */
931 POP_FRAME
932 iret
933
934
935 .data
936 /*
937 * Addresses of interrupt handlers.
938 * XresumeNN: Resumption addresses for HWIs.
939 */
940 .globl _ihandlers
941 _ihandlers:
942 /*
943 * used by:
944 * ipl.s: doreti_unpend
945 */
946 .long Xresume0, Xresume1, Xresume2, Xresume3
947 .long Xresume4, Xresume5, Xresume6, Xresume7
948 .long Xresume8, Xresume9, Xresume10, Xresume11
949 .long Xresume12, Xresume13, Xresume14, Xresume15
950 .long Xresume16, Xresume17, Xresume18, Xresume19
951 .long Xresume20, Xresume21, Xresume22, Xresume23
952 /*
953 * used by:
954 * ipl.s: doreti_unpend
955 * apic_ipl.s: splz_unpend
956 */
957 .long _swi_null, swi_net, _swi_null, _swi_null
958 .long _swi_vm, _swi_null, _softclock, swi_ast
959
960 imasks: /* masks for interrupt handlers */
961 .space NHWI*4 /* padding; HWI masks are elsewhere */
962
963 .long SWI_TTY_MASK, SWI_NET_MASK, SWI_CAMNET_MASK, SWI_CAMBIO_MASK
964 .long SWI_VM_MASK, 0, SWI_CLOCK_MASK, SWI_AST_MASK
965
966 /* active flag for lazy masking */
967 iactive:
968 .long 0
969
970 #ifdef COUNT_XINVLTLB_HITS
971 .globl _xhits
972 _xhits:
973 .space (NCPU * 4), 0
974 #endif /* COUNT_XINVLTLB_HITS */
975
976 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
977 .globl _stopped_cpus, _started_cpus
978 _stopped_cpus:
979 .long 0
980 _started_cpus:
981 .long 0
982
983 #ifdef BETTER_CLOCK
984 .globl _checkstate_probed_cpus
985 _checkstate_probed_cpus:
986 .long 0
987 #endif /* BETTER_CLOCK */
988 .globl _checkstate_need_ast
989 _checkstate_need_ast:
990 .long 0
991 _checkstate_pending_ast:
992 .long 0
993 .globl CNAME(forward_irq_misscnt)
994 .globl CNAME(forward_irq_toodeepcnt)
995 .globl CNAME(forward_irq_hitcnt)
996 .globl CNAME(resched_cpus)
997 .globl CNAME(want_resched_cnt)
998 .globl CNAME(cpuast_cnt)
999 .globl CNAME(cpustop_restartfunc)
1000 CNAME(forward_irq_misscnt):
1001 .long 0
1002 CNAME(forward_irq_hitcnt):
1003 .long 0
1004 CNAME(forward_irq_toodeepcnt):
1005 .long 0
1006 CNAME(resched_cpus):
1007 .long 0
1008 CNAME(want_resched_cnt):
1009 .long 0
1010 CNAME(cpuast_cnt):
1011 .long 0
1012 CNAME(cpustop_restartfunc):
1013 .long 0
1014
1015
1016
1017 .globl _apic_pin_trigger
1018 _apic_pin_trigger:
1019 .long 0
1020
1021
1022 /*
1023 * Interrupt counters and names. The format of these and the label names
1024 * must agree with what vmstat expects. The tables are indexed by device
1025 * ids so that we don't have to move the names around as devices are
1026 * attached.
1027 */
1028 #include "vector.h"
1029 .globl _intrcnt, _eintrcnt
1030 _intrcnt:
1031 .space (NR_DEVICES + ICU_LEN) * 4
1032 _eintrcnt:
1033
1034 .globl _intrnames, _eintrnames
1035 _intrnames:
1036 .ascii DEVICE_NAMES
1037 .asciz "stray irq0"
1038 .asciz "stray irq1"
1039 .asciz "stray irq2"
1040 .asciz "stray irq3"
1041 .asciz "stray irq4"
1042 .asciz "stray irq5"
1043 .asciz "stray irq6"
1044 .asciz "stray irq7"
1045 .asciz "stray irq8"
1046 .asciz "stray irq9"
1047 .asciz "stray irq10"
1048 .asciz "stray irq11"
1049 .asciz "stray irq12"
1050 .asciz "stray irq13"
1051 .asciz "stray irq14"
1052 .asciz "stray irq15"
1053 .asciz "stray irq16"
1054 .asciz "stray irq17"
1055 .asciz "stray irq18"
1056 .asciz "stray irq19"
1057 .asciz "stray irq20"
1058 .asciz "stray irq21"
1059 .asciz "stray irq22"
1060 .asciz "stray irq23"
1061 _eintrnames:
1062
1063 .text
Cache object: cc9a69bf3c48b03953ac20f765a90fc8
|