1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD: releng/10.0/sys/amd64/amd64/apic_vector.S 255217 2013-09-04 23:31:29Z kib $
32 */
33
34 /*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39 #include "opt_smp.h"
40
41 #include <machine/asmacros.h>
42 #include <x86/apicreg.h>
43
44 #include "assym.s"
45
46 #ifdef SMP
47 #define LK lock ;
48 #else
49 #define LK
50 #endif
51
52 /*
53 * I/O Interrupt Entry Point. Rather than having one entry point for
54 * each interrupt source, we use one entry point for each 32-bit word
55 * in the ISR. The handler determines the highest bit set in the ISR,
56 * translates that into a vector, and passes the vector to the
57 * lapic_handle_intr() function.
58 */
59 #define ISR_VEC(index, vec_name) \
60 .text ; \
61 SUPERALIGN_TEXT ; \
62 IDTVEC(vec_name) ; \
63 PUSH_FRAME ; \
64 FAKE_MCOUNT(TF_RIP(%rsp)) ; \
65 movq lapic, %rdx ; /* pointer to local APIC */ \
66 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \
67 bsrl %eax, %eax ; /* index of highest set bit in ISR */ \
68 jz 1f ; \
69 addl $(32 * index),%eax ; \
70 movq %rsp, %rsi ; \
71 movl %eax, %edi ; /* pass the IRQ */ \
72 call lapic_handle_intr ; \
73 1: ; \
74 MEXITCOUNT ; \
75 jmp doreti
76
77 /*
78 * Handle "spurious INTerrupts".
79 * Notes:
80 * This is different than the "spurious INTerrupt" generated by an
81 * 8259 PIC for missing INTs. See the APIC documentation for details.
82 * This routine should NOT do an 'EOI' cycle.
83 */
84 .text
85 SUPERALIGN_TEXT
86 IDTVEC(spuriousint)
87
88 /* No EOI cycle used here */
89
90 jmp doreti_iret
91
92 ISR_VEC(1, apic_isr1)
93 ISR_VEC(2, apic_isr2)
94 ISR_VEC(3, apic_isr3)
95 ISR_VEC(4, apic_isr4)
96 ISR_VEC(5, apic_isr5)
97 ISR_VEC(6, apic_isr6)
98 ISR_VEC(7, apic_isr7)
99
100 /*
101 * Local APIC periodic timer handler.
102 */
103 .text
104 SUPERALIGN_TEXT
105 IDTVEC(timerint)
106 PUSH_FRAME
107 FAKE_MCOUNT(TF_RIP(%rsp))
108 movq %rsp, %rdi
109 call lapic_handle_timer
110 MEXITCOUNT
111 jmp doreti
112
113 /*
114 * Local APIC CMCI handler.
115 */
116 .text
117 SUPERALIGN_TEXT
118 IDTVEC(cmcint)
119 PUSH_FRAME
120 FAKE_MCOUNT(TF_RIP(%rsp))
121 call lapic_handle_cmc
122 MEXITCOUNT
123 jmp doreti
124
125 /*
126 * Local APIC error interrupt handler.
127 */
128 .text
129 SUPERALIGN_TEXT
130 IDTVEC(errorint)
131 PUSH_FRAME
132 FAKE_MCOUNT(TF_RIP(%rsp))
133 call lapic_handle_error
134 MEXITCOUNT
135 jmp doreti
136
137 #ifdef XENHVM
138 /*
139 * Xen event channel upcall interrupt handler.
140 * Only used when the hypervisor supports direct vector callbacks.
141 */
142 .text
143 SUPERALIGN_TEXT
144 IDTVEC(xen_intr_upcall)
145 PUSH_FRAME
146 FAKE_MCOUNT(TF_RIP(%rsp))
147 movq %rsp, %rdi
148 call xen_intr_handle_upcall
149 MEXITCOUNT
150 jmp doreti
151 #endif
152
153 #ifdef SMP
154 /*
155 * Global address space TLB shootdown.
156 */
157 .text
158
159 #define NAKE_INTR_CS 24
160
161 SUPERALIGN_TEXT
162 global_invltlb:
163 movq %cr4,%rax
164 andq $~0x80,%rax /* PGE */
165 movq %rax,%cr4
166 orq $0x80,%rax
167 movq %rax,%cr4
168 invltlb_ret_clear_pm_save:
169 movq smp_tlb_pmap,%rdx
170 testq %rdx,%rdx
171 jz invltlb_ret_rdx
172 testb $SEL_RPL_MASK,NAKE_INTR_CS(%rsp)
173 jz 1f
174 swapgs
175 1:
176 movl PCPU(CPUID),%eax
177 jz 2f
178 swapgs
179 2:
180 LK btcl %eax,PM_SAVE(%rdx)
181 SUPERALIGN_TEXT
182 invltlb_ret_rdx:
183 popq %rdx
184 invltlb_ret_rax:
185 movq lapic, %rax
186 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
187 LK incl smp_tlb_wait
188 popq %rax
189 jmp doreti_iret
190
191 SUPERALIGN_TEXT
192 IDTVEC(invltlb_pcid)
193 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
194 PUSH_FRAME
195 movl PCPU(CPUID), %eax
196 #ifdef COUNT_XINVLTLB_HITS
197 incl xhits_gbl(,%rax,4)
198 #endif
199 #ifdef COUNT_IPIS
200 movq ipi_invltlb_counts(,%rax,8),%rax
201 incq (%rax)
202 #endif
203 POP_FRAME
204 #endif
205
206 pushq %rax
207 pushq %rdx
208
209 movq %cr3,%rax
210
211 movq $smp_tlb_invpcid,%rdx
212 cmpl $0,(%rdx)
213 je global_invltlb
214 cmpl $-1,(%rdx)
215 je global_invltlb
216
217 /*
218 * Only invalidate TLB for entries with current PCID.
219 */
220 cmpl $0,invpcid_works
221 je 1f
222 /* Use invpcid if available. */
223 movl $1,%eax /* INVPCID_CTX */
224 /* invpcid (%rdx),%rax */
225 .byte 0x66,0x0f,0x38,0x82,0x02
226 jmp invltlb_ret_clear_pm_save
227 1:
228 /* Otherwise reload %cr3 twice. */
229 movq pcid_cr3,%rdx
230 cmpq %rax,%rdx
231 je 2f
232 movq %rdx,%cr3 /* Invalidate, bit 63 is zero. */
233 btsq $63,%rax
234 2:
235 movq %rax,%cr3
236 jmp invltlb_ret_clear_pm_save
237
238 SUPERALIGN_TEXT
239 IDTVEC(invltlb)
240 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
241 PUSH_FRAME
242 movl PCPU(CPUID), %eax
243 #ifdef COUNT_XINVLTLB_HITS
244 incl xhits_gbl(,%rax,4)
245 #endif
246 #ifdef COUNT_IPIS
247 movq ipi_invltlb_counts(,%rax,8),%rax
248 incq (%rax)
249 #endif
250 POP_FRAME
251 #endif
252
253 pushq %rax
254 movq %cr3, %rax /* invalidate the TLB */
255 movq %rax, %cr3
256 jmp invltlb_ret_rax
257
258 /*
259 * Single page TLB shootdown
260 */
261 .text
262 SUPERALIGN_TEXT
263 IDTVEC(invlpg_pcid)
264 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
265 PUSH_FRAME
266 movl PCPU(CPUID), %eax
267 #ifdef COUNT_XINVLTLB_HITS
268 incl xhits_pg(,%rax,4)
269 #endif
270 #ifdef COUNT_IPIS
271 movq ipi_invlpg_counts(,%rax,8),%rax
272 incq (%rax)
273 #endif
274 POP_FRAME
275 #endif
276
277 pushq %rax
278 pushq %rdx
279 movq $smp_tlb_invpcid,%rdx
280 cmpl $0,invpcid_works
281 jne 2f
282
283 /* kernel pmap - use invlpg to invalidate global mapping */
284 cmpl $0,(%rdx)
285 je 3f
286 cmpl $-1,(%rdx)
287 je global_invltlb
288
289 /*
290 * PCID supported, but INVPCID is not.
291 * Temporarily switch to the target address space and do INVLPG.
292 */
293 pushq %rcx
294 movq %cr3,%rcx
295 movq pcid_cr3,%rax
296 cmp %rcx,%rax
297 je 1f
298 btsq $63,%rax
299 movq %rax,%cr3
300 1: movq 8(%rdx),%rax
301 invlpg (%rax)
302 btsq $63,%rcx
303 movq %rcx,%cr3
304 popq %rcx
305 jmp invltlb_ret_rdx
306
307 /*
308 * Invalidate the TLB entry using INVPCID_ADDR.
309 */
310 2:
311 xorl %eax,%eax
312 /* invpcid (%rdx),%rax */
313 .byte 0x66,0x0f,0x38,0x82,0x02
314 jmp invltlb_ret_rdx
315
316 /*
317 * PCID is not supported or kernel pmap.
318 * Invalidate single page using INVLPG.
319 */
320 3:
321 movq 8(%rdx),%rax
322 invlpg (%rax)
323 jmp invltlb_ret_rdx
324
325 SUPERALIGN_TEXT
326 IDTVEC(invlpg)
327 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
328 PUSH_FRAME
329 movl PCPU(CPUID), %eax
330 #ifdef COUNT_XINVLTLB_HITS
331 incl xhits_pg(,%rax,4)
332 #endif
333 #ifdef COUNT_IPIS
334 movq ipi_invlpg_counts(,%rax,8),%rax
335 incq (%rax)
336 #endif
337 POP_FRAME
338 #endif
339
340 pushq %rax
341 movq smp_tlb_invpcid+8,%rax
342 invlpg (%rax) /* invalidate single page */
343 jmp invltlb_ret_rax
344
345 /*
346 * Page range TLB shootdown.
347 */
348 .text
349 SUPERALIGN_TEXT
350 IDTVEC(invlrng)
351 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
352 PUSH_FRAME
353 movl PCPU(CPUID), %eax
354 #ifdef COUNT_XINVLTLB_HITS
355 incl xhits_rng(,%rax,4)
356 #endif
357 #ifdef COUNT_IPIS
358 movq ipi_invlrng_counts(,%rax,8),%rax
359 incq (%rax)
360 #endif
361 POP_FRAME
362 #endif
363
364 pushq %rax
365 pushq %rdx
366 movq $smp_tlb_invpcid,%rdx
367 cmpl $0,pmap_pcid_enabled
368 je invlrng_single_page
369
370 /* kernel pmap - use invlpg to invalidate global mapping */
371 cmpl $0,(%rdx)
372 je invlrng_single_page
373 cmpl $-1,(%rdx)
374 je global_invltlb
375 cmpl $0,invpcid_works
376 jne invlrng_invpcid
377
378 pushq %rcx
379 movq %cr3,%rcx
380 movq pcid_cr3,%rax
381 cmpq %rcx,%rax
382 je 1f
383 btsq $63,%rax
384 movq %rax,%cr3
385 1:
386 movq 8(%rdx),%rdx
387 movq smp_tlb_addr2,%rax
388 2:
389 invlpg (%rdx)
390 addq $PAGE_SIZE,%rdx
391 cmpq %rax,%rdx
392 jb 2b
393 btsq $63,%rcx
394 movq %rcx,%cr3
395 popq %rcx
396 jmp invltlb_ret_rdx
397
398 invlrng_invpcid:
399 pushq %rcx
400 subq $16,%rsp
401 movq (%rdx),%rcx
402 movq %rcx,(%rsp)
403 movq 8(%rdx),%rax
404 movq %rax,8(%rsp)
405 movq smp_tlb_addr2,%rcx
406 subq %rax,%rcx
407 shrq $PAGE_SHIFT,%rcx
408 1:
409 // invpcid (%rdx),%rax
410 .byte 0x66,0x0f,0x38,0x82,0x02
411 addq $PAGE_SIZE,8(%rsp)
412 dec %rcx
413 jne 1b
414 addq $16,%rsp
415 popq %rcx
416 jmp invltlb_ret_rdx
417
418 invlrng_single_page:
419 movq 8(%rdx),%rdx
420 movq smp_tlb_addr2,%rax
421 1: invlpg (%rdx) /* invalidate single page */
422 addq $PAGE_SIZE,%rdx
423 cmpq %rax,%rdx
424 jb 1b
425 jmp invltlb_ret_rdx
426
427 /*
428 * Invalidate cache.
429 */
430 .text
431 SUPERALIGN_TEXT
432 IDTVEC(invlcache)
433 #ifdef COUNT_IPIS
434 PUSH_FRAME
435 movl PCPU(CPUID), %eax
436 movq ipi_invlcache_counts(,%rax,8),%rax
437 incq (%rax)
438 POP_FRAME
439 #endif
440
441 pushq %rax
442 wbinvd
443 jmp invltlb_ret_rax
444
445 /*
446 * Handler for IPIs sent via the per-cpu IPI bitmap.
447 */
448 .text
449 SUPERALIGN_TEXT
450 IDTVEC(ipi_intr_bitmap_handler)
451 PUSH_FRAME
452
453 movq lapic, %rdx
454 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
455
456 FAKE_MCOUNT(TF_RIP(%rsp))
457
458 call ipi_bitmap_handler
459 MEXITCOUNT
460 jmp doreti
461
462 /*
463 * Executed by a CPU when it receives an IPI_STOP from another CPU.
464 */
465 .text
466 SUPERALIGN_TEXT
467 IDTVEC(cpustop)
468 PUSH_FRAME
469
470 movq lapic, %rax
471 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
472
473 call cpustop_handler
474 jmp doreti
475
476 /*
477 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
478 */
479 .text
480 SUPERALIGN_TEXT
481 IDTVEC(cpususpend)
482 PUSH_FRAME
483
484 call cpususpend_handler
485 movq lapic, %rax
486 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
487 jmp doreti
488
489 /*
490 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
491 *
492 * - Calls the generic rendezvous action function.
493 */
494 .text
495 SUPERALIGN_TEXT
496 IDTVEC(rendezvous)
497 PUSH_FRAME
498 #ifdef COUNT_IPIS
499 movl PCPU(CPUID), %eax
500 movq ipi_rendezvous_counts(,%rax,8), %rax
501 incq (%rax)
502 #endif
503 call smp_rendezvous_action
504 movq lapic, %rax
505 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
506 jmp doreti
507 #endif /* SMP */
Cache object: f858836e9c84f0a58ff64e303ecb78b4
|