1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 * Copyright (c) 2014-2018 The FreeBSD Foundation
6 * All rights reserved.
7 *
8 * Portions of this software were developed by
9 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
10 * the FreeBSD Foundation.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * from: vector.s, 386BSD 0.1 unknown origin
37 * $FreeBSD: releng/12.0/sys/amd64/amd64/apic_vector.S 331254 2018-03-20 17:58:51Z emaste $
38 */
39
40 /*
41 * Interrupt entry points for external interrupts triggered by I/O APICs
42 * as well as IPI handlers.
43 */
44
45 #include "opt_smp.h"
46
47 #include "assym.inc"
48
49 #include <machine/asmacros.h>
50 #include <machine/specialreg.h>
51 #include <x86/apicreg.h>
52
53 #ifdef SMP
54 #define LK lock ;
55 #else
56 #define LK
57 #endif
58
59 .text
60 SUPERALIGN_TEXT
61 /* End Of Interrupt to APIC */
62 as_lapic_eoi:
63 cmpl $0,x2apic_mode
64 jne 1f
65 movq lapic_map,%rax
66 movl $0,LA_EOI(%rax)
67 ret
68 1:
69 movl $MSR_APIC_EOI,%ecx
70 xorl %eax,%eax
71 xorl %edx,%edx
72 wrmsr
73 ret
74
75 /*
76 * I/O Interrupt Entry Point. Rather than having one entry point for
77 * each interrupt source, we use one entry point for each 32-bit word
78 * in the ISR. The handler determines the highest bit set in the ISR,
79 * translates that into a vector, and passes the vector to the
80 * lapic_handle_intr() function.
81 */
82 .macro ISR_VEC index, vec_name
83 INTR_HANDLER \vec_name
84 FAKE_MCOUNT(TF_RIP(%rsp))
85 cmpl $0,x2apic_mode
86 je 1f
87 movl $(MSR_APIC_ISR0 + \index),%ecx
88 rdmsr
89 jmp 2f
90 1:
91 movq lapic_map, %rdx /* pointer to local APIC */
92 movl LA_ISR + 16 * (\index)(%rdx), %eax /* load ISR */
93 2:
94 bsrl %eax, %eax /* index of highest set bit in ISR */
95 jz 3f
96 addl $(32 * \index),%eax
97 movq %rsp, %rsi
98 movl %eax, %edi /* pass the IRQ */
99 call lapic_handle_intr
100 3:
101 MEXITCOUNT
102 jmp doreti
103 .endm
104
105 /*
106 * Handle "spurious INTerrupts".
107 * Notes:
108 * This is different than the "spurious INTerrupt" generated by an
109 * 8259 PIC for missing INTs. See the APIC documentation for details.
110 * This routine should NOT do an 'EOI' cycle.
111 */
112 .text
113 SUPERALIGN_TEXT
114 IDTVEC(spuriousint)
115 /* No EOI cycle used here */
116 jmp doreti_iret
117
118 ISR_VEC 1, apic_isr1
119 ISR_VEC 2, apic_isr2
120 ISR_VEC 3, apic_isr3
121 ISR_VEC 4, apic_isr4
122 ISR_VEC 5, apic_isr5
123 ISR_VEC 6, apic_isr6
124 ISR_VEC 7, apic_isr7
125
126 /*
127 * Local APIC periodic timer handler.
128 */
129 INTR_HANDLER timerint
130 FAKE_MCOUNT(TF_RIP(%rsp))
131 movq %rsp, %rdi
132 call lapic_handle_timer
133 MEXITCOUNT
134 jmp doreti
135
136 /*
137 * Local APIC CMCI handler.
138 */
139 INTR_HANDLER cmcint
140 FAKE_MCOUNT(TF_RIP(%rsp))
141 call lapic_handle_cmc
142 MEXITCOUNT
143 jmp doreti
144
145 /*
146 * Local APIC error interrupt handler.
147 */
148 INTR_HANDLER errorint
149 FAKE_MCOUNT(TF_RIP(%rsp))
150 call lapic_handle_error
151 MEXITCOUNT
152 jmp doreti
153
154 #ifdef XENHVM
155 /*
156 * Xen event channel upcall interrupt handler.
157 * Only used when the hypervisor supports direct vector callbacks.
158 */
159 INTR_HANDLER xen_intr_upcall
160 FAKE_MCOUNT(TF_RIP(%rsp))
161 movq %rsp, %rdi
162 call xen_intr_handle_upcall
163 MEXITCOUNT
164 jmp doreti
165 #endif
166
167 #ifdef SMP
168 /*
169 * Global address space TLB shootdown.
170 */
171 .text
172
173 SUPERALIGN_TEXT
174 invltlb_ret:
175 call as_lapic_eoi
176 jmp ld_regs
177
178 SUPERALIGN_TEXT
179 INTR_HANDLER invltlb
180 call invltlb_handler
181 jmp invltlb_ret
182
183 INTR_HANDLER invltlb_pcid
184 call invltlb_pcid_handler
185 jmp invltlb_ret
186
187 INTR_HANDLER invltlb_invpcid_nopti
188 call invltlb_invpcid_handler
189 jmp invltlb_ret
190
191 INTR_HANDLER invltlb_invpcid_pti
192 call invltlb_invpcid_pti_handler
193 jmp invltlb_ret
194
195 /*
196 * Single page TLB shootdown
197 */
198 INTR_HANDLER invlpg
199 call invlpg_handler
200 jmp invltlb_ret
201
202 INTR_HANDLER invlpg_invpcid
203 call invlpg_invpcid_handler
204 jmp invltlb_ret
205
206 INTR_HANDLER invlpg_pcid
207 call invlpg_pcid_handler
208 jmp invltlb_ret
209
210 /*
211 * Page range TLB shootdown.
212 */
213 INTR_HANDLER invlrng
214 call invlrng_handler
215 jmp invltlb_ret
216
217 INTR_HANDLER invlrng_invpcid
218 call invlrng_invpcid_handler
219 jmp invltlb_ret
220
221 INTR_HANDLER invlrng_pcid
222 call invlrng_pcid_handler
223 jmp invltlb_ret
224
225 /*
226 * Invalidate cache.
227 */
228 INTR_HANDLER invlcache
229 call invlcache_handler
230 jmp invltlb_ret
231
232 /*
233 * Handler for IPIs sent via the per-cpu IPI bitmap.
234 */
235 INTR_HANDLER ipi_intr_bitmap_handler
236 call as_lapic_eoi
237 FAKE_MCOUNT(TF_RIP(%rsp))
238 call ipi_bitmap_handler
239 MEXITCOUNT
240 jmp doreti
241
242 /*
243 * Executed by a CPU when it receives an IPI_STOP from another CPU.
244 */
245 INTR_HANDLER cpustop
246 call as_lapic_eoi
247 call cpustop_handler
248 jmp doreti
249
250 /*
251 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
252 */
253 INTR_HANDLER cpususpend
254 call cpususpend_handler
255 call as_lapic_eoi
256 jmp doreti
257
258 /*
259 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
260 *
261 * - Calls the generic rendezvous action function.
262 */
263 INTR_HANDLER rendezvous
264 #ifdef COUNT_IPIS
265 movl PCPU(CPUID), %eax
266 movq ipi_rendezvous_counts(,%rax,8), %rax
267 incq (%rax)
268 #endif
269 call smp_rendezvous_action
270 call as_lapic_eoi
271 jmp doreti
272
273 /*
274 * IPI handler whose purpose is to interrupt the CPU with minimum overhead.
275 * This is used by bhyve to force a host cpu executing in guest context to
276 * trap into the hypervisor.
277 *
278 * This handler is different from other IPI handlers in the following aspects:
279 *
280 * 1. It doesn't push a trapframe on the stack.
281 *
282 * This implies that a DDB backtrace involving 'justreturn' will skip the
283 * function that was interrupted by this handler.
284 *
285 * 2. It doesn't 'swapgs' when userspace is interrupted.
286 *
287 * The 'justreturn' handler does not access any pcpu data so it is not an
288 * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI
289 * whose handler already doesn't trust GS.base when kernel code is interrupted.
290 */
291 .text
292 SUPERALIGN_TEXT
293 IDTVEC(justreturn)
294 pushq %rax
295 pushq %rcx
296 pushq %rdx
297 call as_lapic_eoi
298 popq %rdx
299 popq %rcx
300 popq %rax
301 jmp doreti_iret
302
303 INTR_HANDLER justreturn1
304 call as_lapic_eoi
305 jmp doreti
306
307 #endif /* SMP */
Cache object: baef793a4f4013c205bfc26e94a3e5d0
|