1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD: releng/9.0/sys/amd64/amd64/apic_vector.S 217360 2011-01-13 17:00:22Z jhb $
32 */
33
34 /*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39 #include "opt_smp.h"
40
41 #include <machine/asmacros.h>
42 #include <x86/apicreg.h>
43
44 #include "assym.s"
45
46 /*
47 * I/O Interrupt Entry Point. Rather than having one entry point for
48 * each interrupt source, we use one entry point for each 32-bit word
49 * in the ISR. The handler determines the highest bit set in the ISR,
50 * translates that into a vector, and passes the vector to the
51 * lapic_handle_intr() function.
52 */
53 #define ISR_VEC(index, vec_name) \
54 .text ; \
55 SUPERALIGN_TEXT ; \
56 IDTVEC(vec_name) ; \
57 PUSH_FRAME ; \
58 FAKE_MCOUNT(TF_RIP(%rsp)) ; \
59 movq lapic, %rdx ; /* pointer to local APIC */ \
60 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \
61 bsrl %eax, %eax ; /* index of highest set bit in ISR */ \
62 jz 1f ; \
63 addl $(32 * index),%eax ; \
64 movq %rsp, %rsi ; \
65 movl %eax, %edi ; /* pass the IRQ */ \
66 call lapic_handle_intr ; \
67 1: ; \
68 MEXITCOUNT ; \
69 jmp doreti
70
71 /*
72 * Handle "spurious INTerrupts".
73 * Notes:
74 * This is different than the "spurious INTerrupt" generated by an
75 * 8259 PIC for missing INTs. See the APIC documentation for details.
76 * This routine should NOT do an 'EOI' cycle.
77 */
78 .text
79 SUPERALIGN_TEXT
80 IDTVEC(spuriousint)
81
82 /* No EOI cycle used here */
83
84 jmp doreti_iret
85
86 ISR_VEC(1, apic_isr1)
87 ISR_VEC(2, apic_isr2)
88 ISR_VEC(3, apic_isr3)
89 ISR_VEC(4, apic_isr4)
90 ISR_VEC(5, apic_isr5)
91 ISR_VEC(6, apic_isr6)
92 ISR_VEC(7, apic_isr7)
93
94 /*
95 * Local APIC periodic timer handler.
96 */
97 .text
98 SUPERALIGN_TEXT
99 IDTVEC(timerint)
100 PUSH_FRAME
101 FAKE_MCOUNT(TF_RIP(%rsp))
102 movq %rsp, %rdi
103 call lapic_handle_timer
104 MEXITCOUNT
105 jmp doreti
106
107 /*
108 * Local APIC CMCI handler.
109 */
110 .text
111 SUPERALIGN_TEXT
112 IDTVEC(cmcint)
113 PUSH_FRAME
114 FAKE_MCOUNT(TF_RIP(%rsp))
115 call lapic_handle_cmc
116 MEXITCOUNT
117 jmp doreti
118
119 /*
120 * Local APIC error interrupt handler.
121 */
122 .text
123 SUPERALIGN_TEXT
124 IDTVEC(errorint)
125 PUSH_FRAME
126 FAKE_MCOUNT(TF_RIP(%rsp))
127 call lapic_handle_error
128 MEXITCOUNT
129 jmp doreti
130
131 #ifdef SMP
132 /*
133 * Global address space TLB shootdown.
134 */
135 .text
136 SUPERALIGN_TEXT
137 IDTVEC(invltlb)
138 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
139 PUSH_FRAME
140 movl PCPU(CPUID), %eax
141 #ifdef COUNT_XINVLTLB_HITS
142 incl xhits_gbl(,%rax,4)
143 #endif
144 #ifdef COUNT_IPIS
145 movq ipi_invltlb_counts(,%rax,8),%rax
146 incq (%rax)
147 #endif
148 POP_FRAME
149 #endif
150
151 pushq %rax
152
153 movq %cr3, %rax /* invalidate the TLB */
154 movq %rax, %cr3
155
156 movq lapic, %rax
157 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
158
159 lock
160 incl smp_tlb_wait
161
162 popq %rax
163 jmp doreti_iret
164
165 /*
166 * Single page TLB shootdown
167 */
168 .text
169 SUPERALIGN_TEXT
170 IDTVEC(invlpg)
171 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
172 PUSH_FRAME
173 movl PCPU(CPUID), %eax
174 #ifdef COUNT_XINVLTLB_HITS
175 incl xhits_pg(,%rax,4)
176 #endif
177 #ifdef COUNT_IPIS
178 movq ipi_invlpg_counts(,%rax,8),%rax
179 incq (%rax)
180 #endif
181 POP_FRAME
182 #endif
183
184 pushq %rax
185
186 movq smp_tlb_addr1, %rax
187 invlpg (%rax) /* invalidate single page */
188
189 movq lapic, %rax
190 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
191
192 lock
193 incl smp_tlb_wait
194
195 popq %rax
196 jmp doreti_iret
197
198 /*
199 * Page range TLB shootdown.
200 */
201 .text
202 SUPERALIGN_TEXT
203 IDTVEC(invlrng)
204 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
205 PUSH_FRAME
206 movl PCPU(CPUID), %eax
207 #ifdef COUNT_XINVLTLB_HITS
208 incl xhits_rng(,%rax,4)
209 #endif
210 #ifdef COUNT_IPIS
211 movq ipi_invlrng_counts(,%rax,8),%rax
212 incq (%rax)
213 #endif
214 POP_FRAME
215 #endif
216
217 pushq %rax
218 pushq %rdx
219
220 movq smp_tlb_addr1, %rdx
221 movq smp_tlb_addr2, %rax
222 1: invlpg (%rdx) /* invalidate single page */
223 addq $PAGE_SIZE, %rdx
224 cmpq %rax, %rdx
225 jb 1b
226
227 movq lapic, %rax
228 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
229
230 lock
231 incl smp_tlb_wait
232
233 popq %rdx
234 popq %rax
235 jmp doreti_iret
236
237 /*
238 * Invalidate cache.
239 */
240 .text
241 SUPERALIGN_TEXT
242 IDTVEC(invlcache)
243 #ifdef COUNT_IPIS
244 PUSH_FRAME
245 movl PCPU(CPUID), %eax
246 movq ipi_invlcache_counts(,%rax,8),%rax
247 incq (%rax)
248 POP_FRAME
249 #endif
250
251 pushq %rax
252
253 wbinvd
254
255 movq lapic, %rax
256 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
257
258 lock
259 incl smp_tlb_wait
260
261 popq %rax
262 jmp doreti_iret
263
264 /*
265 * Handler for IPIs sent via the per-cpu IPI bitmap.
266 */
267 .text
268 SUPERALIGN_TEXT
269 IDTVEC(ipi_intr_bitmap_handler)
270 PUSH_FRAME
271
272 movq lapic, %rdx
273 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
274
275 FAKE_MCOUNT(TF_RIP(%rsp))
276
277 call ipi_bitmap_handler
278 MEXITCOUNT
279 jmp doreti
280
281 /*
282 * Executed by a CPU when it receives an IPI_STOP from another CPU.
283 */
284 .text
285 SUPERALIGN_TEXT
286 IDTVEC(cpustop)
287 PUSH_FRAME
288
289 movq lapic, %rax
290 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
291
292 call cpustop_handler
293 jmp doreti
294
295 /*
296 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
297 */
298 .text
299 SUPERALIGN_TEXT
300 IDTVEC(cpususpend)
301 PUSH_FRAME
302
303 movq lapic, %rax
304 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
305
306 call cpususpend_handler
307
308 POP_FRAME
309 jmp doreti_iret
310
311 /*
312 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
313 *
314 * - Calls the generic rendezvous action function.
315 */
316 .text
317 SUPERALIGN_TEXT
318 IDTVEC(rendezvous)
319 PUSH_FRAME
320 #ifdef COUNT_IPIS
321 movl PCPU(CPUID), %eax
322 movq ipi_rendezvous_counts(,%rax,8), %rax
323 incq (%rax)
324 #endif
325 call smp_rendezvous_action
326 movq lapic, %rax
327 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
328 jmp doreti
329 #endif /* SMP */
Cache object: fc5edc0be44d38ae9b02106ad7e74c38
|