1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD$
32 */
33
34 /*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39 #include "opt_smp.h"
40
41 #include <machine/asmacros.h>
42 #include <machine/apicreg.h>
43
44 #include "assym.s"
45
46 /*
47 * I/O Interrupt Entry Point. Rather than having one entry point for
48 * each interrupt source, we use one entry point for each 32-bit word
49 * in the ISR. The handler determines the highest bit set in the ISR,
50 * translates that into a vector, and passes the vector to the
51 * lapic_handle_intr() function.
52 */
53 #define ISR_VEC(index, vec_name) \
54 .text ; \
55 SUPERALIGN_TEXT ; \
56 IDTVEC(vec_name) ; \
57 PUSH_FRAME ; \
58 SET_KERNEL_SREGS ; \
59 cld ; \
60 FAKE_MCOUNT(TF_EIP(%esp)) ; \
61 movl lapic, %edx ; /* pointer to local APIC */ \
62 movl LA_ISR + 16 * (index)(%edx), %eax ; /* load ISR */ \
63 bsrl %eax, %eax ; /* index of highset set bit in ISR */ \
64 jz 2f ; \
65 addl $(32 * index),%eax ; \
66 1: ; \
67 pushl %esp ; \
68 pushl %eax ; /* pass the IRQ */ \
69 call lapic_handle_intr ; \
70 addl $8, %esp ; /* discard parameter */ \
71 MEXITCOUNT ; \
72 jmp doreti ; \
73 2: movl $-1, %eax ; /* send a vector of -1 */ \
74 jmp 1b
75
76 /*
77 * Handle "spurious INTerrupts".
78 * Notes:
79 * This is different than the "spurious INTerrupt" generated by an
80 * 8259 PIC for missing INTs. See the APIC documentation for details.
81 * This routine should NOT do an 'EOI' cycle.
82 */
83 .text
84 SUPERALIGN_TEXT
85 IDTVEC(spuriousint)
86
87 /* No EOI cycle used here */
88
89 iret
90
91 ISR_VEC(1, apic_isr1)
92 ISR_VEC(2, apic_isr2)
93 ISR_VEC(3, apic_isr3)
94 ISR_VEC(4, apic_isr4)
95 ISR_VEC(5, apic_isr5)
96 ISR_VEC(6, apic_isr6)
97 ISR_VEC(7, apic_isr7)
98
99 /*
100 * Local APIC periodic timer handler.
101 */
102 .text
103 SUPERALIGN_TEXT
104 IDTVEC(timerint)
105 PUSH_FRAME
106 SET_KERNEL_SREGS
107 cld
108 FAKE_MCOUNT(TF_EIP(%esp))
109 pushl %esp
110 call lapic_handle_timer
111 add $4, %esp
112 MEXITCOUNT
113 jmp doreti
114
115 /*
116 * Local APIC CMCI handler.
117 */
118 .text
119 SUPERALIGN_TEXT
120 IDTVEC(cmcint)
121 PUSH_FRAME
122 SET_KERNEL_SREGS
123 cld
124 FAKE_MCOUNT(TF_EIP(%esp))
125 call lapic_handle_cmc
126 MEXITCOUNT
127 jmp doreti
128
129 /*
130 * Local APIC error interrupt handler.
131 */
132 .text
133 SUPERALIGN_TEXT
134 IDTVEC(errorint)
135 PUSH_FRAME
136 SET_KERNEL_SREGS
137 cld
138 FAKE_MCOUNT(TF_EIP(%esp))
139 call lapic_handle_error
140 MEXITCOUNT
141 jmp doreti
142
143 #ifdef SMP
144 /*
145 * Global address space TLB shootdown.
146 */
147 .text
148 SUPERALIGN_TEXT
149 IDTVEC(invltlb)
150 pushl %eax
151 pushl %ds
152 movl $KDSEL, %eax /* Kernel data selector */
153 movl %eax, %ds
154
155 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
156 pushl %fs
157 movl $KPSEL, %eax /* Private space selector */
158 movl %eax, %fs
159 movl PCPU(CPUID), %eax
160 popl %fs
161 #ifdef COUNT_XINVLTLB_HITS
162 incl xhits_gbl(,%eax,4)
163 #endif
164 #ifdef COUNT_IPIS
165 movl ipi_invltlb_counts(,%eax,4),%eax
166 incl (%eax)
167 #endif
168 #endif
169
170 movl %cr3, %eax /* invalidate the TLB */
171 movl %eax, %cr3
172
173 movl lapic, %eax
174 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
175
176 lock
177 incl smp_tlb_wait
178
179 popl %ds
180 popl %eax
181 iret
182
183 /*
184 * Single page TLB shootdown
185 */
186 .text
187 SUPERALIGN_TEXT
188 IDTVEC(invlpg)
189 pushl %eax
190 pushl %ds
191 movl $KDSEL, %eax /* Kernel data selector */
192 movl %eax, %ds
193
194 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
195 pushl %fs
196 movl $KPSEL, %eax /* Private space selector */
197 movl %eax, %fs
198 movl PCPU(CPUID), %eax
199 popl %fs
200 #ifdef COUNT_XINVLTLB_HITS
201 incl xhits_pg(,%eax,4)
202 #endif
203 #ifdef COUNT_IPIS
204 movl ipi_invlpg_counts(,%eax,4),%eax
205 incl (%eax)
206 #endif
207 #endif
208
209 movl smp_tlb_addr1, %eax
210 invlpg (%eax) /* invalidate single page */
211
212 movl lapic, %eax
213 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
214
215 lock
216 incl smp_tlb_wait
217
218 popl %ds
219 popl %eax
220 iret
221
222 /*
223 * Page range TLB shootdown.
224 */
225 .text
226 SUPERALIGN_TEXT
227 IDTVEC(invlrng)
228 pushl %eax
229 pushl %edx
230 pushl %ds
231 movl $KDSEL, %eax /* Kernel data selector */
232 movl %eax, %ds
233
234 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
235 pushl %fs
236 movl $KPSEL, %eax /* Private space selector */
237 movl %eax, %fs
238 movl PCPU(CPUID), %eax
239 popl %fs
240 #ifdef COUNT_XINVLTLB_HITS
241 incl xhits_rng(,%eax,4)
242 #endif
243 #ifdef COUNT_IPIS
244 movl ipi_invlrng_counts(,%eax,4),%eax
245 incl (%eax)
246 #endif
247 #endif
248
249 movl smp_tlb_addr1, %edx
250 movl smp_tlb_addr2, %eax
251 1: invlpg (%edx) /* invalidate single page */
252 addl $PAGE_SIZE, %edx
253 cmpl %eax, %edx
254 jb 1b
255
256 movl lapic, %eax
257 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
258
259 lock
260 incl smp_tlb_wait
261
262 popl %ds
263 popl %edx
264 popl %eax
265 iret
266
267 /*
268 * Invalidate cache.
269 */
270 .text
271 SUPERALIGN_TEXT
272 IDTVEC(invlcache)
273 pushl %eax
274 pushl %ds
275 movl $KDSEL, %eax /* Kernel data selector */
276 movl %eax, %ds
277
278 #ifdef COUNT_IPIS
279 pushl %fs
280 movl $KPSEL, %eax /* Private space selector */
281 movl %eax, %fs
282 movl PCPU(CPUID), %eax
283 popl %fs
284 movl ipi_invlcache_counts(,%eax,4),%eax
285 incl (%eax)
286 #endif
287
288 wbinvd
289
290 movl lapic, %eax
291 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
292
293 lock
294 incl smp_tlb_wait
295
296 popl %ds
297 popl %eax
298 iret
299
300 /*
301 * Handler for IPIs sent via the per-cpu IPI bitmap.
302 */
303 #ifndef XEN
304 .text
305 SUPERALIGN_TEXT
306 IDTVEC(ipi_intr_bitmap_handler)
307 PUSH_FRAME
308 SET_KERNEL_SREGS
309 cld
310
311 movl lapic, %edx
312 movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */
313
314 FAKE_MCOUNT(TF_EIP(%esp))
315
316 call ipi_bitmap_handler
317 MEXITCOUNT
318 jmp doreti
319 #endif
320 /*
321 * Executed by a CPU when it receives an IPI_STOP from another CPU.
322 */
323 .text
324 SUPERALIGN_TEXT
325 IDTVEC(cpustop)
326 PUSH_FRAME
327 SET_KERNEL_SREGS
328 cld
329
330 movl lapic, %eax
331 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
332
333 call cpustop_handler
334
335 POP_FRAME
336 iret
337
338 /*
339 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
340 *
341 * - Calls the generic rendezvous action function.
342 */
343 .text
344 SUPERALIGN_TEXT
345 IDTVEC(rendezvous)
346 PUSH_FRAME
347 SET_KERNEL_SREGS
348 cld
349
350 #ifdef COUNT_IPIS
351 movl PCPU(CPUID), %eax
352 movl ipi_rendezvous_counts(,%eax,4), %eax
353 incl (%eax)
354 #endif
355 call smp_rendezvous_action
356
357 movl lapic, %eax
358 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
359 POP_FRAME
360 iret
361
362 /*
363 * Clean up when we lose out on the lazy context switch optimization.
364 * ie: when we are about to release a PTD but a cpu is still borrowing it.
365 */
366 SUPERALIGN_TEXT
367 IDTVEC(lazypmap)
368 PUSH_FRAME
369 SET_KERNEL_SREGS
370 cld
371
372 call pmap_lazyfix_action
373
374 movl lapic, %eax
375 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
376 POP_FRAME
377 iret
378 #endif /* SMP */
Cache object: cdf216af4f73eb6c1e12c323b33c4890
|