1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD$
32 */
33
34 /*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39 #include <machine/asmacros.h>
40 #include <machine/apicreg.h>
41 #include <machine/smptests.h>
42
43 #include "assym.s"
44
45 /*
46 * Macros to create and destroy a trap frame.
47 */
48 #define PUSH_FRAME \
49 pushl $0 ; /* dummy error code */ \
50 pushl $0 ; /* dummy trap type */ \
51 pushal ; /* 8 ints */ \
52 pushl %ds ; /* save data and extra segments ... */ \
53 pushl %es ; \
54 pushl %fs
55
56 #define POP_FRAME \
57 popl %fs ; \
58 popl %es ; \
59 popl %ds ; \
60 popal ; \
61 addl $4+4,%esp
62
63 /*
64 * I/O Interrupt Entry Point. Rather than having one entry point for
65 * each interrupt source, we use one entry point for each 32-bit word
66 * in the ISR. The handler determines the highest bit set in the ISR,
67 * translates that into a vector, and passes the vector to the
68 * lapic_handle_intr() function.
69 */
70 #define ISR_VEC(index, vec_name) \
71 .text ; \
72 SUPERALIGN_TEXT ; \
73 IDTVEC(vec_name) ; \
74 PUSH_FRAME ; \
75 movl $KDSEL, %eax ; /* reload with kernel's data segment */ \
76 movl %eax, %ds ; \
77 movl %eax, %es ; \
78 movl $KPSEL, %eax ; /* reload with per-CPU data segment */ \
79 movl %eax, %fs ; \
80 FAKE_MCOUNT(TF_EIP(%esp)) ; \
81 movl lapic, %edx ; /* pointer to local APIC */ \
82 movl LA_ISR + 16 * (index)(%edx), %eax ; /* load ISR */ \
83 bsrl %eax, %eax ; /* index of highset set bit in ISR */ \
84 jz 2f ; \
85 addl $(32 * index),%eax ; \
86 1: ; \
87 pushl %eax ; /* pass the IRQ */ \
88 call lapic_handle_intr ; \
89 addl $4, %esp ; /* discard parameter */ \
90 MEXITCOUNT ; \
91 jmp doreti ; \
92 2: movl $-1, %eax ; /* send a vector of -1 */ \
93 jmp 1b
94
95 /*
96 * Handle "spurious INTerrupts".
97 * Notes:
98 * This is different than the "spurious INTerrupt" generated by an
99 * 8259 PIC for missing INTs. See the APIC documentation for details.
100 * This routine should NOT do an 'EOI' cycle.
101 */
102 .text
103 SUPERALIGN_TEXT
104 IDTVEC(spuriousint)
105
106 /* No EOI cycle used here */
107
108 iret
109
110 ISR_VEC(1, apic_isr1)
111 ISR_VEC(2, apic_isr2)
112 ISR_VEC(3, apic_isr3)
113 ISR_VEC(4, apic_isr4)
114 ISR_VEC(5, apic_isr5)
115 ISR_VEC(6, apic_isr6)
116 ISR_VEC(7, apic_isr7)
117
118 /*
119 * Local APIC periodic timer handler.
120 */
121 .text
122 SUPERALIGN_TEXT
123 IDTVEC(timerint)
124 PUSH_FRAME
125 movl $KDSEL, %eax /* reload with kernel's data segment */
126 movl %eax, %ds
127 movl %eax, %es
128 movl $KPSEL, %eax
129 movl %eax, %fs
130
131 movl lapic, %edx
132 movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */
133
134 FAKE_MCOUNT(TF_EIP(%esp))
135
136 pushl $0 /* XXX convert trapframe to clockframe */
137 call lapic_handle_timer
138 addl $4, %esp /* XXX convert clockframe to trapframe */
139 MEXITCOUNT
140 jmp doreti
141
142 #ifdef SMP
143 /*
144 * Global address space TLB shootdown.
145 */
146 .text
147 SUPERALIGN_TEXT
148 IDTVEC(invltlb)
149 pushl %eax
150 pushl %ds
151 movl $KDSEL, %eax /* Kernel data selector */
152 movl %eax, %ds
153
154 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
155 pushl %fs
156 movl $KPSEL, %eax /* Private space selector */
157 movl %eax, %fs
158 movl PCPU(CPUID), %eax
159 popl %fs
160 #ifdef COUNT_XINVLTLB_HITS
161 incl xhits_gbl(,%eax,4)
162 #endif
163 #ifdef COUNT_IPIS
164 movl ipi_invltlb_counts(,%eax,4),%eax
165 incl (%eax)
166 #endif
167 #endif
168
169 movl %cr3, %eax /* invalidate the TLB */
170 movl %eax, %cr3
171
172 movl lapic, %eax
173 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
174
175 lock
176 incl smp_tlb_wait
177
178 popl %ds
179 popl %eax
180 iret
181
182 /*
183 * Single page TLB shootdown
184 */
185 .text
186 SUPERALIGN_TEXT
187 IDTVEC(invlpg)
188 pushl %eax
189 pushl %ds
190 movl $KDSEL, %eax /* Kernel data selector */
191 movl %eax, %ds
192
193 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
194 pushl %fs
195 movl $KPSEL, %eax /* Private space selector */
196 movl %eax, %fs
197 movl PCPU(CPUID), %eax
198 popl %fs
199 #ifdef COUNT_XINVLTLB_HITS
200 incl xhits_pg(,%eax,4)
201 #endif
202 #ifdef COUNT_IPIS
203 movl ipi_invlpg_counts(,%eax,4),%eax
204 incl (%eax)
205 #endif
206 #endif
207
208 movl smp_tlb_addr1, %eax
209 invlpg (%eax) /* invalidate single page */
210
211 movl lapic, %eax
212 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
213
214 lock
215 incl smp_tlb_wait
216
217 popl %ds
218 popl %eax
219 iret
220
221 /*
222 * Page range TLB shootdown.
223 */
224 .text
225 SUPERALIGN_TEXT
226 IDTVEC(invlrng)
227 pushl %eax
228 pushl %edx
229 pushl %ds
230 movl $KDSEL, %eax /* Kernel data selector */
231 movl %eax, %ds
232
233 #if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
234 pushl %fs
235 movl $KPSEL, %eax /* Private space selector */
236 movl %eax, %fs
237 movl PCPU(CPUID), %eax
238 popl %fs
239 #ifdef COUNT_XINVLTLB_HITS
240 incl xhits_rng(,%eax,4)
241 #endif
242 #ifdef COUNT_IPIS
243 movl ipi_invlrng_counts(,%eax,4),%eax
244 incl (%eax)
245 #endif
246 #endif
247
248 movl smp_tlb_addr1, %edx
249 movl smp_tlb_addr2, %eax
250 1: invlpg (%edx) /* invalidate single page */
251 addl $PAGE_SIZE, %edx
252 cmpl %eax, %edx
253 jb 1b
254
255 movl lapic, %eax
256 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
257
258 lock
259 incl smp_tlb_wait
260
261 popl %ds
262 popl %edx
263 popl %eax
264 iret
265
266 /*
267 * Invalidate cache.
268 */
269 .text
270 SUPERALIGN_TEXT
271 IDTVEC(invlcache)
272 pushl %eax
273 pushl %ds
274 movl $KDSEL, %eax /* Kernel data selector */
275 movl %eax, %ds
276
277 #ifdef COUNT_IPIS
278 pushl %fs
279 movl $KPSEL, %eax /* Private space selector */
280 movl %eax, %fs
281 movl PCPU(CPUID), %eax
282 popl %fs
283 movl ipi_invlcache_counts(,%eax,4),%eax
284 incl (%eax)
285 #endif
286
287 wbinvd
288
289 movl lapic, %eax
290 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
291
292 lock
293 incl smp_tlb_wait
294
295 popl %ds
296 popl %eax
297 iret
298
299 /*
300 * Handler for IPIs sent via the per-cpu IPI bitmap.
301 */
302 .text
303 SUPERALIGN_TEXT
304 IDTVEC(ipi_intr_bitmap_handler)
305
306 PUSH_FRAME
307 movl $KDSEL, %eax /* reload with kernel's data segment */
308 movl %eax, %ds
309 movl %eax, %es
310 movl $KPSEL, %eax
311 movl %eax, %fs
312
313 movl lapic, %edx
314 movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */
315
316 FAKE_MCOUNT(TF_EIP(%esp))
317
318 pushl $0 /* XXX convert trapframe to clockframe */
319 call ipi_bitmap_handler
320 addl $4, %esp /* XXX convert clockframe to trapframe */
321 MEXITCOUNT
322 jmp doreti
323
324 /*
325 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
326 *
327 * - Signals its receipt.
328 * - Waits for permission to restart.
329 * - Signals its restart.
330 */
331 .text
332 SUPERALIGN_TEXT
333 IDTVEC(cpustop)
334 pushl %ebp
335 movl %esp, %ebp
336 pushl %eax
337 pushl %ecx
338 pushl %edx
339 pushl %ds /* save current data segment */
340 pushl %es
341 pushl %fs
342
343 movl $KDSEL, %eax
344 movl %eax, %ds /* use KERNEL data segment */
345 movl %eax, %es
346 movl $KPSEL, %eax
347 movl %eax, %fs
348
349 movl lapic, %eax
350 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
351
352 movl PCPU(CPUID), %eax
353 imull $PCB_SIZE, %eax
354 leal CNAME(stoppcbs)(%eax), %eax
355 pushl %eax
356 call CNAME(savectx) /* Save process context */
357 addl $4, %esp
358
359 movl PCPU(CPUID), %eax
360
361 lock
362 btsl %eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
363 1:
364 btl %eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
365 jnc 1b
366
367 lock
368 btrl %eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
369 lock
370 btrl %eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
371
372 test %eax, %eax
373 jnz 2f
374
375 movl CNAME(cpustop_restartfunc), %eax
376 test %eax, %eax
377 jz 2f
378 movl $0, CNAME(cpustop_restartfunc) /* One-shot */
379
380 call *%eax
381 2:
382 popl %fs
383 popl %es
384 popl %ds /* restore previous data segment */
385 popl %edx
386 popl %ecx
387 popl %eax
388 movl %ebp, %esp
389 popl %ebp
390 iret
391
392 /*
393 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
394 *
395 * - Calls the generic rendezvous action function.
396 */
397 .text
398 SUPERALIGN_TEXT
399 IDTVEC(rendezvous)
400 PUSH_FRAME
401 movl $KDSEL, %eax
402 movl %eax, %ds /* use KERNEL data segment */
403 movl %eax, %es
404 movl $KPSEL, %eax
405 movl %eax, %fs
406
407 #ifdef COUNT_IPIS
408 movl PCPU(CPUID), %eax
409 movl ipi_rendezvous_counts(,%eax,4), %eax
410 incl (%eax)
411 #endif
412 call smp_rendezvous_action
413
414 movl lapic, %eax
415 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
416 POP_FRAME
417 iret
418
419 /*
420 * Clean up when we lose out on the lazy context switch optimization.
421 * ie: when we are about to release a PTD but a cpu is still borrowing it.
422 */
423 SUPERALIGN_TEXT
424 IDTVEC(lazypmap)
425 PUSH_FRAME
426 movl $KDSEL, %eax
427 movl %eax, %ds /* use KERNEL data segment */
428 movl %eax, %es
429 movl $KPSEL, %eax
430 movl %eax, %fs
431
432 #ifdef COUNT_IPIS
433 movl PCPU(CPUID), %eax
434 movl ipi_lazypmap_counts(,%eax,4), %eax
435 incl (%eax)
436 #endif
437 call pmap_lazyfix_action
438
439 movl lapic, %eax
440 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
441 POP_FRAME
442 iret
443 #endif /* SMP */
Cache object: d271e8f6670a224e9bd63663606b7ae5
|