1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. All advertising materials mentioning features or use of this software
15 * must display the following acknowledgement:
16 * This product includes software developed by the University of
17 * California, Berkeley and its contributors.
18 * 4. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * from: vector.s, 386BSD 0.1 unknown origin
35 * $FreeBSD: releng/5.2/sys/amd64/amd64/apic_vector.S 123180 2003-12-06 23:19:47Z peter $
36 */
37
38 /*
39 * Interrupt entry points for external interrupts triggered by I/O APICs
40 * as well as IPI handlers.
41 */
42
43 #include <machine/asmacros.h>
44 #include <machine/apicreg.h>
45
46 #include "assym.s"
47
48 /*
49 * Macros to create and destroy a trap frame.
50 */
51 #define PUSH_FRAME \
52 subq $TF_RIP,%rsp ; /* skip dummy tf_err and tf_trapno */ \
53 testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
54 jz 1f ; /* Yes, dont swapgs again */ \
55 swapgs ; \
56 1: movq %rdi,TF_RDI(%rsp) ; \
57 movq %rsi,TF_RSI(%rsp) ; \
58 movq %rdx,TF_RDX(%rsp) ; \
59 movq %rcx,TF_RCX(%rsp) ; \
60 movq %r8,TF_R8(%rsp) ; \
61 movq %r9,TF_R9(%rsp) ; \
62 movq %rax,TF_RAX(%rsp) ; \
63 movq %rbx,TF_RBX(%rsp) ; \
64 movq %rbp,TF_RBP(%rsp) ; \
65 movq %r10,TF_R10(%rsp) ; \
66 movq %r11,TF_R11(%rsp) ; \
67 movq %r12,TF_R12(%rsp) ; \
68 movq %r13,TF_R13(%rsp) ; \
69 movq %r14,TF_R14(%rsp) ; \
70 movq %r15,TF_R15(%rsp)
71
72 #define POP_FRAME \
73 movq TF_RDI(%rsp),%rdi ; \
74 movq TF_RSI(%rsp),%rsi ; \
75 movq TF_RDX(%rsp),%rdx ; \
76 movq TF_RCX(%rsp),%rcx ; \
77 movq TF_R8(%rsp),%r8 ; \
78 movq TF_R9(%rsp),%r9 ; \
79 movq TF_RAX(%rsp),%rax ; \
80 movq TF_RBX(%rsp),%rbx ; \
81 movq TF_RBP(%rsp),%rbp ; \
82 movq TF_R10(%rsp),%r10 ; \
83 movq TF_R11(%rsp),%r11 ; \
84 movq TF_R12(%rsp),%r12 ; \
85 movq TF_R13(%rsp),%r13 ; \
86 movq TF_R14(%rsp),%r14 ; \
87 movq TF_R15(%rsp),%r15 ; \
88 testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
89 jz 1f ; /* keep kernel GS.base */ \
90 cli ; \
91 swapgs ; \
92 1: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */
93
94
95 /*
96 * I/O Interrupt Entry Point. Rather than having one entry point for
97 * each interrupt source, we use one entry point for each 32-bit word
98 * in the ISR. The handler determines the highest bit set in the ISR,
99 * translates that into a vector, and passes the vector to the
100 * lapic_handle_intr() function.
101 */
102 #define ISR_VEC(index, vec_name) \
103 .text ; \
104 SUPERALIGN_TEXT ; \
105 IDTVEC(vec_name) ; \
106 PUSH_FRAME ; \
107 FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid double count */ \
108 movq lapic, %rdx ; /* pointer to local APIC */ \
109 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \
110 bsrl %eax, %eax ; /* index of highset set bit in ISR */ \
111 jz 2f ; \
112 addl $(32 * index),%eax ; \
113 1: ; \
114 movq %rax, %rdi ; /* pass the IRQ */ \
115 call lapic_handle_intr ; \
116 MEXITCOUNT ; \
117 jmp doreti ; \
118 2: movl $-1, %eax ; /* send a vector of -1 */ \
119 jmp 1b
120
121 /*
122 * Handle "spurious INTerrupts".
123 * Notes:
124 * This is different than the "spurious INTerrupt" generated by an
125 * 8259 PIC for missing INTs. See the APIC documentation for details.
126 * This routine should NOT do an 'EOI' cycle.
127 */
128 .text
129 SUPERALIGN_TEXT
130 IDTVEC(spuriousint)
131
132 /* No EOI cycle used here */
133
134 iretq
135
136 MCOUNT_LABEL(bintr2)
137 ISR_VEC(1, apic_isr1)
138 ISR_VEC(2, apic_isr2)
139 ISR_VEC(3, apic_isr3)
140 ISR_VEC(4, apic_isr4)
141 ISR_VEC(5, apic_isr5)
142 ISR_VEC(6, apic_isr6)
143 ISR_VEC(7, apic_isr7)
144 MCOUNT_LABEL(eintr2)
145
146 #ifdef SMP
147 /*
148 * Global address space TLB shootdown.
149 */
150 .text
151 SUPERALIGN_TEXT
152 IDTVEC(invltlb)
153 pushq %rax
154
155 movq %cr3, %rax /* invalidate the TLB */
156 movq %rax, %cr3
157
158 movq lapic, %rax
159 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
160
161 lock
162 incl smp_tlb_wait
163
164 popq %rax
165 iretq
166
167 /*
168 * Single page TLB shootdown
169 */
170 .text
171 SUPERALIGN_TEXT
172 IDTVEC(invlpg)
173 pushq %rax
174
175 movq smp_tlb_addr1, %rax
176 invlpg (%rax) /* invalidate single page */
177
178 movq lapic, %rax
179 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
180
181 lock
182 incl smp_tlb_wait
183
184 popq %rax
185 iretq
186
187 /*
188 * Page range TLB shootdown.
189 */
190 .text
191 SUPERALIGN_TEXT
192 IDTVEC(invlrng)
193 pushq %rax
194 pushq %rdx
195
196 movq smp_tlb_addr1, %rdx
197 movq smp_tlb_addr2, %rax
198 1: invlpg (%rdx) /* invalidate single page */
199 addq $PAGE_SIZE, %rdx
200 cmpq %rax, %rdx
201 jb 1b
202
203 movq lapic, %rax
204 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
205
206 lock
207 incl smp_tlb_wait
208
209 popq %rdx
210 popq %rax
211 iretq
212
213 /*
214 * Forward hardclock to another CPU. Pushes a clockframe and calls
215 * forwarded_hardclock().
216 */
217 .text
218 SUPERALIGN_TEXT
219 IDTVEC(hardclock)
220 PUSH_FRAME
221
222 movq lapic, %rdx
223 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
224
225 call forwarded_hardclock
226 MEXITCOUNT
227 jmp doreti
228
229 /*
230 * Forward statclock to another CPU. Pushes a clockframe and calls
231 * forwarded_statclock().
232 */
233 .text
234 SUPERALIGN_TEXT
235 IDTVEC(statclock)
236 PUSH_FRAME
237
238 movq lapic, %rdx
239 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
240
241 FAKE_MCOUNT(13*4(%esp))
242
243 call forwarded_statclock
244 MEXITCOUNT
245 jmp doreti
246
247 /*
248 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
249 *
250 * The other CPU has already executed aston() or need_resched() on our
251 * current process, so we simply need to ack the interrupt and return
252 * via doreti to run ast().
253 */
254
255 .text
256 SUPERALIGN_TEXT
257 IDTVEC(cpuast)
258 PUSH_FRAME
259
260 movq lapic, %rdx
261 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
262
263 FAKE_MCOUNT(13*4(%esp))
264
265 MEXITCOUNT
266 jmp doreti
267
268 /*
269 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
270 *
271 * - Signals its receipt.
272 * - Waits for permission to restart.
273 * - Signals its restart.
274 */
275 .text
276 SUPERALIGN_TEXT
277 IDTVEC(cpustop)
278 PUSH_FRAME
279
280 movq lapic, %rax
281 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
282
283 movl PCPU(CPUID), %eax
284 imull $PCB_SIZE, %eax
285 leaq stoppcbs(%rax), %rdi
286 call savectx /* Save process context */
287
288 movl PCPU(CPUID), %eax
289
290 lock
291 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
292 1:
293 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
294 jnc 1b
295
296 lock
297 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
298 lock
299 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
300
301 test %eax, %eax
302 jnz 2f
303
304 movq cpustop_restartfunc, %rax
305 testq %rax, %rax
306 jz 2f
307 movq $0, cpustop_restartfunc /* One-shot */
308
309 call *%rax
310 2:
311 POP_FRAME
312 iretq
313
314 /*
315 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
316 *
317 * - Calls the generic rendezvous action function.
318 */
319 .text
320 SUPERALIGN_TEXT
321 IDTVEC(rendezvous)
322 PUSH_FRAME
323 call smp_rendezvous_action
324 movq lapic, %rax
325 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
326 POP_FRAME /* Why not doreti? */
327 iretq
328
329 #ifdef LAZY_SWITCH
330 /*
331 * Clean up when we lose out on the lazy context switch optimization.
332 * ie: when we are about to release a PTD but a cpu is still borrowing it.
333 */
334 SUPERALIGN_TEXT
335 IDTVEC(lazypmap)
336 PUSH_FRAME
337 call pmap_lazyfix_action
338 movq lapic, %rax
339 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
340 POP_FRAME /* Why not doreti? */
341 iretq
342 #endif
343 #endif /* SMP */
Cache object: a0a141f187c3be51b53ee0a7559075b7
|