1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD: releng/6.4/sys/amd64/amd64/apic_vector.S 169216 2007-05-02 15:40:15Z jhb $
32 */
33
34 /*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39 #include <machine/asmacros.h>
40 #include <machine/apicreg.h>
41
42 #include "assym.s"
43
44 /*
45 * Macros to create and destroy a trap frame.
46 */
47 #define PUSH_FRAME \
48 subq $TF_RIP,%rsp ; /* skip dummy tf_err and tf_trapno */ \
49 testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
50 jz 1f ; /* Yes, dont swapgs again */ \
51 swapgs ; \
52 1: movq %rdi,TF_RDI(%rsp) ; \
53 movq %rsi,TF_RSI(%rsp) ; \
54 movq %rdx,TF_RDX(%rsp) ; \
55 movq %rcx,TF_RCX(%rsp) ; \
56 movq %r8,TF_R8(%rsp) ; \
57 movq %r9,TF_R9(%rsp) ; \
58 movq %rax,TF_RAX(%rsp) ; \
59 movq %rbx,TF_RBX(%rsp) ; \
60 movq %rbp,TF_RBP(%rsp) ; \
61 movq %r10,TF_R10(%rsp) ; \
62 movq %r11,TF_R11(%rsp) ; \
63 movq %r12,TF_R12(%rsp) ; \
64 movq %r13,TF_R13(%rsp) ; \
65 movq %r14,TF_R14(%rsp) ; \
66 movq %r15,TF_R15(%rsp)
67
68 #define POP_FRAME \
69 movq TF_RDI(%rsp),%rdi ; \
70 movq TF_RSI(%rsp),%rsi ; \
71 movq TF_RDX(%rsp),%rdx ; \
72 movq TF_RCX(%rsp),%rcx ; \
73 movq TF_R8(%rsp),%r8 ; \
74 movq TF_R9(%rsp),%r9 ; \
75 movq TF_RAX(%rsp),%rax ; \
76 movq TF_RBX(%rsp),%rbx ; \
77 movq TF_RBP(%rsp),%rbp ; \
78 movq TF_R10(%rsp),%r10 ; \
79 movq TF_R11(%rsp),%r11 ; \
80 movq TF_R12(%rsp),%r12 ; \
81 movq TF_R13(%rsp),%r13 ; \
82 movq TF_R14(%rsp),%r14 ; \
83 movq TF_R15(%rsp),%r15 ; \
84 testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
85 jz 1f ; /* keep kernel GS.base */ \
86 cli ; \
87 swapgs ; \
88 1: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */
89
90
91 /*
92 * I/O Interrupt Entry Point. Rather than having one entry point for
93 * each interrupt source, we use one entry point for each 32-bit word
94 * in the ISR. The handler determines the highest bit set in the ISR,
95 * translates that into a vector, and passes the vector to the
96 * lapic_handle_intr() function.
97 */
98 #define ISR_VEC(index, vec_name) \
99 .text ; \
100 SUPERALIGN_TEXT ; \
101 IDTVEC(vec_name) ; \
102 PUSH_FRAME ; \
103 FAKE_MCOUNT(TF_RIP(%rsp)) ; \
104 movq lapic, %rdx ; /* pointer to local APIC */ \
105 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \
106 bsrl %eax, %eax ; /* index of highset set bit in ISR */ \
107 jz 2f ; \
108 addl $(32 * index),%eax ; \
109 1: ; \
110 movq %rax, %rdi ; /* pass the IRQ */ \
111 call lapic_handle_intr ; \
112 MEXITCOUNT ; \
113 jmp doreti ; \
114 2: movl $-1, %eax ; /* send a vector of -1 */ \
115 jmp 1b
116
117 /*
118 * Handle "spurious INTerrupts".
119 * Notes:
120 * This is different than the "spurious INTerrupt" generated by an
121 * 8259 PIC for missing INTs. See the APIC documentation for details.
122 * This routine should NOT do an 'EOI' cycle.
123 */
124 .text
125 SUPERALIGN_TEXT
126 IDTVEC(spuriousint)
127
128 /* No EOI cycle used here */
129
130 iretq
131
132 ISR_VEC(1, apic_isr1)
133 ISR_VEC(2, apic_isr2)
134 ISR_VEC(3, apic_isr3)
135 ISR_VEC(4, apic_isr4)
136 ISR_VEC(5, apic_isr5)
137 ISR_VEC(6, apic_isr6)
138 ISR_VEC(7, apic_isr7)
139
140 /*
141 * Local APIC periodic timer handler.
142 */
143 .text
144 SUPERALIGN_TEXT
145 IDTVEC(timerint)
146 PUSH_FRAME
147
148 movq lapic, %rdx
149 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
150
151 FAKE_MCOUNT(TF_RIP(%rsp))
152
153 call lapic_handle_timer
154 MEXITCOUNT
155 jmp doreti
156
157 #ifdef SMP
158 /*
159 * Global address space TLB shootdown.
160 */
161 .text
162 SUPERALIGN_TEXT
163 IDTVEC(invltlb)
164 pushq %rax
165
166 movq %cr3, %rax /* invalidate the TLB */
167 movq %rax, %cr3
168
169 movq lapic, %rax
170 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
171
172 lock
173 incl smp_tlb_wait
174
175 popq %rax
176 iretq
177
178 /*
179 * Single page TLB shootdown
180 */
181 .text
182 SUPERALIGN_TEXT
183 IDTVEC(invlpg)
184 pushq %rax
185
186 movq smp_tlb_addr1, %rax
187 invlpg (%rax) /* invalidate single page */
188
189 movq lapic, %rax
190 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
191
192 lock
193 incl smp_tlb_wait
194
195 popq %rax
196 iretq
197
198 /*
199 * Page range TLB shootdown.
200 */
201 .text
202 SUPERALIGN_TEXT
203 IDTVEC(invlrng)
204 pushq %rax
205 pushq %rdx
206
207 movq smp_tlb_addr1, %rdx
208 movq smp_tlb_addr2, %rax
209 1: invlpg (%rdx) /* invalidate single page */
210 addq $PAGE_SIZE, %rdx
211 cmpq %rax, %rdx
212 jb 1b
213
214 movq lapic, %rax
215 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
216
217 lock
218 incl smp_tlb_wait
219
220 popq %rdx
221 popq %rax
222 iretq
223
224 /*
225 * Invalidate cache.
226 */
227 .text
228 SUPERALIGN_TEXT
229 IDTVEC(invlcache)
230 pushq %rax
231
232 wbinvd
233
234 movq lapic, %rax
235 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
236
237 lock
238 incl smp_tlb_wait
239
240 popq %rax
241 iretq
242
243 /*
244 * Handler for IPIs sent via the per-cpu IPI bitmap.
245 */
246 .text
247 SUPERALIGN_TEXT
248 IDTVEC(ipi_intr_bitmap_handler)
249
250 PUSH_FRAME
251
252 movq lapic, %rdx
253 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
254
255 FAKE_MCOUNT(TF_RIP(%rsp))
256
257 call ipi_bitmap_handler
258 MEXITCOUNT
259 jmp doreti
260
261 /*
262 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
263 *
264 * The other CPU has already executed aston() or need_resched() on our
265 * current process, so we simply need to ack the interrupt and return
266 * via doreti to run ast().
267 */
268
269 .text
270 SUPERALIGN_TEXT
271 IDTVEC(cpuast)
272 PUSH_FRAME
273
274 movq lapic, %rdx
275 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
276
277 FAKE_MCOUNT(TF_RIP(%rsp))
278
279 MEXITCOUNT
280 jmp doreti
281
282 /*
283 * Executed by a CPU when it receives an IPI_STOP from another CPU.
284 */
285 .text
286 SUPERALIGN_TEXT
287 IDTVEC(cpustop)
288 PUSH_FRAME
289
290 movq lapic, %rax
291 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
292
293 movl PCPU(CPUID), %eax
294 imull $PCB_SIZE, %eax
295 leaq stoppcbs(%rax), %rdi
296 call savectx /* Save process context */
297
298 movl PCPU(CPUID), %eax
299
300 lock
301 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
302 1:
303 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
304 jnc 1b
305
306 lock
307 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
308 lock
309 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
310
311 test %eax, %eax
312 jnz 2f
313
314 movq cpustop_restartfunc, %rax
315 testq %rax, %rax
316 jz 2f
317 movq $0, cpustop_restartfunc /* One-shot */
318
319 call *%rax
320 2:
321 POP_FRAME
322 iretq
323
324 /*
325 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
326 *
327 * - Calls the generic rendezvous action function.
328 */
329 .text
330 SUPERALIGN_TEXT
331 IDTVEC(rendezvous)
332 PUSH_FRAME
333 call smp_rendezvous_action
334 movq lapic, %rax
335 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
336 POP_FRAME /* Why not doreti? */
337 iretq
338 #endif /* SMP */
Cache object: cec3056259291837666da9f1a92363ac
|