1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD: releng/6.2/sys/amd64/amd64/apic_vector.S 147687 2005-06-30 05:33:26Z peter $
32 */
33
34 /*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39 #include <machine/asmacros.h>
40 #include <machine/apicreg.h>
41
42 #include "assym.s"
43
44 /*
45 * Macros to create and destroy a trap frame.
46 */
47 #define PUSH_FRAME \
48 subq $TF_RIP,%rsp ; /* skip dummy tf_err and tf_trapno */ \
49 testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
50 jz 1f ; /* Yes, dont swapgs again */ \
51 swapgs ; \
52 1: movq %rdi,TF_RDI(%rsp) ; \
53 movq %rsi,TF_RSI(%rsp) ; \
54 movq %rdx,TF_RDX(%rsp) ; \
55 movq %rcx,TF_RCX(%rsp) ; \
56 movq %r8,TF_R8(%rsp) ; \
57 movq %r9,TF_R9(%rsp) ; \
58 movq %rax,TF_RAX(%rsp) ; \
59 movq %rbx,TF_RBX(%rsp) ; \
60 movq %rbp,TF_RBP(%rsp) ; \
61 movq %r10,TF_R10(%rsp) ; \
62 movq %r11,TF_R11(%rsp) ; \
63 movq %r12,TF_R12(%rsp) ; \
64 movq %r13,TF_R13(%rsp) ; \
65 movq %r14,TF_R14(%rsp) ; \
66 movq %r15,TF_R15(%rsp)
67
68 #define POP_FRAME \
69 movq TF_RDI(%rsp),%rdi ; \
70 movq TF_RSI(%rsp),%rsi ; \
71 movq TF_RDX(%rsp),%rdx ; \
72 movq TF_RCX(%rsp),%rcx ; \
73 movq TF_R8(%rsp),%r8 ; \
74 movq TF_R9(%rsp),%r9 ; \
75 movq TF_RAX(%rsp),%rax ; \
76 movq TF_RBX(%rsp),%rbx ; \
77 movq TF_RBP(%rsp),%rbp ; \
78 movq TF_R10(%rsp),%r10 ; \
79 movq TF_R11(%rsp),%r11 ; \
80 movq TF_R12(%rsp),%r12 ; \
81 movq TF_R13(%rsp),%r13 ; \
82 movq TF_R14(%rsp),%r14 ; \
83 movq TF_R15(%rsp),%r15 ; \
84 testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
85 jz 1f ; /* keep kernel GS.base */ \
86 cli ; \
87 swapgs ; \
88 1: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */
89
90
91 /*
92 * I/O Interrupt Entry Point. Rather than having one entry point for
93 * each interrupt source, we use one entry point for each 32-bit word
94 * in the ISR. The handler determines the highest bit set in the ISR,
95 * translates that into a vector, and passes the vector to the
96 * lapic_handle_intr() function.
97 */
98 #define ISR_VEC(index, vec_name) \
99 .text ; \
100 SUPERALIGN_TEXT ; \
101 IDTVEC(vec_name) ; \
102 PUSH_FRAME ; \
103 FAKE_MCOUNT(TF_RIP(%rsp)) ; \
104 movq lapic, %rdx ; /* pointer to local APIC */ \
105 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \
106 bsrl %eax, %eax ; /* index of highset set bit in ISR */ \
107 jz 2f ; \
108 addl $(32 * index),%eax ; \
109 1: ; \
110 movq %rax, %rdi ; /* pass the IRQ */ \
111 call lapic_handle_intr ; \
112 MEXITCOUNT ; \
113 jmp doreti ; \
114 2: movl $-1, %eax ; /* send a vector of -1 */ \
115 jmp 1b
116
117 /*
118 * Handle "spurious INTerrupts".
119 * Notes:
120 * This is different than the "spurious INTerrupt" generated by an
121 * 8259 PIC for missing INTs. See the APIC documentation for details.
122 * This routine should NOT do an 'EOI' cycle.
123 */
124 .text
125 SUPERALIGN_TEXT
126 IDTVEC(spuriousint)
127
128 /* No EOI cycle used here */
129
130 iretq
131
132 ISR_VEC(1, apic_isr1)
133 ISR_VEC(2, apic_isr2)
134 ISR_VEC(3, apic_isr3)
135 ISR_VEC(4, apic_isr4)
136 ISR_VEC(5, apic_isr5)
137 ISR_VEC(6, apic_isr6)
138 ISR_VEC(7, apic_isr7)
139
140 /*
141 * Local APIC periodic timer handler.
142 */
143 .text
144 SUPERALIGN_TEXT
145 IDTVEC(timerint)
146 PUSH_FRAME
147
148 movq lapic, %rdx
149 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
150
151 FAKE_MCOUNT(TF_RIP(%rsp))
152
153 call lapic_handle_timer
154 MEXITCOUNT
155 jmp doreti
156
157 #ifdef SMP
158 /*
159 * Global address space TLB shootdown.
160 */
161 .text
162 SUPERALIGN_TEXT
163 IDTVEC(invltlb)
164 pushq %rax
165
166 movq %cr3, %rax /* invalidate the TLB */
167 movq %rax, %cr3
168
169 movq lapic, %rax
170 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
171
172 lock
173 incl smp_tlb_wait
174
175 popq %rax
176 iretq
177
178 /*
179 * Single page TLB shootdown
180 */
181 .text
182 SUPERALIGN_TEXT
183 IDTVEC(invlpg)
184 pushq %rax
185
186 movq smp_tlb_addr1, %rax
187 invlpg (%rax) /* invalidate single page */
188
189 movq lapic, %rax
190 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
191
192 lock
193 incl smp_tlb_wait
194
195 popq %rax
196 iretq
197
198 /*
199 * Page range TLB shootdown.
200 */
201 .text
202 SUPERALIGN_TEXT
203 IDTVEC(invlrng)
204 pushq %rax
205 pushq %rdx
206
207 movq smp_tlb_addr1, %rdx
208 movq smp_tlb_addr2, %rax
209 1: invlpg (%rdx) /* invalidate single page */
210 addq $PAGE_SIZE, %rdx
211 cmpq %rax, %rdx
212 jb 1b
213
214 movq lapic, %rax
215 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
216
217 lock
218 incl smp_tlb_wait
219
220 popq %rdx
221 popq %rax
222 iretq
223
224 /*
225 * Forward hardclock to another CPU. Pushes a clockframe and calls
226 * forwarded_hardclock().
227 */
228 .text
229 SUPERALIGN_TEXT
230 IDTVEC(ipi_intr_bitmap_handler)
231
232 PUSH_FRAME
233
234 movq lapic, %rdx
235 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
236
237 FAKE_MCOUNT(TF_RIP(%rsp))
238
239 call ipi_bitmap_handler
240 MEXITCOUNT
241 jmp doreti
242
243 /*
244 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
245 *
246 * The other CPU has already executed aston() or need_resched() on our
247 * current process, so we simply need to ack the interrupt and return
248 * via doreti to run ast().
249 */
250
251 .text
252 SUPERALIGN_TEXT
253 IDTVEC(cpuast)
254 PUSH_FRAME
255
256 movq lapic, %rdx
257 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
258
259 FAKE_MCOUNT(TF_RIP(%rsp))
260
261 MEXITCOUNT
262 jmp doreti
263
264 /*
265 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
266 *
267 * - Signals its receipt.
268 * - Waits for permission to restart.
269 * - Signals its restart.
270 */
271 .text
272 SUPERALIGN_TEXT
273 IDTVEC(cpustop)
274 PUSH_FRAME
275
276 movq lapic, %rax
277 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
278
279 movl PCPU(CPUID), %eax
280 imull $PCB_SIZE, %eax
281 leaq stoppcbs(%rax), %rdi
282 call savectx /* Save process context */
283
284 movl PCPU(CPUID), %eax
285
286 lock
287 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
288 1:
289 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
290 jnc 1b
291
292 lock
293 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
294 lock
295 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
296
297 test %eax, %eax
298 jnz 2f
299
300 movq cpustop_restartfunc, %rax
301 testq %rax, %rax
302 jz 2f
303 movq $0, cpustop_restartfunc /* One-shot */
304
305 call *%rax
306 2:
307 POP_FRAME
308 iretq
309
310 /*
311 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
312 *
313 * - Calls the generic rendezvous action function.
314 */
315 .text
316 SUPERALIGN_TEXT
317 IDTVEC(rendezvous)
318 PUSH_FRAME
319 call smp_rendezvous_action
320 movq lapic, %rax
321 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
322 POP_FRAME /* Why not doreti? */
323 iretq
324 #endif /* SMP */
Cache object: c14cda0dafb577c64423a800f23ffd3e
|