1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD: releng/5.3/sys/amd64/amd64/apic_vector.S 129653 2004-05-24 12:08:56Z bde $
32 */
33
34 /*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39 #include <machine/asmacros.h>
40 #include <machine/apicreg.h>
41
42 #include "assym.s"
43
44 /*
45 * Macros to create and destroy a trap frame.
46 */
47 #define PUSH_FRAME \
48 subq $TF_RIP,%rsp ; /* skip dummy tf_err and tf_trapno */ \
49 testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
50 jz 1f ; /* Yes, dont swapgs again */ \
51 swapgs ; \
52 1: movq %rdi,TF_RDI(%rsp) ; \
53 movq %rsi,TF_RSI(%rsp) ; \
54 movq %rdx,TF_RDX(%rsp) ; \
55 movq %rcx,TF_RCX(%rsp) ; \
56 movq %r8,TF_R8(%rsp) ; \
57 movq %r9,TF_R9(%rsp) ; \
58 movq %rax,TF_RAX(%rsp) ; \
59 movq %rbx,TF_RBX(%rsp) ; \
60 movq %rbp,TF_RBP(%rsp) ; \
61 movq %r10,TF_R10(%rsp) ; \
62 movq %r11,TF_R11(%rsp) ; \
63 movq %r12,TF_R12(%rsp) ; \
64 movq %r13,TF_R13(%rsp) ; \
65 movq %r14,TF_R14(%rsp) ; \
66 movq %r15,TF_R15(%rsp)
67
68 #define POP_FRAME \
69 movq TF_RDI(%rsp),%rdi ; \
70 movq TF_RSI(%rsp),%rsi ; \
71 movq TF_RDX(%rsp),%rdx ; \
72 movq TF_RCX(%rsp),%rcx ; \
73 movq TF_R8(%rsp),%r8 ; \
74 movq TF_R9(%rsp),%r9 ; \
75 movq TF_RAX(%rsp),%rax ; \
76 movq TF_RBX(%rsp),%rbx ; \
77 movq TF_RBP(%rsp),%rbp ; \
78 movq TF_R10(%rsp),%r10 ; \
79 movq TF_R11(%rsp),%r11 ; \
80 movq TF_R12(%rsp),%r12 ; \
81 movq TF_R13(%rsp),%r13 ; \
82 movq TF_R14(%rsp),%r14 ; \
83 movq TF_R15(%rsp),%r15 ; \
84 testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
85 jz 1f ; /* keep kernel GS.base */ \
86 cli ; \
87 swapgs ; \
88 1: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */
89
90
91 /*
92 * I/O Interrupt Entry Point. Rather than having one entry point for
93 * each interrupt source, we use one entry point for each 32-bit word
94 * in the ISR. The handler determines the highest bit set in the ISR,
95 * translates that into a vector, and passes the vector to the
96 * lapic_handle_intr() function.
97 */
98 #define ISR_VEC(index, vec_name) \
99 .text ; \
100 SUPERALIGN_TEXT ; \
101 IDTVEC(vec_name) ; \
102 PUSH_FRAME ; \
103 FAKE_MCOUNT(TF_RIP(%rsp)) ; \
104 movq lapic, %rdx ; /* pointer to local APIC */ \
105 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \
106 bsrl %eax, %eax ; /* index of highset set bit in ISR */ \
107 jz 2f ; \
108 addl $(32 * index),%eax ; \
109 1: ; \
110 movq %rax, %rdi ; /* pass the IRQ */ \
111 call lapic_handle_intr ; \
112 MEXITCOUNT ; \
113 jmp doreti ; \
114 2: movl $-1, %eax ; /* send a vector of -1 */ \
115 jmp 1b
116
117 /*
118 * Handle "spurious INTerrupts".
119 * Notes:
120 * This is different than the "spurious INTerrupt" generated by an
121 * 8259 PIC for missing INTs. See the APIC documentation for details.
122 * This routine should NOT do an 'EOI' cycle.
123 */
124 .text
125 SUPERALIGN_TEXT
126 IDTVEC(spuriousint)
127
128 /* No EOI cycle used here */
129
130 iretq
131
132 ISR_VEC(1, apic_isr1)
133 ISR_VEC(2, apic_isr2)
134 ISR_VEC(3, apic_isr3)
135 ISR_VEC(4, apic_isr4)
136 ISR_VEC(5, apic_isr5)
137 ISR_VEC(6, apic_isr6)
138 ISR_VEC(7, apic_isr7)
139
140 #ifdef SMP
141 /*
142 * Global address space TLB shootdown.
143 */
144 .text
145 SUPERALIGN_TEXT
146 IDTVEC(invltlb)
147 pushq %rax
148
149 movq %cr3, %rax /* invalidate the TLB */
150 movq %rax, %cr3
151
152 movq lapic, %rax
153 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
154
155 lock
156 incl smp_tlb_wait
157
158 popq %rax
159 iretq
160
161 /*
162 * Single page TLB shootdown
163 */
164 .text
165 SUPERALIGN_TEXT
166 IDTVEC(invlpg)
167 pushq %rax
168
169 movq smp_tlb_addr1, %rax
170 invlpg (%rax) /* invalidate single page */
171
172 movq lapic, %rax
173 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
174
175 lock
176 incl smp_tlb_wait
177
178 popq %rax
179 iretq
180
181 /*
182 * Page range TLB shootdown.
183 */
184 .text
185 SUPERALIGN_TEXT
186 IDTVEC(invlrng)
187 pushq %rax
188 pushq %rdx
189
190 movq smp_tlb_addr1, %rdx
191 movq smp_tlb_addr2, %rax
192 1: invlpg (%rdx) /* invalidate single page */
193 addq $PAGE_SIZE, %rdx
194 cmpq %rax, %rdx
195 jb 1b
196
197 movq lapic, %rax
198 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
199
200 lock
201 incl smp_tlb_wait
202
203 popq %rdx
204 popq %rax
205 iretq
206
207 /*
208 * Forward hardclock to another CPU. Pushes a clockframe and calls
209 * forwarded_hardclock().
210 */
211 .text
212 SUPERALIGN_TEXT
213 IDTVEC(hardclock)
214 PUSH_FRAME
215
216 movq lapic, %rdx
217 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
218
219 call forwarded_hardclock
220 MEXITCOUNT
221 jmp doreti
222
223 /*
224 * Forward statclock to another CPU. Pushes a clockframe and calls
225 * forwarded_statclock().
226 */
227 .text
228 SUPERALIGN_TEXT
229 IDTVEC(statclock)
230 PUSH_FRAME
231
232 movq lapic, %rdx
233 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
234
235 FAKE_MCOUNT(TF_RIP(%rsp))
236
237 call forwarded_statclock
238 MEXITCOUNT
239 jmp doreti
240
241 /*
242 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
243 *
244 * The other CPU has already executed aston() or need_resched() on our
245 * current process, so we simply need to ack the interrupt and return
246 * via doreti to run ast().
247 */
248
249 .text
250 SUPERALIGN_TEXT
251 IDTVEC(cpuast)
252 PUSH_FRAME
253
254 movq lapic, %rdx
255 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
256
257 FAKE_MCOUNT(TF_RIP(%rsp))
258
259 MEXITCOUNT
260 jmp doreti
261
262 /*
263 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
264 *
265 * - Signals its receipt.
266 * - Waits for permission to restart.
267 * - Signals its restart.
268 */
269 .text
270 SUPERALIGN_TEXT
271 IDTVEC(cpustop)
272 PUSH_FRAME
273
274 movq lapic, %rax
275 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
276
277 movl PCPU(CPUID), %eax
278 imull $PCB_SIZE, %eax
279 leaq stoppcbs(%rax), %rdi
280 call savectx /* Save process context */
281
282 movl PCPU(CPUID), %eax
283
284 lock
285 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
286 1:
287 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
288 jnc 1b
289
290 lock
291 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
292 lock
293 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
294
295 test %eax, %eax
296 jnz 2f
297
298 movq cpustop_restartfunc, %rax
299 testq %rax, %rax
300 jz 2f
301 movq $0, cpustop_restartfunc /* One-shot */
302
303 call *%rax
304 2:
305 POP_FRAME
306 iretq
307
308 /*
309 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
310 *
311 * - Calls the generic rendezvous action function.
312 */
313 .text
314 SUPERALIGN_TEXT
315 IDTVEC(rendezvous)
316 PUSH_FRAME
317 call smp_rendezvous_action
318 movq lapic, %rax
319 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
320 POP_FRAME /* Why not doreti? */
321 iretq
322 #endif /* SMP */
Cache object: 39e1d036e50828b565bcab542fb54bdc
|