1 /*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1993 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. All advertising materials mentioning features or use of this software
15 * must display the following acknowledgement:
16 * This product includes software developed by the University of
17 * California, Berkeley and its contributors.
18 * 4. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * $FreeBSD: releng/5.2/sys/amd64/amd64/support.S 123010 2003-11-27 17:20:44Z peter $
35 */
36
37 #include "opt_ddb.h"
38
39 #include <machine/asmacros.h>
40 #include <machine/intr_machdep.h>
41 #include <machine/pmap.h>
42
43 #include "assym.s"
44
45 ALIGN_DATA
46 .globl intrcnt, eintrcnt
47 intrcnt:
48 .space INTRCNT_COUNT * 8
49 eintrcnt:
50
51 .globl intrnames, eintrnames
52 intrnames:
53 .space INTRCNT_COUNT * (MAXCOMLEN + 1)
54 eintrnames:
55
56 .text
57
58 /*
59 * bcopy family
60 * void bzero(void *buf, u_int len)
61 */
62
63 /* done */
64 ENTRY(bzero)
65 movq %rsi,%rcx
66 xorq %rax,%rax
67 shrq $3,%rcx
68 cld
69 rep
70 stosq
71 movq %rsi,%rcx
72 andq $7,%rcx
73 rep
74 stosb
75 ret
76
77 /* Address: %rdi */
78 ENTRY(pagezero)
79 lea 4096(%rdi),%rsi
80 xorq %rax,%rax
81 1:
82 movnti %rax,(%rdi)
83 movnti %rax,8(%rdi)
84 movnti %rax,16(%rdi)
85 movnti %rax,24(%rdi)
86 addq $32,%rdi
87 cmpq %rsi,%rdi
88 jne 1b
89 sfence
90 retq
91
92 ENTRY(bcmp)
93 xorq %rax,%rax
94
95 movq %rdx,%rcx
96 shrq $3,%rcx
97 cld /* compare forwards */
98 repe
99 cmpsq
100 jne 1f
101
102 movq %rdx,%rcx
103 andq $7,%rcx
104 repe
105 cmpsb
106 je 2f
107 1:
108 incq %rax
109 2:
110 ret
111
112 /*
113 * bcopy(src, dst, cnt)
114 * rdi, rsi, rdx
115 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
116 */
117 ENTRY(bcopy)
118 xchgq %rsi,%rdi
119 movq %rdx,%rcx
120
121 movq %rdi,%rax
122 subq %rsi,%rax
123 cmpq %rcx,%rax /* overlapping && src < dst? */
124 jb 1f
125
126 shrq $3,%rcx /* copy by 64-bit words */
127 cld /* nope, copy forwards */
128 rep
129 movsq
130 movq %rdx,%rcx
131 andq $7,%rcx /* any bytes left? */
132 rep
133 movsb
134 ret
135
136 /* ALIGN_TEXT */
137 1:
138 addq %rcx,%rdi /* copy backwards */
139 addq %rcx,%rsi
140 decq %rdi
141 decq %rsi
142 andq $7,%rcx /* any fractional bytes? */
143 std
144 rep
145 movsb
146 movq %rdx,%rcx /* copy remainder by 32-bit words */
147 shrq $3,%rcx
148 subq $7,%rsi
149 subq $7,%rdi
150 rep
151 movsq
152 cld
153 ret
154
155 /*
156 * Note: memcpy does not support overlapping copies
157 */
158 ENTRY(memcpy)
159 movq %rdx,%rcx
160 shrq $3,%rcx /* copy by 64-bit words */
161 cld /* copy forwards */
162 rep
163 movsq
164 movq %rdx,%rcx
165 andq $7,%rcx /* any bytes left? */
166 rep
167 movsb
168 ret
169
170 /* fillw(pat, base, cnt) */
171 /* %rdi,%rsi, %rdx */
172 ENTRY(fillw)
173 movq %rdi,%rax
174 movq %rsi,%rdi
175 movq %rdx,%rcx
176 cld
177 rep
178 stosw
179 ret
180
181 /*****************************************************************************/
182 /* copyout and fubyte family */
183 /*****************************************************************************/
184 /*
185 * Access user memory from inside the kernel. These routines and possibly
186 * the math- and DOS emulators should be the only places that do this.
187 *
188 * We have to access the memory with user's permissions, so use a segment
189 * selector with RPL 3. For writes to user space we have to additionally
190 * check the PTE for write permission, because the 386 does not check
191 * write permissions when we are executing with EPL 0. The 486 does check
192 * this if the WP bit is set in CR0, so we can use a simpler version here.
193 *
194 * These routines set curpcb->onfault for the time they execute. When a
195 * protection violation occurs inside the functions, the trap handler
196 * returns to *curpcb->onfault instead of the function.
197 */
198
199 /*
200 * copyout(from_kernel, to_user, len) - MP SAFE
201 * %rdi, %rsi, %rdx
202 */
203 ENTRY(copyout)
204 movq PCPU(CURPCB),%rax
205 movq $copyout_fault,PCB_ONFAULT(%rax)
206 testq %rdx,%rdx /* anything to do? */
207 jz done_copyout
208
209 /*
210 * Check explicitly for non-user addresses. If 486 write protection
211 * is being used, this check is essential because we are in kernel
212 * mode so the h/w does not provide any protection against writing
213 * kernel addresses.
214 */
215
216 /*
217 * First, prevent address wrapping.
218 */
219 movq %rsi,%rax
220 addq %rdx,%rax
221 jc copyout_fault
222 /*
223 * XXX STOP USING VM_MAXUSER_ADDRESS.
224 * It is an end address, not a max, so every time it is used correctly it
225 * looks like there is an off by one error, and of course it caused an off
226 * by one error in several places.
227 */
228 movq $VM_MAXUSER_ADDRESS,%rcx
229 cmpq %rcx,%rax
230 ja copyout_fault
231
232 xchgq %rdi, %rsi
233 /* bcopy(%rsi, %rdi, %rdx) */
234 movq %rdx,%rcx
235
236 shrq $3,%rcx
237 cld
238 rep
239 movsq
240 movb %dl,%cl
241 andb $7,%cl
242 rep
243 movsb
244
245 done_copyout:
246 xorq %rax,%rax
247 movq PCPU(CURPCB),%rdx
248 movq %rax,PCB_ONFAULT(%rdx)
249 ret
250
251 ALIGN_TEXT
252 copyout_fault:
253 movq PCPU(CURPCB),%rdx
254 movq $0,PCB_ONFAULT(%rdx)
255 movq $EFAULT,%rax
256 ret
257
258 /*
259 * copyin(from_user, to_kernel, len) - MP SAFE
260 * %rdi, %rsi, %rdx
261 */
262 ENTRY(copyin)
263 movq PCPU(CURPCB),%rax
264 movq $copyin_fault,PCB_ONFAULT(%rax)
265 testq %rdx,%rdx /* anything to do? */
266 jz done_copyin
267
268 /*
269 * make sure address is valid
270 */
271 movq %rdi,%rax
272 addq %rdx,%rax
273 jc copyin_fault
274 movq $VM_MAXUSER_ADDRESS,%rcx
275 cmpq %rcx,%rax
276 ja copyin_fault
277
278 xchgq %rdi, %rsi
279 movq %rdx, %rcx
280 movb %cl,%al
281 shrq $3,%rcx /* copy longword-wise */
282 cld
283 rep
284 movsq
285 movb %al,%cl
286 andb $7,%cl /* copy remaining bytes */
287 rep
288 movsb
289
290 done_copyin:
291 xorq %rax,%rax
292 movq PCPU(CURPCB),%rdx
293 movq %rax,PCB_ONFAULT(%rdx)
294 ret
295
296 ALIGN_TEXT
297 copyin_fault:
298 movq PCPU(CURPCB),%rdx
299 movq $0,PCB_ONFAULT(%rdx)
300 movq $EFAULT,%rax
301 ret
302
303 /*
304 * casuptr. Compare and set user pointer. Returns -1 or the current value.
305 * dst = %rdi, old = %rsi, new = %rdx
306 */
307 ENTRY(casuptr)
308 movq PCPU(CURPCB),%rcx
309 movq $fusufault,PCB_ONFAULT(%rcx)
310
311 movq $VM_MAXUSER_ADDRESS-4,%rax
312 cmpq %rax,%rdi /* verify address is valid */
313 ja fusufault
314
315 movq %rsi, %rax /* old */
316 #ifdef SMP
317 lock
318 #endif
319 cmpxchgq %rdx, (%rdi) /* new = %rdx */
320
321 /*
322 * The old value is in %eax. If the store succeeded it will be the
323 * value we expected (old) from before the store, otherwise it will
324 * be the current value.
325 */
326
327 movq PCPU(CURPCB),%rcx
328 movq $fusufault,PCB_ONFAULT(%rcx)
329 movq $0,PCB_ONFAULT(%rcx)
330 ret
331
332 /*
333 * fu{byte,sword,word} - MP SAFE
334 *
335 * Fetch a byte (sword, word) from user memory
336 * %rdi
337 */
338 ENTRY(fuword64)
339 movq PCPU(CURPCB),%rcx
340 movq $fusufault,PCB_ONFAULT(%rcx)
341
342 movq $VM_MAXUSER_ADDRESS-8,%rax
343 cmpq %rax,%rdi /* verify address is valid */
344 ja fusufault
345
346 movq (%rdi),%rax
347 movq $0,PCB_ONFAULT(%rcx)
348 ret
349
350 ENTRY(fuword)
351 jmp fuword64
352
353 ENTRY(fuword32)
354 movq PCPU(CURPCB),%rcx
355 movq $fusufault,PCB_ONFAULT(%rcx)
356
357 movq $VM_MAXUSER_ADDRESS-4,%rax
358 cmpq %rax,%rdi /* verify address is valid */
359 ja fusufault
360
361 movl (%rdi),%eax
362 movq $0,PCB_ONFAULT(%rcx)
363 ret
364
365 /*
366 * These two routines are called from the profiling code, potentially
367 * at interrupt time. If they fail, that's okay, good things will
368 * happen later. Fail all the time for now - until the trap code is
369 * able to deal with this.
370 */
371 ALTENTRY(suswintr)
372 ENTRY(fuswintr)
373 movq $-1,%rax
374 ret
375
376 /*
377 * fuword16 - MP SAFE
378 */
379 ENTRY(fuword16)
380 movq PCPU(CURPCB),%rcx
381 movq $fusufault,PCB_ONFAULT(%rcx)
382
383 movq $VM_MAXUSER_ADDRESS-2,%rax
384 cmpq %rax,%rdi
385 ja fusufault
386
387 movzwl (%rdi),%eax
388 movq $0,PCB_ONFAULT(%rcx)
389 ret
390
391 /*
392 * fubyte - MP SAFE
393 */
394 ENTRY(fubyte)
395 movq PCPU(CURPCB),%rcx
396 movq $fusufault,PCB_ONFAULT(%rcx)
397
398 movq $VM_MAXUSER_ADDRESS-1,%rax
399 cmpq %rax,%rdi
400 ja fusufault
401
402 movzbl (%rdi),%eax
403 movq $0,PCB_ONFAULT(%rcx)
404 ret
405
406 ALIGN_TEXT
407 fusufault:
408 movq PCPU(CURPCB),%rcx
409 xorq %rax,%rax
410 movq %rax,PCB_ONFAULT(%rcx)
411 decq %rax
412 ret
413
414 /*
415 * su{byte,sword,word} - MP SAFE
416 *
417 * Write a byte (word, longword) to user memory
418 * addr = %rdi, value = %rsi
419 */
420 ENTRY(suword64)
421 movq PCPU(CURPCB),%rcx
422 movq $fusufault,PCB_ONFAULT(%rcx)
423
424 movq $VM_MAXUSER_ADDRESS-8,%rax
425 cmpq %rax,%rdi /* verify address validity */
426 ja fusufault
427
428 movq %rsi,(%rdi)
429 xorq %rax,%rax
430 movq PCPU(CURPCB),%rcx
431 movq %rax,PCB_ONFAULT(%rcx)
432 ret
433
434 ENTRY(suword)
435 jmp suword64
436
437 ENTRY(suword32)
438 movq PCPU(CURPCB),%rcx
439 movq $fusufault,PCB_ONFAULT(%rcx)
440
441 movq $VM_MAXUSER_ADDRESS-4,%rax
442 cmpq %rax,%rdi /* verify address validity */
443 ja fusufault
444
445 movl %esi,(%rdi)
446 xorq %rax,%rax
447 movq PCPU(CURPCB),%rcx
448 movq %rax,PCB_ONFAULT(%rcx)
449 ret
450
451 /*
452 * suword16 - MP SAFE
453 */
454 ENTRY(suword16)
455 movq PCPU(CURPCB),%rcx
456 movq $fusufault,PCB_ONFAULT(%rcx)
457
458 movq $VM_MAXUSER_ADDRESS-2,%rax
459 cmpq %rax,%rdi /* verify address validity */
460 ja fusufault
461
462 movw %si,(%rdi)
463 xorq %rax,%rax
464 movq PCPU(CURPCB),%rcx /* restore trashed register */
465 movq %rax,PCB_ONFAULT(%rcx)
466 ret
467
468 /*
469 * subyte - MP SAFE
470 */
471 ENTRY(subyte)
472 movq PCPU(CURPCB),%rcx
473 movq $fusufault,PCB_ONFAULT(%rcx)
474
475 movq $VM_MAXUSER_ADDRESS-1,%rax
476 cmpq %rax,%rdi /* verify address validity */
477 ja fusufault
478
479 movl %esi, %eax
480 movb %al,(%rdi)
481 xorq %rax,%rax
482 movq PCPU(CURPCB),%rcx /* restore trashed register */
483 movq %rax,PCB_ONFAULT(%rcx)
484 ret
485
486 /*
487 * copyinstr(from, to, maxlen, int *lencopied) - MP SAFE
488 * %rdi, %rsi, %rdx, %rcx
489 *
490 * copy a string from from to to, stop when a 0 character is reached.
491 * return ENAMETOOLONG if string is longer than maxlen, and
492 * EFAULT on protection violations. If lencopied is non-zero,
493 * return the actual length in *lencopied.
494 */
495 ENTRY(copyinstr)
496 movq %rdx, %r8 /* %r8 = maxlen */
497 movq %rcx, %r9 /* %r9 = *len */
498 xchgq %rdi, %rsi /* %rdi = from, %rsi = to */
499 movq PCPU(CURPCB),%rcx
500 movq $cpystrflt,PCB_ONFAULT(%rcx)
501
502 movq $VM_MAXUSER_ADDRESS,%rax
503
504 /* make sure 'from' is within bounds */
505 subq %rsi,%rax
506 jbe cpystrflt
507
508 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
509 cmpq %rdx,%rax
510 jae 1f
511 movq %rax,%rdx
512 movq %rax,%r8
513 1:
514 incq %rdx
515 cld
516
517 2:
518 decq %rdx
519 jz 3f
520
521 lodsb
522 stosb
523 orb %al,%al
524 jnz 2b
525
526 /* Success -- 0 byte reached */
527 decq %rdx
528 xorq %rax,%rax
529 jmp cpystrflt_x
530 3:
531 /* rdx is zero - return ENAMETOOLONG or EFAULT */
532 movq $VM_MAXUSER_ADDRESS,%rax
533 cmpq %rax,%rsi
534 jae cpystrflt
535 4:
536 movq $ENAMETOOLONG,%rax
537 jmp cpystrflt_x
538
539 cpystrflt:
540 movq $EFAULT,%rax
541
542 cpystrflt_x:
543 /* set *lencopied and return %eax */
544 movq PCPU(CURPCB),%rcx
545 movq $0,PCB_ONFAULT(%rcx)
546
547 testq %r9,%r9
548 jz 1f
549 subq %rdx,%r8
550 movq %r8,(%r9)
551 1:
552 ret
553
554
555 /*
556 * copystr(from, to, maxlen, int *lencopied) - MP SAFE
557 * %rdi, %rsi, %rdx, %rcx
558 */
559 ENTRY(copystr)
560 movq %rdx, %r8 /* %r8 = maxlen */
561
562 xchgq %rdi, %rsi
563 incq %rdx
564 cld
565 1:
566 decq %rdx
567 jz 4f
568 lodsb
569 stosb
570 orb %al,%al
571 jnz 1b
572
573 /* Success -- 0 byte reached */
574 decq %rdx
575 xorq %rax,%rax
576 jmp 6f
577 4:
578 /* rdx is zero -- return ENAMETOOLONG */
579 movq $ENAMETOOLONG,%rax
580
581 6:
582
583 testq %rcx, %rcx
584 jz 7f
585 /* set *lencopied and return %rax */
586 subq %rdx, %r8
587 movq %r8, (%rcx)
588 7:
589 ret
590
591 /*
592 * Handling of special 386 registers and descriptor tables etc
593 * %rdi
594 */
595 /* void lgdt(struct region_descriptor *rdp); */
596 ENTRY(lgdt)
597 /* reload the descriptor table */
598 lgdt (%rdi)
599
600 /* flush the prefetch q */
601 jmp 1f
602 nop
603 1:
604 movl $KDSEL, %eax
605 mov %ax,%ds
606 mov %ax,%es
607 mov %ax,%fs /* Beware, use wrmsr to set 64 bit base */
608 mov %ax,%gs
609 mov %ax,%ss
610
611 /* reload code selector by turning return into intersegmental return */
612 popq %rax
613 pushq $KCSEL
614 pushq %rax
615 lretq
616
617 #ifdef DDB
618 /*****************************************************************************/
619 /* setjump, longjump */
620 /*****************************************************************************/
621
622 ENTRY(setjmp)
623 movq %rbx,0(%rdi) /* save rbx */
624 movq %rsp,8(%rdi) /* save rsp */
625 movq %rbp,16(%rdi) /* save rbp */
626 movq %r12,24(%rdi) /* save r12 */
627 movq %r13,32(%rdi) /* save r13 */
628 movq %r14,40(%rdi) /* save r14 */
629 movq %r15,48(%rdi) /* save r15 */
630 movq 0(%rsp),%rdx /* get rta */
631 movq %rdx,56(%rdi) /* save rip */
632 xorl %eax,%eax /* return(0); */
633 ret
634
635 ENTRY(longjmp)
636 movq 0(%rdi),%rbx /* restore rbx */
637 movq 8(%rdi),%rsp /* restore rsp */
638 movq 16(%rdi),%rbp /* restore rbp */
639 movq 24(%rdi),%r12 /* restore r12 */
640 movq 32(%rdi),%r13 /* restore r13 */
641 movq 40(%rdi),%r14 /* restore r14 */
642 movq 48(%rdi),%r15 /* restore r15 */
643 movq 56(%rdi),%rdx /* get rta */
644 movq %rdx,0(%rsp) /* put in return frame */
645 xorl %eax,%eax /* return(1); */
646 incl %eax
647 ret
648 #endif
Cache object: 0488d20532f3921459bef5c169a2965e
|