1 /*-
2 * Copyright (c) 1993 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by the University of
16 * California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * $FreeBSD: releng/5.1/sys/amd64/amd64/support.S 115402 2003-05-30 00:58:48Z peter $
34 */
35
36 #include "opt_ddb.h"
37
38 #include <machine/asmacros.h>
39 #include <machine/pmap.h>
40
41 #include "assym.s"
42
43
44 .text
45
46 /*
47 * bcopy family
48 * void bzero(void *buf, u_int len)
49 */
50
51 /* done */
52 ENTRY(bzero)
53 movq %rsi,%rcx
54 xorq %rax,%rax
55 shrq $3,%rcx
56 cld
57 rep
58 stosq
59 movq %rsi,%rcx
60 andq $7,%rcx
61 rep
62 stosb
63 ret
64
65 /* Address: %rdi */
66 ENTRY(pagezero)
67 movq $512, %rcx
68 cld
69
70 ALIGN_TEXT
71 1:
72 xorq %rax, %rax
73 repe
74 scasq
75 jnz 2f
76
77 ret
78
79 ALIGN_TEXT
80
81 2:
82 incq %rcx
83 subq $8, %rdi
84
85 movq %rcx, %rdx
86 cmpq $16, %rcx
87
88 jge 3f
89
90 movq %rdi, %r8
91 andq $0x7f, %r8
92 shrq $3, %r8
93 movq $16, %rcx
94 subq %r8, %rcx
95
96 3:
97 subq %rcx, %rdx
98 rep
99 stosq
100
101 movq %rdx, %rcx
102 testq %rdx, %rdx
103 jnz 1b
104
105 ret
106
107
108 ENTRY(bcmp)
109 xorq %rax,%rax
110
111 movq %rdx,%rcx
112 shrq $3,%rcx
113 cld /* compare forwards */
114 repe
115 cmpsq
116 jne 1f
117
118 movq %rdx,%rcx
119 andq $7,%rcx
120 repe
121 cmpsb
122 je 2f
123 1:
124 incq %rax
125 2:
126 ret
127
128 /*
129 * bcopy(src, dst, cnt)
130 * rdi, rsi, rdx
131 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
132 */
133 ENTRY(bcopy)
134 xchgq %rsi,%rdi
135 movq %rdx,%rcx
136
137 movq %rdi,%rax
138 subq %rsi,%rax
139 cmpq %rcx,%rax /* overlapping && src < dst? */
140 jb 1f
141
142 shrq $3,%rcx /* copy by 64-bit words */
143 cld /* nope, copy forwards */
144 rep
145 movsq
146 movq %rdx,%rcx
147 andq $7,%rcx /* any bytes left? */
148 rep
149 movsb
150 ret
151
152 /* ALIGN_TEXT */
153 1:
154 addq %rcx,%rdi /* copy backwards */
155 addq %rcx,%rsi
156 decq %rdi
157 decq %rsi
158 andq $7,%rcx /* any fractional bytes? */
159 std
160 rep
161 movsb
162 movq %rdx,%rcx /* copy remainder by 32-bit words */
163 shrq $3,%rcx
164 subq $7,%rsi
165 subq $7,%rdi
166 rep
167 movsq
168 cld
169 ret
170
171 /*
172 * Note: memcpy does not support overlapping copies
173 */
174 ENTRY(memcpy)
175 movq %rdx,%rcx
176 shrq $3,%rcx /* copy by 64-bit words */
177 cld /* copy forwards */
178 rep
179 movsq
180 movq %rdx,%rcx
181 andq $7,%rcx /* any bytes left? */
182 rep
183 movsb
184 ret
185
186 /* fillw(pat, base, cnt) */
187 /* %rdi,%rsi, %rdx */
188 ENTRY(fillw)
189 movq %rdi,%rax
190 movq %rsi,%rdi
191 movq %rdx,%rcx
192 cld
193 rep
194 stosw
195 ret
196
197 /*****************************************************************************/
198 /* copyout and fubyte family */
199 /*****************************************************************************/
200 /*
201 * Access user memory from inside the kernel. These routines and possibly
202 * the math- and DOS emulators should be the only places that do this.
203 *
204 * We have to access the memory with user's permissions, so use a segment
205 * selector with RPL 3. For writes to user space we have to additionally
206 * check the PTE for write permission, because the 386 does not check
207 * write permissions when we are executing with EPL 0. The 486 does check
208 * this if the WP bit is set in CR0, so we can use a simpler version here.
209 *
210 * These routines set curpcb->onfault for the time they execute. When a
211 * protection violation occurs inside the functions, the trap handler
212 * returns to *curpcb->onfault instead of the function.
213 */
214
215 /*
216 * copyout(from_kernel, to_user, len) - MP SAFE
217 * %rdi, %rsi, %rdx
218 */
219 ENTRY(copyout)
220 movq PCPU(CURPCB),%rax
221 movq $copyout_fault,PCB_ONFAULT(%rax)
222 testq %rdx,%rdx /* anything to do? */
223 jz done_copyout
224
225 /*
226 * Check explicitly for non-user addresses. If 486 write protection
227 * is being used, this check is essential because we are in kernel
228 * mode so the h/w does not provide any protection against writing
229 * kernel addresses.
230 */
231
232 /*
233 * First, prevent address wrapping.
234 */
235 movq %rsi,%rax
236 addq %rdx,%rax
237 jc copyout_fault
238 /*
239 * XXX STOP USING VM_MAXUSER_ADDRESS.
240 * It is an end address, not a max, so every time it is used correctly it
241 * looks like there is an off by one error, and of course it caused an off
242 * by one error in several places.
243 */
244 movq $VM_MAXUSER_ADDRESS,%rcx
245 cmpq %rcx,%rax
246 ja copyout_fault
247
248 xchgq %rdi, %rsi
249 /* bcopy(%rsi, %rdi, %rdx) */
250 movq %rdx,%rcx
251
252 shrq $3,%rcx
253 cld
254 rep
255 movsq
256 movb %dl,%cl
257 andb $7,%cl
258 rep
259 movsb
260
261 done_copyout:
262 xorq %rax,%rax
263 movq PCPU(CURPCB),%rdx
264 movq %rax,PCB_ONFAULT(%rdx)
265 ret
266
267 ALIGN_TEXT
268 copyout_fault:
269 movq PCPU(CURPCB),%rdx
270 movq $0,PCB_ONFAULT(%rdx)
271 movq $EFAULT,%rax
272 ret
273
274 /*
275 * copyin(from_user, to_kernel, len) - MP SAFE
276 * %rdi, %rsi, %rdx
277 */
278 ENTRY(copyin)
279 movq PCPU(CURPCB),%rax
280 movq $copyin_fault,PCB_ONFAULT(%rax)
281 testq %rdx,%rdx /* anything to do? */
282 jz done_copyin
283
284 /*
285 * make sure address is valid
286 */
287 movq %rdi,%rax
288 addq %rdx,%rax
289 jc copyin_fault
290 movq $VM_MAXUSER_ADDRESS,%rcx
291 cmpq %rcx,%rax
292 ja copyin_fault
293
294 xchgq %rdi, %rsi
295 movq %rdx, %rcx
296 movb %cl,%al
297 shrq $3,%rcx /* copy longword-wise */
298 cld
299 rep
300 movsq
301 movb %al,%cl
302 andb $7,%cl /* copy remaining bytes */
303 rep
304 movsb
305
306 done_copyin:
307 xorq %rax,%rax
308 movq PCPU(CURPCB),%rdx
309 movq %rax,PCB_ONFAULT(%rdx)
310 ret
311
312 ALIGN_TEXT
313 copyin_fault:
314 movq PCPU(CURPCB),%rdx
315 movq $0,PCB_ONFAULT(%rdx)
316 movq $EFAULT,%rax
317 ret
318
319 /*
320 * casuptr. Compare and set user pointer. Returns -1 or the current value.
321 * dst = %rdi, old = %rsi, new = %rdx
322 */
323 ENTRY(casuptr)
324 movq PCPU(CURPCB),%rcx
325 movq $fusufault,PCB_ONFAULT(%rcx)
326
327 movq $VM_MAXUSER_ADDRESS-4,%rax
328 cmpq %rax,%rdi /* verify address is valid */
329 ja fusufault
330
331 movq %rsi, %rax /* old */
332 cmpxchgq %rdx, (%rdi) /* new = %rdx */
333
334 /*
335 * The old value is in %eax. If the store succeeded it will be the
336 * value we expected (old) from before the store, otherwise it will
337 * be the current value.
338 */
339
340 movq PCPU(CURPCB),%rcx
341 movq $fusufault,PCB_ONFAULT(%rcx)
342 movq $0,PCB_ONFAULT(%rcx)
343 ret
344
345 /*
346 * fu{byte,sword,word} - MP SAFE
347 *
348 * Fetch a byte (sword, word) from user memory
349 * %rdi
350 */
351 ENTRY(fuword64)
352 movq PCPU(CURPCB),%rcx
353 movq $fusufault,PCB_ONFAULT(%rcx)
354
355 movq $VM_MAXUSER_ADDRESS-8,%rax
356 cmpq %rax,%rdi /* verify address is valid */
357 ja fusufault
358
359 movq (%rdi),%rax
360 movq $0,PCB_ONFAULT(%rcx)
361 ret
362
363 ENTRY(fuword)
364 jmp fuword64
365
366 ENTRY(fuword32)
367 movq PCPU(CURPCB),%rcx
368 movq $fusufault,PCB_ONFAULT(%rcx)
369
370 movq $VM_MAXUSER_ADDRESS-4,%rax
371 cmpq %rax,%rdi /* verify address is valid */
372 ja fusufault
373
374 movl (%rdi),%eax
375 movq $0,PCB_ONFAULT(%rcx)
376 ret
377
378 /*
379 * These two routines are called from the profiling code, potentially
380 * at interrupt time. If they fail, that's okay, good things will
381 * happen later. Fail all the time for now - until the trap code is
382 * able to deal with this.
383 */
384 ALTENTRY(suswintr)
385 ENTRY(fuswintr)
386 movq $-1,%rax
387 ret
388
389 /*
390 * fuword16 - MP SAFE
391 */
392 ENTRY(fuword16)
393 movq PCPU(CURPCB),%rcx
394 movq $fusufault,PCB_ONFAULT(%rcx)
395
396 movq $VM_MAXUSER_ADDRESS-2,%rax
397 cmpq %rax,%rdi
398 ja fusufault
399
400 movzwl (%rdi),%eax
401 movq $0,PCB_ONFAULT(%rcx)
402 ret
403
404 /*
405 * fubyte - MP SAFE
406 */
407 ENTRY(fubyte)
408 movq PCPU(CURPCB),%rcx
409 movq $fusufault,PCB_ONFAULT(%rcx)
410
411 movq $VM_MAXUSER_ADDRESS-1,%rax
412 cmpq %rax,%rdi
413 ja fusufault
414
415 movzbl (%rdi),%eax
416 movq $0,PCB_ONFAULT(%rcx)
417 ret
418
419 ALIGN_TEXT
420 fusufault:
421 movq PCPU(CURPCB),%rcx
422 xorq %rax,%rax
423 movq %rax,PCB_ONFAULT(%rcx)
424 decq %rax
425 ret
426
427 /*
428 * su{byte,sword,word} - MP SAFE
429 *
430 * Write a byte (word, longword) to user memory
431 * addr = %rdi, value = %rsi
432 */
433 ENTRY(suword64)
434 movq PCPU(CURPCB),%rcx
435 movq $fusufault,PCB_ONFAULT(%rcx)
436
437 movq $VM_MAXUSER_ADDRESS-8,%rax
438 cmpq %rax,%rdi /* verify address validity */
439 ja fusufault
440
441 movq %rsi,(%rdi)
442 xorq %rax,%rax
443 movq PCPU(CURPCB),%rcx
444 movq %rax,PCB_ONFAULT(%rcx)
445 ret
446
447 ENTRY(suword)
448 jmp suword64
449
450 ENTRY(suword32)
451 movq PCPU(CURPCB),%rcx
452 movq $fusufault,PCB_ONFAULT(%rcx)
453
454 movq $VM_MAXUSER_ADDRESS-4,%rax
455 cmpq %rax,%rdi /* verify address validity */
456 ja fusufault
457
458 movl %esi,(%rdi)
459 xorq %rax,%rax
460 movq PCPU(CURPCB),%rcx
461 movq %rax,PCB_ONFAULT(%rcx)
462 ret
463
464 /*
465 * suword16 - MP SAFE
466 */
467 ENTRY(suword16)
468 movq PCPU(CURPCB),%rcx
469 movq $fusufault,PCB_ONFAULT(%rcx)
470
471 movq $VM_MAXUSER_ADDRESS-2,%rax
472 cmpq %rax,%rdi /* verify address validity */
473 ja fusufault
474
475 movw %si,(%rdi)
476 xorq %rax,%rax
477 movq PCPU(CURPCB),%rcx /* restore trashed register */
478 movq %rax,PCB_ONFAULT(%rcx)
479 ret
480
481 /*
482 * subyte - MP SAFE
483 */
484 ENTRY(subyte)
485 movq PCPU(CURPCB),%rcx
486 movq $fusufault,PCB_ONFAULT(%rcx)
487
488 movq $VM_MAXUSER_ADDRESS-1,%rax
489 cmpq %rax,%rdi /* verify address validity */
490 ja fusufault
491
492 movl %esi, %eax
493 movb %al,(%rdi)
494 xorq %rax,%rax
495 movq PCPU(CURPCB),%rcx /* restore trashed register */
496 movq %rax,PCB_ONFAULT(%rcx)
497 ret
498
499 /*
500 * copyinstr(from, to, maxlen, int *lencopied) - MP SAFE
501 * %rdi, %rsi, %rdx, %rcx
502 *
503 * copy a string from from to to, stop when a 0 character is reached.
504 * return ENAMETOOLONG if string is longer than maxlen, and
505 * EFAULT on protection violations. If lencopied is non-zero,
506 * return the actual length in *lencopied.
507 */
508 ENTRY(copyinstr)
509 movq %rdx, %r8 /* %r8 = maxlen */
510 movq %rcx, %r9 /* %r9 = *len */
511 xchgq %rdi, %rsi /* %rdi = from, %rsi = to */
512 movq PCPU(CURPCB),%rcx
513 movq $cpystrflt,PCB_ONFAULT(%rcx)
514
515 movq $VM_MAXUSER_ADDRESS,%rax
516
517 /* make sure 'from' is within bounds */
518 subq %rsi,%rax
519 jbe cpystrflt
520
521 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
522 cmpq %rdx,%rax
523 jae 1f
524 movq %rax,%rdx
525 movq %rax,%r8
526 1:
527 incq %rdx
528 cld
529
530 2:
531 decq %rdx
532 jz 3f
533
534 lodsb
535 stosb
536 orb %al,%al
537 jnz 2b
538
539 /* Success -- 0 byte reached */
540 decq %rdx
541 xorq %rax,%rax
542 jmp cpystrflt_x
543 3:
544 /* rdx is zero - return ENAMETOOLONG or EFAULT */
545 movq $VM_MAXUSER_ADDRESS,%rax
546 cmpq %rax,%rsi
547 jae cpystrflt
548 4:
549 movq $ENAMETOOLONG,%rax
550 jmp cpystrflt_x
551
552 cpystrflt:
553 movq $EFAULT,%rax
554
555 cpystrflt_x:
556 /* set *lencopied and return %eax */
557 movq PCPU(CURPCB),%rcx
558 movq $0,PCB_ONFAULT(%rcx)
559
560 testq %r9,%r9
561 jz 1f
562 subq %rdx,%r8
563 movq %r8,(%r9)
564 1:
565 ret
566
567
568 /*
569 * copystr(from, to, maxlen, int *lencopied) - MP SAFE
570 * %rdi, %rsi, %rdx, %rcx
571 */
572 ENTRY(copystr)
573 movq %rdx, %r8 /* %r8 = maxlen */
574
575 xchgq %rdi, %rsi
576 incq %rdx
577 cld
578 1:
579 decq %rdx
580 jz 4f
581 lodsb
582 stosb
583 orb %al,%al
584 jnz 1b
585
586 /* Success -- 0 byte reached */
587 decq %rdx
588 xorq %rax,%rax
589 jmp 6f
590 4:
591 /* rdx is zero -- return ENAMETOOLONG */
592 movq $ENAMETOOLONG,%rax
593
594 6:
595
596 testq %rcx, %rcx
597 jz 7f
598 /* set *lencopied and return %rax */
599 subq %rdx, %r8
600 movq %r8, (%rcx)
601 7:
602 ret
603
604 /*
605 * Handling of special 386 registers and descriptor tables etc
606 * %rdi
607 */
608 /* void lgdt(struct region_descriptor *rdp); */
609 ENTRY(lgdt)
610 /* reload the descriptor table */
611 lgdt (%rdi)
612
613 /* flush the prefetch q */
614 jmp 1f
615 nop
616 1:
617 movl $KDSEL, %eax
618 mov %ax,%ds
619 mov %ax,%es
620 mov %ax,%fs /* Beware, use wrmsr to set 64 bit base */
621 mov %ax,%gs
622 mov %ax,%ss
623
624 /* reload code selector by turning return into intersegmental return */
625 popq %rax
626 pushq $KCSEL
627 pushq %rax
628 lretq
629
630 #ifdef DDB
631 /*****************************************************************************/
632 /* setjump, longjump */
633 /*****************************************************************************/
634
635 ENTRY(setjmp)
636 movq %rbx,0(%rdi) /* save rbx */
637 movq %rsp,8(%rdi) /* save rsp */
638 movq %rbp,16(%rdi) /* save rbp */
639 movq %r12,24(%rdi) /* save r12 */
640 movq %r13,32(%rdi) /* save r13 */
641 movq %r14,40(%rdi) /* save r14 */
642 movq %r15,48(%rdi) /* save r15 */
643 movq 0(%rsp),%rdx /* get rta */
644 movq %rdx,56(%rdi) /* save rip */
645 xorl %eax,%eax /* return(0); */
646 ret
647
648 ENTRY(longjmp)
649 movq 0(%rdi),%rbx /* restore rbx */
650 movq 8(%rdi),%rsp /* restore rsp */
651 movq 16(%rdi),%rbp /* restore rbp */
652 movq 24(%rdi),%r12 /* restore r12 */
653 movq 32(%rdi),%r13 /* restore r13 */
654 movq 40(%rdi),%r14 /* restore r14 */
655 movq 48(%rdi),%r15 /* restore r15 */
656 movq 56(%rdi),%rdx /* get rta */
657 movq %rdx,0(%rsp) /* put in return frame */
658 xorl %eax,%eax /* return(1); */
659 incl %eax
660 ret
661 #endif
Cache object: d7147f8210e856d5ccea1e1e9a3e66df
|