1 /*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1992 Terrence R. Lambert.
4 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to Berkeley by
8 * William Jolitz.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the University of
21 * California, Berkeley and its contributors.
22 * 4. Neither the name of the University nor the names of its contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91
39 */
40
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD: src/sys/amd64/amd64/machdep.c,v 1.618.2.13 2006/02/07 03:10:38 davidxu Exp $");
43
44 #include "opt_atalk.h"
45 #include "opt_atpic.h"
46 #include "opt_compat.h"
47 #include "opt_cpu.h"
48 #include "opt_ddb.h"
49 #include "opt_inet.h"
50 #include "opt_ipx.h"
51 #include "opt_isa.h"
52 #include "opt_kstack_pages.h"
53 #include "opt_maxmem.h"
54 #include "opt_msgbuf.h"
55 #include "opt_perfmon.h"
56
57 #include <sys/param.h>
58 #include <sys/proc.h>
59 #include <sys/systm.h>
60 #include <sys/bio.h>
61 #include <sys/buf.h>
62 #include <sys/bus.h>
63 #include <sys/callout.h>
64 #include <sys/cons.h>
65 #include <sys/cpu.h>
66 #include <sys/eventhandler.h>
67 #include <sys/exec.h>
68 #include <sys/imgact.h>
69 #include <sys/kdb.h>
70 #include <sys/kernel.h>
71 #include <sys/ktr.h>
72 #include <sys/linker.h>
73 #include <sys/lock.h>
74 #include <sys/malloc.h>
75 #include <sys/memrange.h>
76 #include <sys/msgbuf.h>
77 #include <sys/mutex.h>
78 #include <sys/pcpu.h>
79 #include <sys/ptrace.h>
80 #include <sys/reboot.h>
81 #include <sys/sched.h>
82 #include <sys/signalvar.h>
83 #include <sys/sysctl.h>
84 #include <sys/sysent.h>
85 #include <sys/sysproto.h>
86 #include <sys/ucontext.h>
87 #include <sys/vmmeter.h>
88
89 #include <vm/vm.h>
90 #include <vm/vm_extern.h>
91 #include <vm/vm_kern.h>
92 #include <vm/vm_page.h>
93 #include <vm/vm_map.h>
94 #include <vm/vm_object.h>
95 #include <vm/vm_pager.h>
96 #include <vm/vm_param.h>
97
98 #ifdef DDB
99 #ifndef KDB
100 #error KDB must be enabled in order for DDB to work!
101 #endif
102 #endif
103 #include <ddb/ddb.h>
104
105 #include <net/netisr.h>
106
107 #include <machine/clock.h>
108 #include <machine/cpu.h>
109 #include <machine/cputypes.h>
110 #include <machine/intr_machdep.h>
111 #include <machine/md_var.h>
112 #include <machine/metadata.h>
113 #include <machine/pc/bios.h>
114 #include <machine/pcb.h>
115 #include <machine/proc.h>
116 #include <machine/reg.h>
117 #include <machine/sigframe.h>
118 #include <machine/specialreg.h>
119 #ifdef PERFMON
120 #include <machine/perfmon.h>
121 #endif
122 #include <machine/tss.h>
123 #ifdef SMP
124 #include <machine/smp.h>
125 #endif
126
127 #include <amd64/isa/icu.h>
128
129 #include <isa/isareg.h>
130 #include <isa/rtc.h>
131
132 /* Sanity check for __curthread() */
133 CTASSERT(offsetof(struct pcpu, pc_curthread) == 0);
134
135 extern u_int64_t hammer_time(u_int64_t, u_int64_t);
136 extern void dblfault_handler(void);
137
138 extern void printcpuinfo(void); /* XXX header file */
139 extern void identify_cpu(void);
140 extern void panicifcpuunsupported(void);
141
142 #define CS_SECURE(cs) (ISPL(cs) == SEL_UPL)
143 #define EFL_SECURE(ef, oef) ((((ef) ^ (oef)) & ~PSL_USERCHANGE) == 0)
144
145 static void cpu_startup(void *);
146 static void get_fpcontext(struct thread *td, mcontext_t *mcp);
147 static int set_fpcontext(struct thread *td, const mcontext_t *mcp);
148 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL)
149
150 #ifdef DDB
151 extern vm_offset_t ksym_start, ksym_end;
152 #endif
153
154 int _udatasel, _ucodesel, _ucode32sel;
155
156 int cold = 1;
157
158 long Maxmem = 0;
159 long realmem = 0;
160
161 vm_paddr_t phys_avail[20];
162
163 /* must be 2 less so 0 0 can signal end of chunks */
164 #define PHYS_AVAIL_ARRAY_END ((sizeof(phys_avail) / sizeof(vm_offset_t)) - 2)
165
166 struct kva_md_info kmi;
167
168 static struct trapframe proc0_tf;
169 struct region_descriptor r_gdt, r_idt;
170
171 struct pcpu __pcpu[MAXCPU];
172
173 struct mtx icu_lock;
174
175 struct mem_range_softc mem_range_softc;
176
177 static void
178 cpu_startup(dummy)
179 void *dummy;
180 {
181 /*
182 * Good {morning,afternoon,evening,night}.
183 */
184 startrtclock();
185 printcpuinfo();
186 panicifcpuunsupported();
187 #ifdef PERFMON
188 perfmon_init();
189 #endif
190 printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)Maxmem),
191 ptoa((uintmax_t)Maxmem) / 1048576);
192 realmem = Maxmem;
193 /*
194 * Display any holes after the first chunk of extended memory.
195 */
196 if (bootverbose) {
197 int indx;
198
199 printf("Physical memory chunk(s):\n");
200 for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) {
201 vm_paddr_t size;
202
203 size = phys_avail[indx + 1] - phys_avail[indx];
204 printf(
205 "0x%016jx - 0x%016jx, %ju bytes (%ju pages)\n",
206 (uintmax_t)phys_avail[indx],
207 (uintmax_t)phys_avail[indx + 1] - 1,
208 (uintmax_t)size, (uintmax_t)size / PAGE_SIZE);
209 }
210 }
211
212 vm_ksubmap_init(&kmi);
213
214 printf("avail memory = %ju (%ju MB)\n",
215 ptoa((uintmax_t)cnt.v_free_count),
216 ptoa((uintmax_t)cnt.v_free_count) / 1048576);
217
218 /*
219 * Set up buffers, so they can be used to read disk labels.
220 */
221 bufinit();
222 vm_pager_bufferinit();
223
224 cpu_setregs();
225 }
226
227 /*
228 * Send an interrupt to process.
229 *
230 * Stack is set up to allow sigcode stored
231 * at top to call routine, followed by kcall
232 * to sigreturn routine below. After sigreturn
233 * resets the signal mask, the stack, and the
234 * frame pointer, it returns to the user
235 * specified pc, psl.
236 */
237 void
238 sendsig(catcher, sig, mask, code)
239 sig_t catcher;
240 int sig;
241 sigset_t *mask;
242 u_long code;
243 {
244 struct sigframe sf, *sfp;
245 struct proc *p;
246 struct thread *td;
247 struct sigacts *psp;
248 char *sp;
249 struct trapframe *regs;
250 int oonstack;
251
252 td = curthread;
253 p = td->td_proc;
254 PROC_LOCK_ASSERT(p, MA_OWNED);
255 psp = p->p_sigacts;
256 mtx_assert(&psp->ps_mtx, MA_OWNED);
257 regs = td->td_frame;
258 oonstack = sigonstack(regs->tf_rsp);
259
260 /* Save user context. */
261 bzero(&sf, sizeof(sf));
262 sf.sf_uc.uc_sigmask = *mask;
263 sf.sf_uc.uc_stack = td->td_sigstk;
264 sf.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK)
265 ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE;
266 sf.sf_uc.uc_mcontext.mc_onstack = (oonstack) ? 1 : 0;
267 bcopy(regs, &sf.sf_uc.uc_mcontext.mc_rdi, sizeof(*regs));
268 sf.sf_uc.uc_mcontext.mc_len = sizeof(sf.sf_uc.uc_mcontext); /* magic */
269 get_fpcontext(td, &sf.sf_uc.uc_mcontext);
270 fpstate_drop(td);
271
272 /* Allocate space for the signal handler context. */
273 if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
274 SIGISMEMBER(psp->ps_sigonstack, sig)) {
275 sp = td->td_sigstk.ss_sp +
276 td->td_sigstk.ss_size - sizeof(struct sigframe);
277 #if defined(COMPAT_43)
278 td->td_sigstk.ss_flags |= SS_ONSTACK;
279 #endif
280 } else
281 sp = (char *)regs->tf_rsp - sizeof(struct sigframe) - 128;
282 /* Align to 16 bytes. */
283 sfp = (struct sigframe *)((unsigned long)sp & ~0xFul);
284
285 /* Translate the signal if appropriate. */
286 if (p->p_sysent->sv_sigtbl && sig <= p->p_sysent->sv_sigsize)
287 sig = p->p_sysent->sv_sigtbl[_SIG_IDX(sig)];
288
289 /* Build the argument list for the signal handler. */
290 regs->tf_rdi = sig; /* arg 1 in %rdi */
291 regs->tf_rdx = (register_t)&sfp->sf_uc; /* arg 3 in %rdx */
292 if (SIGISMEMBER(psp->ps_siginfo, sig)) {
293 /* Signal handler installed with SA_SIGINFO. */
294 regs->tf_rsi = (register_t)&sfp->sf_si; /* arg 2 in %rsi */
295 sf.sf_ahu.sf_action = (__siginfohandler_t *)catcher;
296
297 /* Fill in POSIX parts */
298 sf.sf_si.si_signo = sig;
299 sf.sf_si.si_code = code;
300 regs->tf_rcx = regs->tf_addr; /* arg 4 in %rcx */
301 } else {
302 /* Old FreeBSD-style arguments. */
303 regs->tf_rsi = code; /* arg 2 in %rsi */
304 regs->tf_rcx = regs->tf_addr; /* arg 4 in %rcx */
305 sf.sf_ahu.sf_handler = catcher;
306 }
307 mtx_unlock(&psp->ps_mtx);
308 PROC_UNLOCK(p);
309
310 /*
311 * Copy the sigframe out to the user's stack.
312 */
313 if (copyout(&sf, sfp, sizeof(*sfp)) != 0) {
314 #ifdef DEBUG
315 printf("process %ld has trashed its stack\n", (long)p->p_pid);
316 #endif
317 PROC_LOCK(p);
318 sigexit(td, SIGILL);
319 }
320
321 regs->tf_rsp = (long)sfp;
322 regs->tf_rip = PS_STRINGS - *(p->p_sysent->sv_szsigcode);
323 regs->tf_rflags &= ~PSL_T;
324 regs->tf_cs = _ucodesel;
325 PROC_LOCK(p);
326 mtx_lock(&psp->ps_mtx);
327 }
328
329 /*
330 * Build siginfo_t for SA thread
331 */
332 void
333 cpu_thread_siginfo(int sig, u_long code, siginfo_t *si)
334 {
335 struct proc *p;
336 struct thread *td;
337 struct trapframe *regs;
338
339 td = curthread;
340 p = td->td_proc;
341 regs = td->td_frame;
342 PROC_LOCK_ASSERT(p, MA_OWNED);
343
344 bzero(si, sizeof(*si));
345 si->si_signo = sig;
346 si->si_code = code;
347 si->si_addr = (void *)regs->tf_addr;
348 /* XXXKSE fill other fields */
349 }
350
351 /*
352 * System call to cleanup state after a signal
353 * has been taken. Reset signal mask and
354 * stack state from context left by sendsig (above).
355 * Return to previous pc and psl as specified by
356 * context left by sendsig. Check carefully to
357 * make sure that the user has not modified the
358 * state to gain improper privileges.
359 *
360 * MPSAFE
361 */
362 int
363 sigreturn(td, uap)
364 struct thread *td;
365 struct sigreturn_args /* {
366 const __ucontext *sigcntxp;
367 } */ *uap;
368 {
369 ucontext_t uc;
370 struct proc *p = td->td_proc;
371 struct trapframe *regs;
372 const ucontext_t *ucp;
373 long rflags;
374 int cs, error, ret;
375
376 error = copyin(uap->sigcntxp, &uc, sizeof(uc));
377 if (error != 0)
378 return (error);
379 ucp = &uc;
380 regs = td->td_frame;
381 rflags = ucp->uc_mcontext.mc_rflags;
382 /*
383 * Don't allow users to change privileged or reserved flags.
384 */
385 /*
386 * XXX do allow users to change the privileged flag PSL_RF.
387 * The cpu sets PSL_RF in tf_rflags for faults. Debuggers
388 * should sometimes set it there too. tf_rflags is kept in
389 * the signal context during signal handling and there is no
390 * other place to remember it, so the PSL_RF bit may be
391 * corrupted by the signal handler without us knowing.
392 * Corruption of the PSL_RF bit at worst causes one more or
393 * one less debugger trap, so allowing it is fairly harmless.
394 */
395 if (!EFL_SECURE(rflags & ~PSL_RF, regs->tf_rflags & ~PSL_RF)) {
396 printf("sigreturn: rflags = 0x%lx\n", rflags);
397 return (EINVAL);
398 }
399
400 /*
401 * Don't allow users to load a valid privileged %cs. Let the
402 * hardware check for invalid selectors, excess privilege in
403 * other selectors, invalid %eip's and invalid %esp's.
404 */
405 cs = ucp->uc_mcontext.mc_cs;
406 if (!CS_SECURE(cs)) {
407 printf("sigreturn: cs = 0x%x\n", cs);
408 trapsignal(td, SIGBUS, T_PROTFLT);
409 return (EINVAL);
410 }
411
412 ret = set_fpcontext(td, &ucp->uc_mcontext);
413 if (ret != 0)
414 return (ret);
415 bcopy(&ucp->uc_mcontext.mc_rdi, regs, sizeof(*regs));
416
417 PROC_LOCK(p);
418 #if defined(COMPAT_43)
419 if (ucp->uc_mcontext.mc_onstack & 1)
420 td->td_sigstk.ss_flags |= SS_ONSTACK;
421 else
422 td->td_sigstk.ss_flags &= ~SS_ONSTACK;
423 #endif
424
425 td->td_sigmask = ucp->uc_sigmask;
426 SIG_CANTMASK(td->td_sigmask);
427 signotify(td);
428 PROC_UNLOCK(p);
429 td->td_pcb->pcb_flags |= PCB_FULLCTX;
430 return (EJUSTRETURN);
431 }
432
433 #ifdef COMPAT_FREEBSD4
434 int
435 freebsd4_sigreturn(struct thread *td, struct freebsd4_sigreturn_args *uap)
436 {
437
438 return sigreturn(td, (struct sigreturn_args *)uap);
439 }
440 #endif
441
442
443 /*
444 * Machine dependent boot() routine
445 *
446 * I haven't seen anything to put here yet
447 * Possibly some stuff might be grafted back here from boot()
448 */
449 void
450 cpu_boot(int howto)
451 {
452 }
453
454 /* Get current clock frequency for the given cpu id. */
455 int
456 cpu_est_clockrate(int cpu_id, uint64_t *rate)
457 {
458 register_t reg;
459 uint64_t tsc1, tsc2;
460
461 if (pcpu_find(cpu_id) == NULL || rate == NULL)
462 return (EINVAL);
463
464 /* If we're booting, trust the rate calibrated moments ago. */
465 if (cold) {
466 *rate = tsc_freq;
467 return (0);
468 }
469
470 #ifdef SMP
471 /* Schedule ourselves on the indicated cpu. */
472 mtx_lock_spin(&sched_lock);
473 sched_bind(curthread, cpu_id);
474 mtx_unlock_spin(&sched_lock);
475 #endif
476
477 /* Calibrate by measuring a short delay. */
478 reg = intr_disable();
479 tsc1 = rdtsc();
480 DELAY(1000);
481 tsc2 = rdtsc();
482 intr_restore(reg);
483
484 #ifdef SMP
485 mtx_lock_spin(&sched_lock);
486 sched_unbind(curthread);
487 mtx_unlock_spin(&sched_lock);
488 #endif
489
490 /*
491 * Calculate the difference in readings, convert to Mhz, and
492 * subtract 0.5% of the total. Empirical testing has shown that
493 * overhead in DELAY() works out to approximately this value.
494 */
495 tsc2 -= tsc1;
496 *rate = tsc2 * 1000 - tsc2 * 5;
497 return (0);
498 }
499
500 /*
501 * Shutdown the CPU as much as possible
502 */
503 void
504 cpu_halt(void)
505 {
506 for (;;)
507 __asm__ ("hlt");
508 }
509
510 /*
511 * Hook to idle the CPU when possible. In the SMP case we default to
512 * off because a halted cpu will not currently pick up a new thread in the
513 * run queue until the next timer tick. If turned on this will result in
514 * approximately a 4.2% loss in real time performance in buildworld tests
515 * (but improves user and sys times oddly enough), and saves approximately
516 * 5% in power consumption on an idle machine (tests w/2xCPU 1.1GHz P3).
517 *
518 * XXX we need to have a cpu mask of idle cpus and generate an IPI or
519 * otherwise generate some sort of interrupt to wake up cpus sitting in HLT.
520 * Then we can have our cake and eat it too.
521 *
522 * XXX I'm turning it on for SMP as well by default for now. It seems to
523 * help lock contention somewhat, and this is critical for HTT. -Peter
524 */
525 static int cpu_idle_hlt = 1;
526 SYSCTL_INT(_machdep, OID_AUTO, cpu_idle_hlt, CTLFLAG_RW,
527 &cpu_idle_hlt, 0, "Idle loop HLT enable");
528
529 static void
530 cpu_idle_default(void)
531 {
532 /*
533 * we must absolutely guarentee that hlt is the
534 * absolute next instruction after sti or we
535 * introduce a timing window.
536 */
537 __asm __volatile("sti; hlt");
538 }
539
540 /*
541 * Note that we have to be careful here to avoid a race between checking
542 * sched_runnable() and actually halting. If we don't do this, we may waste
543 * the time between calling hlt and the next interrupt even though there
544 * is a runnable process.
545 */
546 void
547 cpu_idle(void)
548 {
549
550 #ifdef SMP
551 if (mp_grab_cpu_hlt())
552 return;
553 #endif
554 if (cpu_idle_hlt) {
555 disable_intr();
556 if (sched_runnable())
557 enable_intr();
558 else
559 (*cpu_idle_hook)();
560 }
561 }
562
563 /* Other subsystems (e.g., ACPI) can hook this later. */
564 void (*cpu_idle_hook)(void) = cpu_idle_default;
565
566 /*
567 * Clear registers on exec
568 */
569 void
570 exec_setregs(td, entry, stack, ps_strings)
571 struct thread *td;
572 u_long entry;
573 u_long stack;
574 u_long ps_strings;
575 {
576 struct trapframe *regs = td->td_frame;
577 struct pcb *pcb = td->td_pcb;
578
579 wrmsr(MSR_FSBASE, 0);
580 wrmsr(MSR_KGSBASE, 0); /* User value while we're in the kernel */
581 pcb->pcb_fsbase = 0;
582 pcb->pcb_gsbase = 0;
583 load_ds(_udatasel);
584 load_es(_udatasel);
585 load_fs(_udatasel);
586 load_gs(_udatasel);
587 pcb->pcb_ds = _udatasel;
588 pcb->pcb_es = _udatasel;
589 pcb->pcb_fs = _udatasel;
590 pcb->pcb_gs = _udatasel;
591
592 bzero((char *)regs, sizeof(struct trapframe));
593 regs->tf_rip = entry;
594 regs->tf_rsp = ((stack - 8) & ~0xFul) + 8;
595 regs->tf_rdi = stack; /* argv */
596 regs->tf_rflags = PSL_USER | (regs->tf_rflags & PSL_T);
597 regs->tf_ss = _udatasel;
598 regs->tf_cs = _ucodesel;
599
600 /*
601 * Reset the hardware debug registers if they were in use.
602 * They won't have any meaning for the newly exec'd process.
603 */
604 if (pcb->pcb_flags & PCB_DBREGS) {
605 pcb->pcb_dr0 = 0;
606 pcb->pcb_dr1 = 0;
607 pcb->pcb_dr2 = 0;
608 pcb->pcb_dr3 = 0;
609 pcb->pcb_dr6 = 0;
610 pcb->pcb_dr7 = 0;
611 if (pcb == PCPU_GET(curpcb)) {
612 /*
613 * Clear the debug registers on the running
614 * CPU, otherwise they will end up affecting
615 * the next process we switch to.
616 */
617 reset_dbregs();
618 }
619 pcb->pcb_flags &= ~PCB_DBREGS;
620 }
621
622 /*
623 * Drop the FP state if we hold it, so that the process gets a
624 * clean FP state if it uses the FPU again.
625 */
626 fpstate_drop(td);
627 }
628
629 void
630 cpu_setregs(void)
631 {
632 register_t cr0;
633
634 cr0 = rcr0();
635 /*
636 * CR0_MP, CR0_NE and CR0_TS are also set by npx_probe() for the
637 * BSP. See the comments there about why we set them.
638 */
639 cr0 |= CR0_MP | CR0_NE | CR0_TS | CR0_WP | CR0_AM;
640 load_cr0(cr0);
641 }
642
643 static int
644 sysctl_machdep_adjkerntz(SYSCTL_HANDLER_ARGS)
645 {
646 int error;
647 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2,
648 req);
649 if (!error && req->newptr)
650 resettodr();
651 return (error);
652 }
653
654 SYSCTL_PROC(_machdep, CPU_ADJKERNTZ, adjkerntz, CTLTYPE_INT|CTLFLAG_RW,
655 &adjkerntz, 0, sysctl_machdep_adjkerntz, "I", "");
656
657 SYSCTL_INT(_machdep, CPU_DISRTCSET, disable_rtc_set,
658 CTLFLAG_RW, &disable_rtc_set, 0, "");
659
660 SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock,
661 CTLFLAG_RW, &wall_cmos_clock, 0, "");
662
663 /*
664 * Initialize amd64 and configure to run kernel
665 */
666
667 /*
668 * Initialize segments & interrupt table
669 */
670
671 struct user_segment_descriptor gdt[NGDT * MAXCPU];/* global descriptor table */
672 static struct gate_descriptor idt0[NIDT];
673 struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */
674
675 static char dblfault_stack[PAGE_SIZE] __aligned(16);
676
677 struct amd64tss common_tss[MAXCPU];
678
679 /* software prototypes -- in more palatable form */
680 struct soft_segment_descriptor gdt_segs[] = {
681 /* GNULL_SEL 0 Null Descriptor */
682 { 0x0, /* segment base address */
683 0x0, /* length */
684 0, /* segment type */
685 0, /* segment descriptor priority level */
686 0, /* segment descriptor present */
687 0, /* long */
688 0, /* default 32 vs 16 bit size */
689 0 /* limit granularity (byte/page units)*/ },
690 /* GCODE_SEL 1 Code Descriptor for kernel */
691 { 0x0, /* segment base address */
692 0xfffff, /* length - all address space */
693 SDT_MEMERA, /* segment type */
694 SEL_KPL, /* segment descriptor priority level */
695 1, /* segment descriptor present */
696 1, /* long */
697 0, /* default 32 vs 16 bit size */
698 1 /* limit granularity (byte/page units)*/ },
699 /* GDATA_SEL 2 Data Descriptor for kernel */
700 { 0x0, /* segment base address */
701 0xfffff, /* length - all address space */
702 SDT_MEMRWA, /* segment type */
703 SEL_KPL, /* segment descriptor priority level */
704 1, /* segment descriptor present */
705 1, /* long */
706 0, /* default 32 vs 16 bit size */
707 1 /* limit granularity (byte/page units)*/ },
708 /* GUCODE32_SEL 3 32 bit Code Descriptor for user */
709 { 0x0, /* segment base address */
710 0xfffff, /* length - all address space */
711 SDT_MEMERA, /* segment type */
712 SEL_UPL, /* segment descriptor priority level */
713 1, /* segment descriptor present */
714 0, /* long */
715 1, /* default 32 vs 16 bit size */
716 1 /* limit granularity (byte/page units)*/ },
717 /* GUDATA_SEL 4 32/64 bit Data Descriptor for user */
718 { 0x0, /* segment base address */
719 0xfffff, /* length - all address space */
720 SDT_MEMRWA, /* segment type */
721 SEL_UPL, /* segment descriptor priority level */
722 1, /* segment descriptor present */
723 0, /* long */
724 1, /* default 32 vs 16 bit size */
725 1 /* limit granularity (byte/page units)*/ },
726 /* GUCODE_SEL 5 64 bit Code Descriptor for user */
727 { 0x0, /* segment base address */
728 0xfffff, /* length - all address space */
729 SDT_MEMERA, /* segment type */
730 SEL_UPL, /* segment descriptor priority level */
731 1, /* segment descriptor present */
732 1, /* long */
733 0, /* default 32 vs 16 bit size */
734 1 /* limit granularity (byte/page units)*/ },
735 /* GPROC0_SEL 6 Proc 0 Tss Descriptor */
736 {
737 0x0, /* segment base address */
738 sizeof(struct amd64tss)-1,/* length - all address space */
739 SDT_SYSTSS, /* segment type */
740 SEL_KPL, /* segment descriptor priority level */
741 1, /* segment descriptor present */
742 0, /* long */
743 0, /* unused - default 32 vs 16 bit size */
744 0 /* limit granularity (byte/page units)*/ },
745 /* Actually, the TSS is a system descriptor which is double size */
746 { 0x0, /* segment base address */
747 0x0, /* length */
748 0, /* segment type */
749 0, /* segment descriptor priority level */
750 0, /* segment descriptor present */
751 0, /* long */
752 0, /* default 32 vs 16 bit size */
753 0 /* limit granularity (byte/page units)*/ },
754 };
755
756 void
757 setidt(idx, func, typ, dpl, ist)
758 int idx;
759 inthand_t *func;
760 int typ;
761 int dpl;
762 int ist;
763 {
764 struct gate_descriptor *ip;
765
766 ip = idt + idx;
767 ip->gd_looffset = (uintptr_t)func;
768 ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL);
769 ip->gd_ist = ist;
770 ip->gd_xx = 0;
771 ip->gd_type = typ;
772 ip->gd_dpl = dpl;
773 ip->gd_p = 1;
774 ip->gd_hioffset = ((uintptr_t)func)>>16 ;
775 }
776
777 #define IDTVEC(name) __CONCAT(X,name)
778
779 extern inthand_t
780 IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
781 IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
782 IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
783 IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
784 IDTVEC(xmm), IDTVEC(dblfault),
785 IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
786
787 void
788 sdtossd(sd, ssd)
789 struct user_segment_descriptor *sd;
790 struct soft_segment_descriptor *ssd;
791 {
792
793 ssd->ssd_base = (sd->sd_hibase << 24) | sd->sd_lobase;
794 ssd->ssd_limit = (sd->sd_hilimit << 16) | sd->sd_lolimit;
795 ssd->ssd_type = sd->sd_type;
796 ssd->ssd_dpl = sd->sd_dpl;
797 ssd->ssd_p = sd->sd_p;
798 ssd->ssd_long = sd->sd_long;
799 ssd->ssd_def32 = sd->sd_def32;
800 ssd->ssd_gran = sd->sd_gran;
801 }
802
803 void
804 ssdtosd(ssd, sd)
805 struct soft_segment_descriptor *ssd;
806 struct user_segment_descriptor *sd;
807 {
808
809 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
810 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xff;
811 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
812 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
813 sd->sd_type = ssd->ssd_type;
814 sd->sd_dpl = ssd->ssd_dpl;
815 sd->sd_p = ssd->ssd_p;
816 sd->sd_long = ssd->ssd_long;
817 sd->sd_def32 = ssd->ssd_def32;
818 sd->sd_gran = ssd->ssd_gran;
819 }
820
821 void
822 ssdtosyssd(ssd, sd)
823 struct soft_segment_descriptor *ssd;
824 struct system_segment_descriptor *sd;
825 {
826
827 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
828 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xfffffffffful;
829 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
830 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
831 sd->sd_type = ssd->ssd_type;
832 sd->sd_dpl = ssd->ssd_dpl;
833 sd->sd_p = ssd->ssd_p;
834 sd->sd_gran = ssd->ssd_gran;
835 }
836
837 #if !defined(DEV_ATPIC) && defined(DEV_ISA)
838 #include <isa/isavar.h>
839 u_int
840 isa_irq_pending(void)
841 {
842
843 return (0);
844 }
845 #endif
846
847 #define PHYSMAP_SIZE (2 * 8)
848
849 u_int basemem;
850
851 /*
852 * Populate the (physmap) array with base/bound pairs describing the
853 * available physical memory in the system, then test this memory and
854 * build the phys_avail array describing the actually-available memory.
855 *
856 * If we cannot accurately determine the physical memory map, then use
857 * value from the 0xE801 call, and failing that, the RTC.
858 *
859 * Total memory size may be set by the kernel environment variable
860 * hw.physmem or the compile-time define MAXMEM.
861 *
862 * XXX first should be vm_paddr_t.
863 */
864 static void
865 getmemsize(caddr_t kmdp, u_int64_t first)
866 {
867 int i, physmap_idx, pa_indx;
868 vm_paddr_t pa, physmap[PHYSMAP_SIZE];
869 pt_entry_t *pte;
870 char *cp;
871 struct bios_smap *smapbase, *smap, *smapend;
872 u_int32_t smapsize;
873 quad_t dcons_addr, dcons_size;
874
875 bzero(physmap, sizeof(physmap));
876 basemem = 0;
877 physmap_idx = 0;
878
879 /*
880 * get memory map from INT 15:E820, kindly supplied by the loader.
881 *
882 * subr_module.c says:
883 * "Consumer may safely assume that size value precedes data."
884 * ie: an int32_t immediately precedes smap.
885 */
886 smapbase = (struct bios_smap *)preload_search_info(kmdp,
887 MODINFO_METADATA | MODINFOMD_SMAP);
888 if (smapbase == NULL)
889 panic("No BIOS smap info from loader!");
890
891 smapsize = *((u_int32_t *)smapbase - 1);
892 smapend = (struct bios_smap *)((uintptr_t)smapbase + smapsize);
893
894 for (smap = smapbase; smap < smapend; smap++) {
895 if (boothowto & RB_VERBOSE)
896 printf("SMAP type=%02x base=%016lx len=%016lx\n",
897 smap->type, smap->base, smap->length);
898
899 if (smap->type != 0x01)
900 continue;
901
902 if (smap->length == 0)
903 continue;
904
905 for (i = 0; i <= physmap_idx; i += 2) {
906 if (smap->base < physmap[i + 1]) {
907 if (boothowto & RB_VERBOSE)
908 printf(
909 "Overlapping or non-montonic memory region, ignoring second region\n");
910 goto next_run;
911 }
912 }
913
914 if (smap->base == physmap[physmap_idx + 1]) {
915 physmap[physmap_idx + 1] += smap->length;
916 next_run:
917 continue;
918 }
919
920 physmap_idx += 2;
921 if (physmap_idx == PHYSMAP_SIZE) {
922 printf(
923 "Too many segments in the physical address map, giving up\n");
924 break;
925 }
926 physmap[physmap_idx] = smap->base;
927 physmap[physmap_idx + 1] = smap->base + smap->length;
928 }
929
930 /*
931 * Find the 'base memory' segment for SMP
932 */
933 basemem = 0;
934 for (i = 0; i <= physmap_idx; i += 2) {
935 if (physmap[i] == 0x00000000) {
936 basemem = physmap[i + 1] / 1024;
937 break;
938 }
939 }
940 if (basemem == 0)
941 panic("BIOS smap did not include a basemem segment!");
942
943 #ifdef SMP
944 /* make hole for AP bootstrap code */
945 physmap[1] = mp_bootaddress(physmap[1] / 1024);
946 #endif
947
948 /*
949 * Maxmem isn't the "maximum memory", it's one larger than the
950 * highest page of the physical address space. It should be
951 * called something like "Maxphyspage". We may adjust this
952 * based on ``hw.physmem'' and the results of the memory test.
953 */
954 Maxmem = atop(physmap[physmap_idx + 1]);
955
956 #ifdef MAXMEM
957 Maxmem = MAXMEM / 4;
958 #endif
959
960 /*
961 * hw.physmem is a size in bytes; we also allow k, m, and g suffixes
962 * for the appropriate modifiers. This overrides MAXMEM.
963 */
964 cp = getenv("hw.physmem");
965 if (cp != NULL) {
966 u_int64_t AllowMem, sanity;
967 char *ep;
968
969 sanity = AllowMem = strtouq(cp, &ep, 0);
970 if ((ep != cp) && (*ep != 0)) {
971 switch(*ep) {
972 case 'g':
973 case 'G':
974 AllowMem <<= 10;
975 case 'm':
976 case 'M':
977 AllowMem <<= 10;
978 case 'k':
979 case 'K':
980 AllowMem <<= 10;
981 break;
982 default:
983 AllowMem = sanity = 0;
984 }
985 if (AllowMem < sanity)
986 AllowMem = 0;
987 }
988 if (AllowMem == 0)
989 printf("Ignoring invalid memory size of '%s'\n", cp);
990 else
991 Maxmem = atop(AllowMem);
992 freeenv(cp);
993 }
994
995 if (atop(physmap[physmap_idx + 1]) != Maxmem &&
996 (boothowto & RB_VERBOSE))
997 printf("Physical memory use set to %ldK\n", Maxmem * 4);
998
999 /*
1000 * If Maxmem has been increased beyond what the system has detected,
1001 * extend the last memory segment to the new limit.
1002 */
1003 if (atop(physmap[physmap_idx + 1]) < Maxmem)
1004 physmap[physmap_idx + 1] = ptoa((vm_paddr_t)Maxmem);
1005
1006 /* call pmap initialization to make new kernel address space */
1007 pmap_bootstrap(&first);
1008
1009 /*
1010 * Size up each available chunk of physical memory.
1011 */
1012 physmap[0] = PAGE_SIZE; /* mask off page 0 */
1013 pa_indx = 0;
1014 phys_avail[pa_indx++] = physmap[0];
1015 phys_avail[pa_indx] = physmap[0];
1016 pte = CMAP1;
1017
1018 /*
1019 * Get dcons buffer address
1020 */
1021 if (getenv_quad("dcons.addr", &dcons_addr) == 0 ||
1022 getenv_quad("dcons.size", &dcons_size) == 0)
1023 dcons_addr = 0;
1024
1025 /*
1026 * physmap is in bytes, so when converting to page boundaries,
1027 * round up the start address and round down the end address.
1028 */
1029 for (i = 0; i <= physmap_idx; i += 2) {
1030 vm_paddr_t end;
1031
1032 end = ptoa((vm_paddr_t)Maxmem);
1033 if (physmap[i + 1] < end)
1034 end = trunc_page(physmap[i + 1]);
1035 for (pa = round_page(physmap[i]); pa < end; pa += PAGE_SIZE) {
1036 int tmp, page_bad;
1037 int *ptr = (int *)CADDR1;
1038
1039 /*
1040 * block out kernel memory as not available.
1041 */
1042 if (pa >= 0x100000 && pa < first)
1043 continue;
1044
1045 /*
1046 * block out dcons buffer
1047 */
1048 if (dcons_addr > 0
1049 && pa >= trunc_page(dcons_addr)
1050 && pa < dcons_addr + dcons_size)
1051 continue;
1052
1053 page_bad = FALSE;
1054
1055 /*
1056 * map page into kernel: valid, read/write,non-cacheable
1057 */
1058 *pte = pa | PG_V | PG_RW | PG_N;
1059 invltlb();
1060
1061 tmp = *(int *)ptr;
1062 /*
1063 * Test for alternating 1's and 0's
1064 */
1065 *(volatile int *)ptr = 0xaaaaaaaa;
1066 if (*(volatile int *)ptr != 0xaaaaaaaa)
1067 page_bad = TRUE;
1068 /*
1069 * Test for alternating 0's and 1's
1070 */
1071 *(volatile int *)ptr = 0x55555555;
1072 if (*(volatile int *)ptr != 0x55555555)
1073 page_bad = TRUE;
1074 /*
1075 * Test for all 1's
1076 */
1077 *(volatile int *)ptr = 0xffffffff;
1078 if (*(volatile int *)ptr != 0xffffffff)
1079 page_bad = TRUE;
1080 /*
1081 * Test for all 0's
1082 */
1083 *(volatile int *)ptr = 0x0;
1084 if (*(volatile int *)ptr != 0x0)
1085 page_bad = TRUE;
1086 /*
1087 * Restore original value.
1088 */
1089 *(int *)ptr = tmp;
1090
1091 /*
1092 * Adjust array of valid/good pages.
1093 */
1094 if (page_bad == TRUE)
1095 continue;
1096 /*
1097 * If this good page is a continuation of the
1098 * previous set of good pages, then just increase
1099 * the end pointer. Otherwise start a new chunk.
1100 * Note that "end" points one higher than end,
1101 * making the range >= start and < end.
1102 * If we're also doing a speculative memory
1103 * test and we at or past the end, bump up Maxmem
1104 * so that we keep going. The first bad page
1105 * will terminate the loop.
1106 */
1107 if (phys_avail[pa_indx] == pa) {
1108 phys_avail[pa_indx] += PAGE_SIZE;
1109 } else {
1110 pa_indx++;
1111 if (pa_indx == PHYS_AVAIL_ARRAY_END) {
1112 printf(
1113 "Too many holes in the physical address space, giving up\n");
1114 pa_indx--;
1115 break;
1116 }
1117 phys_avail[pa_indx++] = pa; /* start */
1118 phys_avail[pa_indx] = pa + PAGE_SIZE; /* end */
1119 }
1120 physmem++;
1121 }
1122 }
1123 *pte = 0;
1124 invltlb();
1125
1126 /*
1127 * XXX
1128 * The last chunk must contain at least one page plus the message
1129 * buffer to avoid complicating other code (message buffer address
1130 * calculation, etc.).
1131 */
1132 while (phys_avail[pa_indx - 1] + PAGE_SIZE +
1133 round_page(MSGBUF_SIZE) >= phys_avail[pa_indx]) {
1134 physmem -= atop(phys_avail[pa_indx] - phys_avail[pa_indx - 1]);
1135 phys_avail[pa_indx--] = 0;
1136 phys_avail[pa_indx--] = 0;
1137 }
1138
1139 Maxmem = atop(phys_avail[pa_indx]);
1140
1141 /* Trim off space for the message buffer. */
1142 phys_avail[pa_indx] -= round_page(MSGBUF_SIZE);
1143
1144 avail_end = phys_avail[pa_indx];
1145 }
1146
1147 u_int64_t
1148 hammer_time(u_int64_t modulep, u_int64_t physfree)
1149 {
1150 caddr_t kmdp;
1151 int gsel_tss, off, x;
1152 struct pcpu *pc;
1153 u_int64_t msr;
1154 char *env;
1155
1156 #ifdef DEV_ISA
1157 /* Preemptively mask the atpics and leave them shut down */
1158 outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff);
1159 outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff);
1160 #else
1161 #error "have you forgotten the isa device?";
1162 #endif
1163
1164 thread0.td_kstack = physfree + KERNBASE;
1165 bzero((void *)thread0.td_kstack, KSTACK_PAGES * PAGE_SIZE);
1166 physfree += KSTACK_PAGES * PAGE_SIZE;
1167 thread0.td_pcb = (struct pcb *)
1168 (thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1;
1169
1170 /*
1171 * This may be done better later if it gets more high level
1172 * components in it. If so just link td->td_proc here.
1173 */
1174 proc_linkup(&proc0, &ksegrp0, &thread0);
1175
1176 preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);
1177 preload_bootstrap_relocate(KERNBASE);
1178 kmdp = preload_search_by_type("elf kernel");
1179 if (kmdp == NULL)
1180 kmdp = preload_search_by_type("elf64 kernel");
1181 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
1182 kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *) + KERNBASE;
1183 #ifdef DDB
1184 ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
1185 ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
1186 #endif
1187
1188 /* Init basic tunables, hz etc */
1189 init_param1();
1190
1191 /*
1192 * make gdt memory segments
1193 */
1194 gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&common_tss[0];
1195
1196 for (x = 0; x < NGDT; x++) {
1197 if (x != GPROC0_SEL && x != (GPROC0_SEL + 1))
1198 ssdtosd(&gdt_segs[x], &gdt[x]);
1199 }
1200 ssdtosyssd(&gdt_segs[GPROC0_SEL],
1201 (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
1202
1203 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
1204 r_gdt.rd_base = (long) gdt;
1205 lgdt(&r_gdt);
1206 pc = &__pcpu[0];
1207
1208 wrmsr(MSR_FSBASE, 0); /* User value */
1209 wrmsr(MSR_GSBASE, (u_int64_t)pc);
1210 wrmsr(MSR_KGSBASE, 0); /* User value while in the kernel */
1211
1212 pcpu_init(pc, 0, sizeof(struct pcpu));
1213 PCPU_SET(prvspace, pc);
1214 PCPU_SET(curthread, &thread0);
1215 PCPU_SET(curpcb, thread0.td_pcb);
1216 PCPU_SET(tssp, &common_tss[0]);
1217
1218 /*
1219 * Initialize mutexes.
1220 *
1221 * icu_lock: in order to allow an interrupt to occur in a critical
1222 * section, to set pcpu->ipending (etc...) properly, we
1223 * must be able to get the icu lock, so it can't be
1224 * under witness.
1225 */
1226 mutex_init();
1227 mtx_init(&clock_lock, "clk", NULL, MTX_SPIN);
1228 mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS);
1229
1230 /* exceptions */
1231 for (x = 0; x < NIDT; x++)
1232 setidt(x, &IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0);
1233 setidt(IDT_DE, &IDTVEC(div), SDT_SYSIGT, SEL_KPL, 0);
1234 setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 0);
1235 setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 0);
1236 setidt(IDT_BP, &IDTVEC(bpt), SDT_SYSIGT, SEL_UPL, 0);
1237 setidt(IDT_OF, &IDTVEC(ofl), SDT_SYSIGT, SEL_KPL, 0);
1238 setidt(IDT_BR, &IDTVEC(bnd), SDT_SYSIGT, SEL_KPL, 0);
1239 setidt(IDT_UD, &IDTVEC(ill), SDT_SYSIGT, SEL_KPL, 0);
1240 setidt(IDT_NM, &IDTVEC(dna), SDT_SYSIGT, SEL_KPL, 0);
1241 setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);
1242 setidt(IDT_FPUGP, &IDTVEC(fpusegm), SDT_SYSIGT, SEL_KPL, 0);
1243 setidt(IDT_TS, &IDTVEC(tss), SDT_SYSIGT, SEL_KPL, 0);
1244 setidt(IDT_NP, &IDTVEC(missing), SDT_SYSIGT, SEL_KPL, 0);
1245 setidt(IDT_SS, &IDTVEC(stk), SDT_SYSIGT, SEL_KPL, 0);
1246 setidt(IDT_GP, &IDTVEC(prot), SDT_SYSIGT, SEL_KPL, 0);
1247 setidt(IDT_PF, &IDTVEC(page), SDT_SYSIGT, SEL_KPL, 0);
1248 setidt(IDT_MF, &IDTVEC(fpu), SDT_SYSIGT, SEL_KPL, 0);
1249 setidt(IDT_AC, &IDTVEC(align), SDT_SYSIGT, SEL_KPL, 0);
1250 setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 0);
1251 setidt(IDT_XF, &IDTVEC(xmm), SDT_SYSIGT, SEL_KPL, 0);
1252
1253 r_idt.rd_limit = sizeof(idt0) - 1;
1254 r_idt.rd_base = (long) idt;
1255 lidt(&r_idt);
1256
1257 /*
1258 * Initialize the console before we print anything out.
1259 */
1260 cninit();
1261
1262 #ifdef DEV_ATPIC
1263 elcr_probe();
1264 atpic_startup();
1265 #endif
1266
1267 kdb_init();
1268
1269 #ifdef KDB
1270 if (boothowto & RB_KDB)
1271 kdb_enter("Boot flags requested debugger");
1272 #endif
1273
1274 identify_cpu(); /* Final stage of CPU initialization */
1275 initializecpu(); /* Initialize CPU registers */
1276
1277 /* make an initial tss so cpu can get interrupt stack on syscall! */
1278 common_tss[0].tss_rsp0 = thread0.td_kstack + \
1279 KSTACK_PAGES * PAGE_SIZE - sizeof(struct pcb);
1280 /* Ensure the stack is aligned to 16 bytes */
1281 common_tss[0].tss_rsp0 &= ~0xFul;
1282 PCPU_SET(rsp0, common_tss[0].tss_rsp0);
1283
1284 /* doublefault stack space, runs on ist1 */
1285 common_tss[0].tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)];
1286
1287 /* Set the IO permission bitmap (empty due to tss seg limit) */
1288 common_tss[0].tss_iobase = sizeof(struct amd64tss);
1289
1290 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
1291 ltr(gsel_tss);
1292
1293 /* Set up the fast syscall stuff */
1294 msr = rdmsr(MSR_EFER) | EFER_SCE;
1295 wrmsr(MSR_EFER, msr);
1296 wrmsr(MSR_LSTAR, (u_int64_t)IDTVEC(fast_syscall));
1297 wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
1298 msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
1299 ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
1300 wrmsr(MSR_STAR, msr);
1301 wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);
1302
1303 getmemsize(kmdp, physfree);
1304 init_param2(physmem);
1305
1306 /* now running on new page tables, configured,and u/iom is accessible */
1307
1308 /* Map the message buffer. */
1309 for (off = 0; off < round_page(MSGBUF_SIZE); off += PAGE_SIZE)
1310 pmap_kenter((vm_offset_t)msgbufp + off, avail_end + off);
1311
1312 msgbufinit(msgbufp, MSGBUF_SIZE);
1313 fpuinit();
1314
1315 /* transfer to user mode */
1316
1317 _ucodesel = GSEL(GUCODE_SEL, SEL_UPL);
1318 _udatasel = GSEL(GUDATA_SEL, SEL_UPL);
1319 _ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL);
1320
1321 /* setup proc 0's pcb */
1322 thread0.td_pcb->pcb_flags = 0; /* XXXKSE */
1323 thread0.td_pcb->pcb_cr3 = KPML4phys;
1324 thread0.td_frame = &proc0_tf;
1325
1326 env = getenv("kernelname");
1327 if (env != NULL)
1328 strlcpy(kernelname, env, sizeof(kernelname));
1329
1330 /* Location of kernel stack for locore */
1331 return ((u_int64_t)thread0.td_pcb);
1332 }
1333
1334 void
1335 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
1336 {
1337
1338 pcpu->pc_acpi_id = 0xffffffff;
1339 }
1340
1341 /*
1342 * Construct a PCB from a trapframe. This is called from kdb_trap() where
1343 * we want to start a backtrace from the function that caused us to enter
1344 * the debugger. We have the context in the trapframe, but base the trace
1345 * on the PCB. The PCB doesn't have to be perfect, as long as it contains
1346 * enough for a backtrace.
1347 */
1348 void
1349 makectx(struct trapframe *tf, struct pcb *pcb)
1350 {
1351
1352 pcb->pcb_r12 = tf->tf_r12;
1353 pcb->pcb_r13 = tf->tf_r13;
1354 pcb->pcb_r14 = tf->tf_r14;
1355 pcb->pcb_r15 = tf->tf_r15;
1356 pcb->pcb_rbp = tf->tf_rbp;
1357 pcb->pcb_rbx = tf->tf_rbx;
1358 pcb->pcb_rip = tf->tf_rip;
1359 pcb->pcb_rsp = (ISPL(tf->tf_cs)) ? tf->tf_rsp : (long)(tf + 1) - 8;
1360 }
1361
1362 int
1363 ptrace_set_pc(struct thread *td, unsigned long addr)
1364 {
1365 td->td_frame->tf_rip = addr;
1366 return (0);
1367 }
1368
1369 int
1370 ptrace_single_step(struct thread *td)
1371 {
1372 td->td_frame->tf_rflags |= PSL_T;
1373 return (0);
1374 }
1375
1376 int
1377 ptrace_clear_single_step(struct thread *td)
1378 {
1379 td->td_frame->tf_rflags &= ~PSL_T;
1380 return (0);
1381 }
1382
1383 int
1384 fill_regs(struct thread *td, struct reg *regs)
1385 {
1386 struct trapframe *tp;
1387
1388 tp = td->td_frame;
1389 regs->r_r15 = tp->tf_r15;
1390 regs->r_r14 = tp->tf_r14;
1391 regs->r_r13 = tp->tf_r13;
1392 regs->r_r12 = tp->tf_r12;
1393 regs->r_r11 = tp->tf_r11;
1394 regs->r_r10 = tp->tf_r10;
1395 regs->r_r9 = tp->tf_r9;
1396 regs->r_r8 = tp->tf_r8;
1397 regs->r_rdi = tp->tf_rdi;
1398 regs->r_rsi = tp->tf_rsi;
1399 regs->r_rbp = tp->tf_rbp;
1400 regs->r_rbx = tp->tf_rbx;
1401 regs->r_rdx = tp->tf_rdx;
1402 regs->r_rcx = tp->tf_rcx;
1403 regs->r_rax = tp->tf_rax;
1404 regs->r_rip = tp->tf_rip;
1405 regs->r_cs = tp->tf_cs;
1406 regs->r_rflags = tp->tf_rflags;
1407 regs->r_rsp = tp->tf_rsp;
1408 regs->r_ss = tp->tf_ss;
1409 return (0);
1410 }
1411
1412 int
1413 set_regs(struct thread *td, struct reg *regs)
1414 {
1415 struct trapframe *tp;
1416 register_t rflags;
1417
1418 tp = td->td_frame;
1419 rflags = regs->r_rflags & 0xffffffff;
1420 if (!EFL_SECURE(rflags, tp->tf_rflags) || !CS_SECURE(regs->r_cs))
1421 return (EINVAL);
1422 tp->tf_r15 = regs->r_r15;
1423 tp->tf_r14 = regs->r_r14;
1424 tp->tf_r13 = regs->r_r13;
1425 tp->tf_r12 = regs->r_r12;
1426 tp->tf_r11 = regs->r_r11;
1427 tp->tf_r10 = regs->r_r10;
1428 tp->tf_r9 = regs->r_r9;
1429 tp->tf_r8 = regs->r_r8;
1430 tp->tf_rdi = regs->r_rdi;
1431 tp->tf_rsi = regs->r_rsi;
1432 tp->tf_rbp = regs->r_rbp;
1433 tp->tf_rbx = regs->r_rbx;
1434 tp->tf_rdx = regs->r_rdx;
1435 tp->tf_rcx = regs->r_rcx;
1436 tp->tf_rax = regs->r_rax;
1437 tp->tf_rip = regs->r_rip;
1438 tp->tf_cs = regs->r_cs;
1439 tp->tf_rflags = rflags;
1440 tp->tf_rsp = regs->r_rsp;
1441 tp->tf_ss = regs->r_ss;
1442 td->td_pcb->pcb_flags |= PCB_FULLCTX;
1443 return (0);
1444 }
1445
1446 /* XXX check all this stuff! */
1447 /* externalize from sv_xmm */
1448 static void
1449 fill_fpregs_xmm(struct savefpu *sv_xmm, struct fpreg *fpregs)
1450 {
1451 struct envxmm *penv_fpreg = (struct envxmm *)&fpregs->fpr_env;
1452 struct envxmm *penv_xmm = &sv_xmm->sv_env;
1453 int i;
1454
1455 /* pcb -> fpregs */
1456 bzero(fpregs, sizeof(*fpregs));
1457
1458 /* FPU control/status */
1459 penv_fpreg->en_cw = penv_xmm->en_cw;
1460 penv_fpreg->en_sw = penv_xmm->en_sw;
1461 penv_fpreg->en_tw = penv_xmm->en_tw;
1462 penv_fpreg->en_opcode = penv_xmm->en_opcode;
1463 penv_fpreg->en_rip = penv_xmm->en_rip;
1464 penv_fpreg->en_rdp = penv_xmm->en_rdp;
1465 penv_fpreg->en_mxcsr = penv_xmm->en_mxcsr;
1466 penv_fpreg->en_mxcsr_mask = penv_xmm->en_mxcsr_mask;
1467
1468 /* FPU registers */
1469 for (i = 0; i < 8; ++i)
1470 bcopy(sv_xmm->sv_fp[i].fp_acc.fp_bytes, fpregs->fpr_acc[i], 10);
1471
1472 /* SSE registers */
1473 for (i = 0; i < 16; ++i)
1474 bcopy(sv_xmm->sv_xmm[i].xmm_bytes, fpregs->fpr_xacc[i], 16);
1475 }
1476
1477 /* internalize from fpregs into sv_xmm */
1478 static void
1479 set_fpregs_xmm(struct fpreg *fpregs, struct savefpu *sv_xmm)
1480 {
1481 struct envxmm *penv_xmm = &sv_xmm->sv_env;
1482 struct envxmm *penv_fpreg = (struct envxmm *)&fpregs->fpr_env;
1483 int i;
1484
1485 /* fpregs -> pcb */
1486 /* FPU control/status */
1487 penv_xmm->en_cw = penv_fpreg->en_cw;
1488 penv_xmm->en_sw = penv_fpreg->en_sw;
1489 penv_xmm->en_tw = penv_fpreg->en_tw;
1490 penv_xmm->en_opcode = penv_fpreg->en_opcode;
1491 penv_xmm->en_rip = penv_fpreg->en_rip;
1492 penv_xmm->en_rdp = penv_fpreg->en_rdp;
1493 penv_xmm->en_mxcsr = penv_fpreg->en_mxcsr;
1494 penv_xmm->en_mxcsr_mask = penv_fpreg->en_mxcsr_mask;
1495
1496 /* FPU registers */
1497 for (i = 0; i < 8; ++i)
1498 bcopy(fpregs->fpr_acc[i], sv_xmm->sv_fp[i].fp_acc.fp_bytes, 10);
1499
1500 /* SSE registers */
1501 for (i = 0; i < 16; ++i)
1502 bcopy(fpregs->fpr_xacc[i], sv_xmm->sv_xmm[i].xmm_bytes, 16);
1503 }
1504
1505 /* externalize from td->pcb */
1506 int
1507 fill_fpregs(struct thread *td, struct fpreg *fpregs)
1508 {
1509
1510 fill_fpregs_xmm(&td->td_pcb->pcb_save, fpregs);
1511 return (0);
1512 }
1513
1514 /* internalize to td->pcb */
1515 int
1516 set_fpregs(struct thread *td, struct fpreg *fpregs)
1517 {
1518
1519 set_fpregs_xmm(fpregs, &td->td_pcb->pcb_save);
1520 return (0);
1521 }
1522
1523 /*
1524 * Get machine context.
1525 */
1526 int
1527 get_mcontext(struct thread *td, mcontext_t *mcp, int flags)
1528 {
1529 struct trapframe *tp;
1530
1531 tp = td->td_frame;
1532 PROC_LOCK(curthread->td_proc);
1533 mcp->mc_onstack = sigonstack(tp->tf_rsp);
1534 PROC_UNLOCK(curthread->td_proc);
1535 mcp->mc_r15 = tp->tf_r15;
1536 mcp->mc_r14 = tp->tf_r14;
1537 mcp->mc_r13 = tp->tf_r13;
1538 mcp->mc_r12 = tp->tf_r12;
1539 mcp->mc_r11 = tp->tf_r11;
1540 mcp->mc_r10 = tp->tf_r10;
1541 mcp->mc_r9 = tp->tf_r9;
1542 mcp->mc_r8 = tp->tf_r8;
1543 mcp->mc_rdi = tp->tf_rdi;
1544 mcp->mc_rsi = tp->tf_rsi;
1545 mcp->mc_rbp = tp->tf_rbp;
1546 mcp->mc_rbx = tp->tf_rbx;
1547 mcp->mc_rcx = tp->tf_rcx;
1548 mcp->mc_rflags = tp->tf_rflags;
1549 if (flags & GET_MC_CLEAR_RET) {
1550 mcp->mc_rax = 0;
1551 mcp->mc_rdx = 0;
1552 mcp->mc_rflags &= ~PSL_C;
1553 } else {
1554 mcp->mc_rax = tp->tf_rax;
1555 mcp->mc_rdx = tp->tf_rdx;
1556 }
1557 mcp->mc_rip = tp->tf_rip;
1558 mcp->mc_cs = tp->tf_cs;
1559 mcp->mc_rsp = tp->tf_rsp;
1560 mcp->mc_ss = tp->tf_ss;
1561 mcp->mc_len = sizeof(*mcp);
1562 get_fpcontext(td, mcp);
1563 return (0);
1564 }
1565
1566 /*
1567 * Set machine context.
1568 *
1569 * However, we don't set any but the user modifiable flags, and we won't
1570 * touch the cs selector.
1571 */
1572 int
1573 set_mcontext(struct thread *td, const mcontext_t *mcp)
1574 {
1575 struct trapframe *tp;
1576 long rflags;
1577 int ret;
1578
1579 tp = td->td_frame;
1580 if (mcp->mc_len != sizeof(*mcp))
1581 return (EINVAL);
1582 rflags = (mcp->mc_rflags & PSL_USERCHANGE) |
1583 (tp->tf_rflags & ~PSL_USERCHANGE);
1584 ret = set_fpcontext(td, mcp);
1585 if (ret != 0)
1586 return (ret);
1587 tp->tf_r15 = mcp->mc_r15;
1588 tp->tf_r14 = mcp->mc_r14;
1589 tp->tf_r13 = mcp->mc_r13;
1590 tp->tf_r12 = mcp->mc_r12;
1591 tp->tf_r11 = mcp->mc_r11;
1592 tp->tf_r10 = mcp->mc_r10;
1593 tp->tf_r9 = mcp->mc_r9;
1594 tp->tf_r8 = mcp->mc_r8;
1595 tp->tf_rdi = mcp->mc_rdi;
1596 tp->tf_rsi = mcp->mc_rsi;
1597 tp->tf_rbp = mcp->mc_rbp;
1598 tp->tf_rbx = mcp->mc_rbx;
1599 tp->tf_rdx = mcp->mc_rdx;
1600 tp->tf_rcx = mcp->mc_rcx;
1601 tp->tf_rax = mcp->mc_rax;
1602 tp->tf_rip = mcp->mc_rip;
1603 tp->tf_rflags = rflags;
1604 tp->tf_rsp = mcp->mc_rsp;
1605 tp->tf_ss = mcp->mc_ss;
1606 td->td_pcb->pcb_flags |= PCB_FULLCTX;
1607 return (0);
1608 }
1609
1610 static void
1611 get_fpcontext(struct thread *td, mcontext_t *mcp)
1612 {
1613
1614 mcp->mc_ownedfp = fpugetregs(td, (struct savefpu *)&mcp->mc_fpstate);
1615 mcp->mc_fpformat = fpuformat();
1616 }
1617
1618 static int
1619 set_fpcontext(struct thread *td, const mcontext_t *mcp)
1620 {
1621
1622 if (mcp->mc_fpformat == _MC_FPFMT_NODEV)
1623 return (0);
1624 else if (mcp->mc_fpformat != _MC_FPFMT_XMM)
1625 return (EINVAL);
1626 else if (mcp->mc_ownedfp == _MC_FPOWNED_NONE)
1627 /* We don't care what state is left in the FPU or PCB. */
1628 fpstate_drop(td);
1629 else if (mcp->mc_ownedfp == _MC_FPOWNED_FPU ||
1630 mcp->mc_ownedfp == _MC_FPOWNED_PCB) {
1631 /*
1632 * XXX we violate the dubious requirement that fpusetregs()
1633 * be called with interrupts disabled.
1634 * XXX obsolete on trap-16 systems?
1635 */
1636 fpusetregs(td, (struct savefpu *)&mcp->mc_fpstate);
1637 } else
1638 return (EINVAL);
1639 return (0);
1640 }
1641
1642 void
1643 fpstate_drop(struct thread *td)
1644 {
1645 register_t s;
1646
1647 s = intr_disable();
1648 if (PCPU_GET(fpcurthread) == td)
1649 fpudrop();
1650 /*
1651 * XXX force a full drop of the fpu. The above only drops it if we
1652 * owned it.
1653 *
1654 * XXX I don't much like fpugetregs()'s semantics of doing a full
1655 * drop. Dropping only to the pcb matches fnsave's behaviour.
1656 * We only need to drop to !PCB_INITDONE in sendsig(). But
1657 * sendsig() is the only caller of fpugetregs()... perhaps we just
1658 * have too many layers.
1659 */
1660 curthread->td_pcb->pcb_flags &= ~PCB_FPUINITDONE;
1661 intr_restore(s);
1662 }
1663
1664 int
1665 fill_dbregs(struct thread *td, struct dbreg *dbregs)
1666 {
1667 struct pcb *pcb;
1668
1669 if (td == NULL) {
1670 dbregs->dr[0] = rdr0();
1671 dbregs->dr[1] = rdr1();
1672 dbregs->dr[2] = rdr2();
1673 dbregs->dr[3] = rdr3();
1674 dbregs->dr[6] = rdr6();
1675 dbregs->dr[7] = rdr7();
1676 } else {
1677 pcb = td->td_pcb;
1678 dbregs->dr[0] = pcb->pcb_dr0;
1679 dbregs->dr[1] = pcb->pcb_dr1;
1680 dbregs->dr[2] = pcb->pcb_dr2;
1681 dbregs->dr[3] = pcb->pcb_dr3;
1682 dbregs->dr[6] = pcb->pcb_dr6;
1683 dbregs->dr[7] = pcb->pcb_dr7;
1684 }
1685 dbregs->dr[4] = 0;
1686 dbregs->dr[5] = 0;
1687 dbregs->dr[8] = 0;
1688 dbregs->dr[9] = 0;
1689 dbregs->dr[10] = 0;
1690 dbregs->dr[11] = 0;
1691 dbregs->dr[12] = 0;
1692 dbregs->dr[13] = 0;
1693 dbregs->dr[14] = 0;
1694 dbregs->dr[15] = 0;
1695 return (0);
1696 }
1697
1698 int
1699 set_dbregs(struct thread *td, struct dbreg *dbregs)
1700 {
1701 struct pcb *pcb;
1702 int i;
1703 u_int64_t mask1, mask2;
1704
1705 if (td == NULL) {
1706 load_dr0(dbregs->dr[0]);
1707 load_dr1(dbregs->dr[1]);
1708 load_dr2(dbregs->dr[2]);
1709 load_dr3(dbregs->dr[3]);
1710 load_dr6(dbregs->dr[6]);
1711 load_dr7(dbregs->dr[7]);
1712 } else {
1713 /*
1714 * Don't let an illegal value for dr7 get set. Specifically,
1715 * check for undefined settings. Setting these bit patterns
1716 * result in undefined behaviour and can lead to an unexpected
1717 * TRCTRAP or a general protection fault right here.
1718 */
1719 for (i = 0, mask1 = 0x3<<16, mask2 = 0x2<<16; i < 8;
1720 i++, mask1 <<= 2, mask2 <<= 2)
1721 if ((dbregs->dr[7] & mask1) == mask2)
1722 return (EINVAL);
1723
1724 pcb = td->td_pcb;
1725
1726 /*
1727 * Don't let a process set a breakpoint that is not within the
1728 * process's address space. If a process could do this, it
1729 * could halt the system by setting a breakpoint in the kernel
1730 * (if ddb was enabled). Thus, we need to check to make sure
1731 * that no breakpoints are being enabled for addresses outside
1732 * process's address space, unless, perhaps, we were called by
1733 * uid 0.
1734 *
1735 * XXX - what about when the watched area of the user's
1736 * address space is written into from within the kernel
1737 * ... wouldn't that still cause a breakpoint to be generated
1738 * from within kernel mode?
1739 */
1740
1741 if (suser(td) != 0) {
1742 if (dbregs->dr[7] & 0x3) {
1743 /* dr0 is enabled */
1744 if (dbregs->dr[0] >= VM_MAXUSER_ADDRESS)
1745 return (EINVAL);
1746 }
1747 if (dbregs->dr[7] & 0x3<<2) {
1748 /* dr1 is enabled */
1749 if (dbregs->dr[1] >= VM_MAXUSER_ADDRESS)
1750 return (EINVAL);
1751 }
1752 if (dbregs->dr[7] & 0x3<<4) {
1753 /* dr2 is enabled */
1754 if (dbregs->dr[2] >= VM_MAXUSER_ADDRESS)
1755 return (EINVAL);
1756 }
1757 if (dbregs->dr[7] & 0x3<<6) {
1758 /* dr3 is enabled */
1759 if (dbregs->dr[3] >= VM_MAXUSER_ADDRESS)
1760 return (EINVAL);
1761 }
1762 }
1763
1764 pcb->pcb_dr0 = dbregs->dr[0];
1765 pcb->pcb_dr1 = dbregs->dr[1];
1766 pcb->pcb_dr2 = dbregs->dr[2];
1767 pcb->pcb_dr3 = dbregs->dr[3];
1768 pcb->pcb_dr6 = dbregs->dr[6];
1769 pcb->pcb_dr7 = dbregs->dr[7];
1770
1771 pcb->pcb_flags |= PCB_DBREGS;
1772 }
1773
1774 return (0);
1775 }
1776
1777 void
1778 reset_dbregs(void)
1779 {
1780
1781 load_dr7(0); /* Turn off the control bits first */
1782 load_dr0(0);
1783 load_dr1(0);
1784 load_dr2(0);
1785 load_dr3(0);
1786 load_dr6(0);
1787 }
1788
1789 /*
1790 * Return > 0 if a hardware breakpoint has been hit, and the
1791 * breakpoint was in user space. Return 0, otherwise.
1792 */
1793 int
1794 user_dbreg_trap(void)
1795 {
1796 u_int64_t dr7, dr6; /* debug registers dr6 and dr7 */
1797 u_int64_t bp; /* breakpoint bits extracted from dr6 */
1798 int nbp; /* number of breakpoints that triggered */
1799 caddr_t addr[4]; /* breakpoint addresses */
1800 int i;
1801
1802 dr7 = rdr7();
1803 if ((dr7 & 0x000000ff) == 0) {
1804 /*
1805 * all GE and LE bits in the dr7 register are zero,
1806 * thus the trap couldn't have been caused by the
1807 * hardware debug registers
1808 */
1809 return 0;
1810 }
1811
1812 nbp = 0;
1813 dr6 = rdr6();
1814 bp = dr6 & 0x0000000f;
1815
1816 if (!bp) {
1817 /*
1818 * None of the breakpoint bits are set meaning this
1819 * trap was not caused by any of the debug registers
1820 */
1821 return 0;
1822 }
1823
1824 /*
1825 * at least one of the breakpoints were hit, check to see
1826 * which ones and if any of them are user space addresses
1827 */
1828
1829 if (bp & 0x01) {
1830 addr[nbp++] = (caddr_t)rdr0();
1831 }
1832 if (bp & 0x02) {
1833 addr[nbp++] = (caddr_t)rdr1();
1834 }
1835 if (bp & 0x04) {
1836 addr[nbp++] = (caddr_t)rdr2();
1837 }
1838 if (bp & 0x08) {
1839 addr[nbp++] = (caddr_t)rdr3();
1840 }
1841
1842 for (i=0; i<nbp; i++) {
1843 if (addr[i] <
1844 (caddr_t)VM_MAXUSER_ADDRESS) {
1845 /*
1846 * addr[i] is in user space
1847 */
1848 return nbp;
1849 }
1850 }
1851
1852 /*
1853 * None of the breakpoints are in user space.
1854 */
1855 return 0;
1856 }
1857
1858 #ifdef KDB
1859
1860 /*
1861 * Provide inb() and outb() as functions. They are normally only
1862 * available as macros calling inlined functions, thus cannot be
1863 * called from the debugger.
1864 *
1865 * The actual code is stolen from <machine/cpufunc.h>, and de-inlined.
1866 */
1867
1868 #undef inb
1869 #undef outb
1870
1871 /* silence compiler warnings */
1872 u_char inb(u_int);
1873 void outb(u_int, u_char);
1874
1875 u_char
1876 inb(u_int port)
1877 {
1878 u_char data;
1879 /*
1880 * We use %%dx and not %1 here because i/o is done at %dx and not at
1881 * %edx, while gcc generates inferior code (movw instead of movl)
1882 * if we tell it to load (u_short) port.
1883 */
1884 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
1885 return (data);
1886 }
1887
1888 void
1889 outb(u_int port, u_char data)
1890 {
1891 u_char al;
1892 /*
1893 * Use an unnecessary assignment to help gcc's register allocator.
1894 * This make a large difference for gcc-1.40 and a tiny difference
1895 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for
1896 * best results. gcc-2.6.0 can't handle this.
1897 */
1898 al = data;
1899 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
1900 }
1901
1902 #endif /* KDB */
Cache object: dc4600629d2d4d18002aa13eb4406514
|