1 /*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1992 Terrence R. Lambert.
4 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to Berkeley by
8 * William Jolitz.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the University of
21 * California, Berkeley and its contributors.
22 * 4. Neither the name of the University nor the names of its contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91
39 */
40
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
43
44 #include "opt_atalk.h"
45 #include "opt_atpic.h"
46 #include "opt_compat.h"
47 #include "opt_cpu.h"
48 #include "opt_ddb.h"
49 #include "opt_inet.h"
50 #include "opt_ipx.h"
51 #include "opt_isa.h"
52 #include "opt_kstack_pages.h"
53 #include "opt_maxmem.h"
54 #include "opt_msgbuf.h"
55 #include "opt_perfmon.h"
56
57 #include <sys/param.h>
58 #include <sys/proc.h>
59 #include <sys/systm.h>
60 #include <sys/bio.h>
61 #include <sys/buf.h>
62 #include <sys/bus.h>
63 #include <sys/callout.h>
64 #include <sys/clock.h>
65 #include <sys/cons.h>
66 #include <sys/cpu.h>
67 #include <sys/eventhandler.h>
68 #include <sys/exec.h>
69 #include <sys/imgact.h>
70 #include <sys/kdb.h>
71 #include <sys/kernel.h>
72 #include <sys/ktr.h>
73 #include <sys/linker.h>
74 #include <sys/lock.h>
75 #include <sys/malloc.h>
76 #include <sys/memrange.h>
77 #include <sys/msgbuf.h>
78 #include <sys/mutex.h>
79 #include <sys/pcpu.h>
80 #include <sys/ptrace.h>
81 #include <sys/reboot.h>
82 #include <sys/sched.h>
83 #include <sys/signalvar.h>
84 #include <sys/sysctl.h>
85 #include <sys/sysent.h>
86 #include <sys/sysproto.h>
87 #include <sys/ucontext.h>
88 #include <sys/vmmeter.h>
89
90 #include <vm/vm.h>
91 #include <vm/vm_extern.h>
92 #include <vm/vm_kern.h>
93 #include <vm/vm_page.h>
94 #include <vm/vm_map.h>
95 #include <vm/vm_object.h>
96 #include <vm/vm_pager.h>
97 #include <vm/vm_param.h>
98
99 #ifdef DDB
100 #ifndef KDB
101 #error KDB must be enabled in order for DDB to work!
102 #endif
103 #endif
104 #include <ddb/ddb.h>
105
106 #include <net/netisr.h>
107
108 #include <machine/clock.h>
109 #include <machine/cpu.h>
110 #include <machine/cputypes.h>
111 #include <machine/intr_machdep.h>
112 #include <machine/mca.h>
113 #include <machine/md_var.h>
114 #include <machine/metadata.h>
115 #include <machine/pc/bios.h>
116 #include <machine/pcb.h>
117 #include <machine/proc.h>
118 #include <machine/reg.h>
119 #include <machine/sigframe.h>
120 #include <machine/specialreg.h>
121 #ifdef PERFMON
122 #include <machine/perfmon.h>
123 #endif
124 #include <machine/tss.h>
125 #ifdef SMP
126 #include <machine/smp.h>
127 #endif
128
129 #ifdef DEV_ATPIC
130 #include <amd64/isa/icu.h>
131 #else
132 #include <machine/apicvar.h>
133 #endif
134
135 #include <isa/isareg.h>
136 #include <isa/rtc.h>
137
138 /* Sanity check for __curthread() */
139 CTASSERT(offsetof(struct pcpu, pc_curthread) == 0);
140
141 extern u_int64_t hammer_time(u_int64_t, u_int64_t);
142
143 extern void printcpuinfo(void); /* XXX header file */
144 extern void identify_cpu(void);
145 extern void panicifcpuunsupported(void);
146
147 #define CS_SECURE(cs) (ISPL(cs) == SEL_UPL)
148 #define EFL_SECURE(ef, oef) ((((ef) ^ (oef)) & ~PSL_USERCHANGE) == 0)
149
150 static void cpu_startup(void *);
151 static void get_fpcontext(struct thread *td, mcontext_t *mcp);
152 static int set_fpcontext(struct thread *td, const mcontext_t *mcp);
153 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
154
155 #ifdef DDB
156 extern vm_offset_t ksym_start, ksym_end;
157 #endif
158
159 /* Intel ICH registers */
160 #define ICH_PMBASE 0x400
161 #define ICH_SMI_EN ICH_PMBASE + 0x30
162
163 int _udatasel, _ucodesel, _ucode32sel;
164
165 int cold = 1;
166
167 long Maxmem = 0;
168 long realmem = 0;
169
170 /*
171 * The number of PHYSMAP entries must be one less than the number of
172 * PHYSSEG entries because the PHYSMAP entry that spans the largest
173 * physical address that is accessible by ISA DMA is split into two
174 * PHYSSEG entries.
175 */
176 #define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1))
177
178 vm_paddr_t phys_avail[PHYSMAP_SIZE + 2];
179 vm_paddr_t dump_avail[PHYSMAP_SIZE + 2];
180
181 /* must be 2 less so 0 0 can signal end of chunks */
182 #define PHYS_AVAIL_ARRAY_END ((sizeof(phys_avail) / sizeof(phys_avail[0])) - 2)
183 #define DUMP_AVAIL_ARRAY_END ((sizeof(dump_avail) / sizeof(dump_avail[0])) - 2)
184
185 struct kva_md_info kmi;
186
187 static struct trapframe proc0_tf;
188 struct region_descriptor r_gdt, r_idt;
189
190 struct pcpu __pcpu[MAXCPU];
191
192 struct mtx icu_lock;
193
194 struct mem_range_softc mem_range_softc;
195
196 static void
197 cpu_startup(dummy)
198 void *dummy;
199 {
200 char *sysenv;
201
202 /*
203 * On MacBooks, we need to disallow the legacy USB circuit to
204 * generate an SMI# because this can cause several problems,
205 * namely: incorrect CPU frequency detection and failure to
206 * start the APs.
207 * We do this by disabling a bit in the SMI_EN (SMI Control and
208 * Enable register) of the Intel ICH LPC Interface Bridge.
209 */
210 sysenv = getenv("smbios.system.product");
211 if (sysenv != NULL) {
212 if (strncmp(sysenv, "MacBook", 7) == 0) {
213 if (bootverbose)
214 printf("Disabling LEGACY_USB_EN bit on "
215 "Intel ICH.\n");
216 outl(ICH_SMI_EN, inl(ICH_SMI_EN) & ~0x8);
217 }
218 freeenv(sysenv);
219 }
220
221 /*
222 * Good {morning,afternoon,evening,night}.
223 */
224 startrtclock();
225 printcpuinfo();
226 panicifcpuunsupported();
227 #ifdef PERFMON
228 perfmon_init();
229 #endif
230 printf("usable memory = %ju (%ju MB)\n", ptoa((uintmax_t)physmem),
231 ptoa((uintmax_t)physmem) / 1048576);
232 realmem = Maxmem;
233 /*
234 * Display any holes after the first chunk of extended memory.
235 */
236 if (bootverbose) {
237 int indx;
238
239 printf("Physical memory chunk(s):\n");
240 for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) {
241 vm_paddr_t size;
242
243 size = phys_avail[indx + 1] - phys_avail[indx];
244 printf(
245 "0x%016jx - 0x%016jx, %ju bytes (%ju pages)\n",
246 (uintmax_t)phys_avail[indx],
247 (uintmax_t)phys_avail[indx + 1] - 1,
248 (uintmax_t)size, (uintmax_t)size / PAGE_SIZE);
249 }
250 }
251
252 vm_ksubmap_init(&kmi);
253
254 printf("avail memory = %ju (%ju MB)\n",
255 ptoa((uintmax_t)cnt.v_free_count),
256 ptoa((uintmax_t)cnt.v_free_count) / 1048576);
257
258 /*
259 * Set up buffers, so they can be used to read disk labels.
260 */
261 bufinit();
262 vm_pager_bufferinit();
263
264 cpu_setregs();
265 }
266
267 /*
268 * Send an interrupt to process.
269 *
270 * Stack is set up to allow sigcode stored
271 * at top to call routine, followed by kcall
272 * to sigreturn routine below. After sigreturn
273 * resets the signal mask, the stack, and the
274 * frame pointer, it returns to the user
275 * specified pc, psl.
276 */
277 void
278 sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
279 {
280 struct sigframe sf, *sfp;
281 struct proc *p;
282 struct thread *td;
283 struct sigacts *psp;
284 char *sp;
285 struct trapframe *regs;
286 int sig;
287 int oonstack;
288
289 td = curthread;
290 p = td->td_proc;
291 PROC_LOCK_ASSERT(p, MA_OWNED);
292 sig = ksi->ksi_signo;
293 psp = p->p_sigacts;
294 mtx_assert(&psp->ps_mtx, MA_OWNED);
295 regs = td->td_frame;
296 oonstack = sigonstack(regs->tf_rsp);
297
298 /* Save user context. */
299 bzero(&sf, sizeof(sf));
300 sf.sf_uc.uc_sigmask = *mask;
301 sf.sf_uc.uc_stack = td->td_sigstk;
302 sf.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK)
303 ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE;
304 sf.sf_uc.uc_mcontext.mc_onstack = (oonstack) ? 1 : 0;
305 bcopy(regs, &sf.sf_uc.uc_mcontext.mc_rdi, sizeof(*regs));
306 sf.sf_uc.uc_mcontext.mc_len = sizeof(sf.sf_uc.uc_mcontext); /* magic */
307 get_fpcontext(td, &sf.sf_uc.uc_mcontext);
308 fpstate_drop(td);
309 bzero(sf.sf_uc.uc_mcontext.mc_spare,
310 sizeof(sf.sf_uc.uc_mcontext.mc_spare));
311 bzero(sf.sf_uc.__spare__, sizeof(sf.sf_uc.__spare__));
312
313 /* Allocate space for the signal handler context. */
314 if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
315 SIGISMEMBER(psp->ps_sigonstack, sig)) {
316 sp = td->td_sigstk.ss_sp +
317 td->td_sigstk.ss_size - sizeof(struct sigframe);
318 #if defined(COMPAT_43)
319 td->td_sigstk.ss_flags |= SS_ONSTACK;
320 #endif
321 } else
322 sp = (char *)regs->tf_rsp - sizeof(struct sigframe) - 128;
323 /* Align to 16 bytes. */
324 sfp = (struct sigframe *)((unsigned long)sp & ~0xFul);
325
326 /* Translate the signal if appropriate. */
327 if (p->p_sysent->sv_sigtbl && sig <= p->p_sysent->sv_sigsize)
328 sig = p->p_sysent->sv_sigtbl[_SIG_IDX(sig)];
329
330 /* Build the argument list for the signal handler. */
331 regs->tf_rdi = sig; /* arg 1 in %rdi */
332 regs->tf_rdx = (register_t)&sfp->sf_uc; /* arg 3 in %rdx */
333 bzero(&sf.sf_si, sizeof(sf.sf_si));
334 if (SIGISMEMBER(psp->ps_siginfo, sig)) {
335 /* Signal handler installed with SA_SIGINFO. */
336 regs->tf_rsi = (register_t)&sfp->sf_si; /* arg 2 in %rsi */
337 sf.sf_ahu.sf_action = (__siginfohandler_t *)catcher;
338
339 /* Fill in POSIX parts */
340 sf.sf_si = ksi->ksi_info;
341 sf.sf_si.si_signo = sig; /* maybe a translated signal */
342 regs->tf_rcx = (register_t)ksi->ksi_addr; /* arg 4 in %rcx */
343 } else {
344 /* Old FreeBSD-style arguments. */
345 regs->tf_rsi = ksi->ksi_code; /* arg 2 in %rsi */
346 regs->tf_rcx = (register_t)ksi->ksi_addr; /* arg 4 in %rcx */
347 sf.sf_ahu.sf_handler = catcher;
348 }
349 mtx_unlock(&psp->ps_mtx);
350 PROC_UNLOCK(p);
351
352 /*
353 * Copy the sigframe out to the user's stack.
354 */
355 if (copyout(&sf, sfp, sizeof(*sfp)) != 0) {
356 #ifdef DEBUG
357 printf("process %ld has trashed its stack\n", (long)p->p_pid);
358 #endif
359 PROC_LOCK(p);
360 sigexit(td, SIGILL);
361 }
362
363 regs->tf_rsp = (long)sfp;
364 regs->tf_rip = PS_STRINGS - *(p->p_sysent->sv_szsigcode);
365 regs->tf_rflags &= ~(PSL_T | PSL_D);
366 regs->tf_cs = _ucodesel;
367 PROC_LOCK(p);
368 mtx_lock(&psp->ps_mtx);
369 }
370
371 /*
372 * System call to cleanup state after a signal
373 * has been taken. Reset signal mask and
374 * stack state from context left by sendsig (above).
375 * Return to previous pc and psl as specified by
376 * context left by sendsig. Check carefully to
377 * make sure that the user has not modified the
378 * state to gain improper privileges.
379 *
380 * MPSAFE
381 */
382 int
383 sigreturn(td, uap)
384 struct thread *td;
385 struct sigreturn_args /* {
386 const struct __ucontext *sigcntxp;
387 } */ *uap;
388 {
389 ucontext_t uc;
390 struct proc *p = td->td_proc;
391 struct trapframe *regs;
392 const ucontext_t *ucp;
393 long rflags;
394 int cs, error, ret;
395 ksiginfo_t ksi;
396
397 error = copyin(uap->sigcntxp, &uc, sizeof(uc));
398 if (error != 0)
399 return (error);
400 ucp = &uc;
401 regs = td->td_frame;
402 rflags = ucp->uc_mcontext.mc_rflags;
403 /*
404 * Don't allow users to change privileged or reserved flags.
405 */
406 /*
407 * XXX do allow users to change the privileged flag PSL_RF.
408 * The cpu sets PSL_RF in tf_rflags for faults. Debuggers
409 * should sometimes set it there too. tf_rflags is kept in
410 * the signal context during signal handling and there is no
411 * other place to remember it, so the PSL_RF bit may be
412 * corrupted by the signal handler without us knowing.
413 * Corruption of the PSL_RF bit at worst causes one more or
414 * one less debugger trap, so allowing it is fairly harmless.
415 */
416 if (!EFL_SECURE(rflags & ~PSL_RF, regs->tf_rflags & ~PSL_RF)) {
417 printf("sigreturn: rflags = 0x%lx\n", rflags);
418 return (EINVAL);
419 }
420
421 /*
422 * Don't allow users to load a valid privileged %cs. Let the
423 * hardware check for invalid selectors, excess privilege in
424 * other selectors, invalid %eip's and invalid %esp's.
425 */
426 cs = ucp->uc_mcontext.mc_cs;
427 if (!CS_SECURE(cs)) {
428 printf("sigreturn: cs = 0x%x\n", cs);
429 ksiginfo_init_trap(&ksi);
430 ksi.ksi_signo = SIGBUS;
431 ksi.ksi_code = BUS_OBJERR;
432 ksi.ksi_trapno = T_PROTFLT;
433 ksi.ksi_addr = (void *)regs->tf_rip;
434 trapsignal(td, &ksi);
435 return (EINVAL);
436 }
437
438 ret = set_fpcontext(td, &ucp->uc_mcontext);
439 if (ret != 0)
440 return (ret);
441 bcopy(&ucp->uc_mcontext.mc_rdi, regs, sizeof(*regs));
442
443 PROC_LOCK(p);
444 #if defined(COMPAT_43)
445 if (ucp->uc_mcontext.mc_onstack & 1)
446 td->td_sigstk.ss_flags |= SS_ONSTACK;
447 else
448 td->td_sigstk.ss_flags &= ~SS_ONSTACK;
449 #endif
450
451 td->td_sigmask = ucp->uc_sigmask;
452 SIG_CANTMASK(td->td_sigmask);
453 signotify(td);
454 PROC_UNLOCK(p);
455 td->td_pcb->pcb_flags |= PCB_FULLCTX;
456 return (EJUSTRETURN);
457 }
458
459 #ifdef COMPAT_FREEBSD4
460 int
461 freebsd4_sigreturn(struct thread *td, struct freebsd4_sigreturn_args *uap)
462 {
463
464 return sigreturn(td, (struct sigreturn_args *)uap);
465 }
466 #endif
467
468
469 /*
470 * Machine dependent boot() routine
471 *
472 * I haven't seen anything to put here yet
473 * Possibly some stuff might be grafted back here from boot()
474 */
475 void
476 cpu_boot(int howto)
477 {
478 }
479
480 /* Get current clock frequency for the given cpu id. */
481 int
482 cpu_est_clockrate(int cpu_id, uint64_t *rate)
483 {
484 register_t reg;
485 uint64_t tsc1, tsc2;
486
487 if (pcpu_find(cpu_id) == NULL || rate == NULL)
488 return (EINVAL);
489
490 /* If we're booting, trust the rate calibrated moments ago. */
491 if (cold) {
492 *rate = tsc_freq;
493 return (0);
494 }
495
496 #ifdef SMP
497 /* Schedule ourselves on the indicated cpu. */
498 thread_lock(curthread);
499 sched_bind(curthread, cpu_id);
500 thread_unlock(curthread);
501 #endif
502
503 /* Calibrate by measuring a short delay. */
504 reg = intr_disable();
505 tsc1 = rdtsc();
506 DELAY(1000);
507 tsc2 = rdtsc();
508 intr_restore(reg);
509
510 #ifdef SMP
511 thread_lock(curthread);
512 sched_unbind(curthread);
513 thread_unlock(curthread);
514 #endif
515
516 /*
517 * Calculate the difference in readings, convert to Mhz, and
518 * subtract 0.5% of the total. Empirical testing has shown that
519 * overhead in DELAY() works out to approximately this value.
520 */
521 tsc2 -= tsc1;
522 *rate = tsc2 * 1000 - tsc2 * 5;
523 return (0);
524 }
525
526 /*
527 * Shutdown the CPU as much as possible
528 */
529 void
530 cpu_halt(void)
531 {
532 for (;;)
533 __asm__ ("hlt");
534 }
535
536 /*
537 * Hook to idle the CPU when possible. In the SMP case we default to
538 * off because a halted cpu will not currently pick up a new thread in the
539 * run queue until the next timer tick. If turned on this will result in
540 * approximately a 4.2% loss in real time performance in buildworld tests
541 * (but improves user and sys times oddly enough), and saves approximately
542 * 5% in power consumption on an idle machine (tests w/2xCPU 1.1GHz P3).
543 *
544 * XXX we need to have a cpu mask of idle cpus and generate an IPI or
545 * otherwise generate some sort of interrupt to wake up cpus sitting in HLT.
546 * Then we can have our cake and eat it too.
547 *
548 * XXX I'm turning it on for SMP as well by default for now. It seems to
549 * help lock contention somewhat, and this is critical for HTT. -Peter
550 */
551 static int cpu_idle_hlt = 1;
552 TUNABLE_INT("machdep.cpu_idle_hlt", &cpu_idle_hlt);
553 SYSCTL_INT(_machdep, OID_AUTO, cpu_idle_hlt, CTLFLAG_RW,
554 &cpu_idle_hlt, 0, "Idle loop HLT enable");
555
556 static void
557 cpu_idle_default(void)
558 {
559 /*
560 * we must absolutely guarentee that hlt is the
561 * absolute next instruction after sti or we
562 * introduce a timing window.
563 */
564 __asm __volatile("sti; hlt");
565 }
566
567 /*
568 * Note that we have to be careful here to avoid a race between checking
569 * sched_runnable() and actually halting. If we don't do this, we may waste
570 * the time between calling hlt and the next interrupt even though there
571 * is a runnable process.
572 */
573 void
574 cpu_idle(void)
575 {
576
577 #ifdef SMP
578 if (mp_grab_cpu_hlt())
579 return;
580 #endif
581 if (cpu_idle_hlt) {
582 disable_intr();
583 if (sched_runnable())
584 enable_intr();
585 else
586 (*cpu_idle_hook)();
587 }
588 }
589
590 /* Other subsystems (e.g., ACPI) can hook this later. */
591 void (*cpu_idle_hook)(void) = cpu_idle_default;
592
593 /*
594 * Reset registers to default values on exec.
595 */
596 void
597 exec_setregs(td, entry, stack, ps_strings)
598 struct thread *td;
599 u_long entry;
600 u_long stack;
601 u_long ps_strings;
602 {
603 struct trapframe *regs = td->td_frame;
604 struct pcb *pcb = td->td_pcb;
605
606 critical_enter();
607 wrmsr(MSR_FSBASE, 0);
608 wrmsr(MSR_KGSBASE, 0); /* User value while we're in the kernel */
609 pcb->pcb_fsbase = 0;
610 pcb->pcb_gsbase = 0;
611 critical_exit();
612 pcb->pcb_flags &= ~(PCB_32BIT | PCB_GS32BIT);
613 load_ds(_udatasel);
614 load_es(_udatasel);
615 load_fs(_udatasel);
616 load_gs(_udatasel);
617 pcb->pcb_ds = _udatasel;
618 pcb->pcb_es = _udatasel;
619 pcb->pcb_fs = _udatasel;
620 pcb->pcb_gs = _udatasel;
621 pcb->pcb_initial_fpucw = __INITIAL_FPUCW__;
622
623 bzero((char *)regs, sizeof(struct trapframe));
624 regs->tf_rip = entry;
625 regs->tf_rsp = ((stack - 8) & ~0xFul) + 8;
626 regs->tf_rdi = stack; /* argv */
627 regs->tf_rflags = PSL_USER | (regs->tf_rflags & PSL_T);
628 regs->tf_ss = _udatasel;
629 regs->tf_cs = _ucodesel;
630
631 /*
632 * Reset the hardware debug registers if they were in use.
633 * They won't have any meaning for the newly exec'd process.
634 */
635 if (pcb->pcb_flags & PCB_DBREGS) {
636 pcb->pcb_dr0 = 0;
637 pcb->pcb_dr1 = 0;
638 pcb->pcb_dr2 = 0;
639 pcb->pcb_dr3 = 0;
640 pcb->pcb_dr6 = 0;
641 pcb->pcb_dr7 = 0;
642 if (pcb == PCPU_GET(curpcb)) {
643 /*
644 * Clear the debug registers on the running
645 * CPU, otherwise they will end up affecting
646 * the next process we switch to.
647 */
648 reset_dbregs();
649 }
650 pcb->pcb_flags &= ~PCB_DBREGS;
651 }
652
653 /*
654 * Drop the FP state if we hold it, so that the process gets a
655 * clean FP state if it uses the FPU again.
656 */
657 fpstate_drop(td);
658 }
659
660 void
661 cpu_setregs(void)
662 {
663 register_t cr0;
664
665 cr0 = rcr0();
666 /*
667 * CR0_MP, CR0_NE and CR0_TS are also set by npx_probe() for the
668 * BSP. See the comments there about why we set them.
669 */
670 cr0 |= CR0_MP | CR0_NE | CR0_TS | CR0_WP | CR0_AM;
671 load_cr0(cr0);
672 }
673
674 /*
675 * Initialize amd64 and configure to run kernel
676 */
677
678 /*
679 * Initialize segments & interrupt table
680 */
681
682 struct user_segment_descriptor gdt[NGDT * MAXCPU];/* global descriptor tables */
683 static struct gate_descriptor idt0[NIDT];
684 struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */
685
686 static char dblfault_stack[PAGE_SIZE] __aligned(16);
687
688 static char nmi0_stack[PAGE_SIZE] __aligned(16);
689 CTASSERT(sizeof(struct nmi_pcpu) == 16);
690
691 struct amd64tss common_tss[MAXCPU];
692
693 /* software prototypes -- in more palatable form */
694 struct soft_segment_descriptor gdt_segs[] = {
695 /* GNULL_SEL 0 Null Descriptor */
696 { 0x0, /* segment base address */
697 0x0, /* length */
698 0, /* segment type */
699 0, /* segment descriptor priority level */
700 0, /* segment descriptor present */
701 0, /* long */
702 0, /* default 32 vs 16 bit size */
703 0 /* limit granularity (byte/page units)*/ },
704 /* GCODE_SEL 1 Code Descriptor for kernel */
705 { 0x0, /* segment base address */
706 0xfffff, /* length - all address space */
707 SDT_MEMERA, /* segment type */
708 SEL_KPL, /* segment descriptor priority level */
709 1, /* segment descriptor present */
710 1, /* long */
711 0, /* default 32 vs 16 bit size */
712 1 /* limit granularity (byte/page units)*/ },
713 /* GDATA_SEL 2 Data Descriptor for kernel */
714 { 0x0, /* segment base address */
715 0xfffff, /* length - all address space */
716 SDT_MEMRWA, /* segment type */
717 SEL_KPL, /* segment descriptor priority level */
718 1, /* segment descriptor present */
719 1, /* long */
720 0, /* default 32 vs 16 bit size */
721 1 /* limit granularity (byte/page units)*/ },
722 /* GUCODE32_SEL 3 32 bit Code Descriptor for user */
723 { 0x0, /* segment base address */
724 0xfffff, /* length - all address space */
725 SDT_MEMERA, /* segment type */
726 SEL_UPL, /* segment descriptor priority level */
727 1, /* segment descriptor present */
728 0, /* long */
729 1, /* default 32 vs 16 bit size */
730 1 /* limit granularity (byte/page units)*/ },
731 /* GUDATA_SEL 4 32/64 bit Data Descriptor for user */
732 { 0x0, /* segment base address */
733 0xfffff, /* length - all address space */
734 SDT_MEMRWA, /* segment type */
735 SEL_UPL, /* segment descriptor priority level */
736 1, /* segment descriptor present */
737 0, /* long */
738 1, /* default 32 vs 16 bit size */
739 1 /* limit granularity (byte/page units)*/ },
740 /* GUCODE_SEL 5 64 bit Code Descriptor for user */
741 { 0x0, /* segment base address */
742 0xfffff, /* length - all address space */
743 SDT_MEMERA, /* segment type */
744 SEL_UPL, /* segment descriptor priority level */
745 1, /* segment descriptor present */
746 1, /* long */
747 0, /* default 32 vs 16 bit size */
748 1 /* limit granularity (byte/page units)*/ },
749 /* GPROC0_SEL 6 Proc 0 Tss Descriptor */
750 {
751 0x0, /* segment base address */
752 sizeof(struct amd64tss)-1,/* length */
753 SDT_SYSTSS, /* segment type */
754 SEL_KPL, /* segment descriptor priority level */
755 1, /* segment descriptor present */
756 0, /* long */
757 0, /* unused - default 32 vs 16 bit size */
758 0 /* limit granularity (byte/page units)*/ },
759 /* Actually, the TSS is a system descriptor which is double size */
760 { 0x0, /* segment base address */
761 0x0, /* length */
762 0, /* segment type */
763 0, /* segment descriptor priority level */
764 0, /* segment descriptor present */
765 0, /* long */
766 0, /* default 32 vs 16 bit size */
767 0 /* limit granularity (byte/page units)*/ },
768 /* GUGS32_SEL 8 32 bit GS Descriptor for user */
769 { 0x0, /* segment base address */
770 0xfffff, /* length - all address space */
771 SDT_MEMRWA, /* segment type */
772 SEL_UPL, /* segment descriptor priority level */
773 1, /* segment descriptor present */
774 0, /* long */
775 1, /* default 32 vs 16 bit size */
776 1 /* limit granularity (byte/page units)*/ },
777 };
778
779 void
780 setidt(idx, func, typ, dpl, ist)
781 int idx;
782 inthand_t *func;
783 int typ;
784 int dpl;
785 int ist;
786 {
787 struct gate_descriptor *ip;
788
789 ip = idt + idx;
790 ip->gd_looffset = (uintptr_t)func;
791 ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL);
792 ip->gd_ist = ist;
793 ip->gd_xx = 0;
794 ip->gd_type = typ;
795 ip->gd_dpl = dpl;
796 ip->gd_p = 1;
797 ip->gd_hioffset = ((uintptr_t)func)>>16 ;
798 }
799
800 extern inthand_t
801 IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
802 IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
803 IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
804 IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
805 IDTVEC(xmm), IDTVEC(dblfault),
806 IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
807
808 void
809 sdtossd(sd, ssd)
810 struct user_segment_descriptor *sd;
811 struct soft_segment_descriptor *ssd;
812 {
813
814 ssd->ssd_base = (sd->sd_hibase << 24) | sd->sd_lobase;
815 ssd->ssd_limit = (sd->sd_hilimit << 16) | sd->sd_lolimit;
816 ssd->ssd_type = sd->sd_type;
817 ssd->ssd_dpl = sd->sd_dpl;
818 ssd->ssd_p = sd->sd_p;
819 ssd->ssd_long = sd->sd_long;
820 ssd->ssd_def32 = sd->sd_def32;
821 ssd->ssd_gran = sd->sd_gran;
822 }
823
824 void
825 ssdtosd(ssd, sd)
826 struct soft_segment_descriptor *ssd;
827 struct user_segment_descriptor *sd;
828 {
829
830 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
831 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xff;
832 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
833 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
834 sd->sd_type = ssd->ssd_type;
835 sd->sd_dpl = ssd->ssd_dpl;
836 sd->sd_p = ssd->ssd_p;
837 sd->sd_long = ssd->ssd_long;
838 sd->sd_def32 = ssd->ssd_def32;
839 sd->sd_gran = ssd->ssd_gran;
840 }
841
842 void
843 ssdtosyssd(ssd, sd)
844 struct soft_segment_descriptor *ssd;
845 struct system_segment_descriptor *sd;
846 {
847
848 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
849 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xfffffffffful;
850 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
851 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
852 sd->sd_type = ssd->ssd_type;
853 sd->sd_dpl = ssd->ssd_dpl;
854 sd->sd_p = ssd->ssd_p;
855 sd->sd_gran = ssd->ssd_gran;
856 }
857
858 #if !defined(DEV_ATPIC) && defined(DEV_ISA)
859 #include <isa/isavar.h>
860 u_int
861 isa_irq_pending(void)
862 {
863
864 return (0);
865 }
866 #endif
867
868 u_int basemem;
869
870 static int
871 add_smap_entry(struct bios_smap *smap, vm_paddr_t *physmap, int *physmap_idxp)
872 {
873 int i, insert_idx, physmap_idx;
874
875 physmap_idx = *physmap_idxp;
876
877 if (boothowto & RB_VERBOSE)
878 printf("SMAP type=%02x base=%016lx len=%016lx\n",
879 smap->type, smap->base, smap->length);
880
881 if (smap->type != SMAP_TYPE_MEMORY)
882 return (1);
883
884 if (smap->length == 0)
885 return (0);
886
887 /*
888 * Find insertion point while checking for overlap. Start off by
889 * assuming the new entry will be added to the end.
890 */
891 insert_idx = physmap_idx + 2;
892 for (i = 0; i <= physmap_idx; i += 2) {
893 if (smap->base < physmap[i + 1]) {
894 if (smap->base + smap->length <= physmap[i]) {
895 insert_idx = i;
896 break;
897 }
898 if (boothowto & RB_VERBOSE)
899 printf(
900 "Overlapping memory regions, ignoring second region\n");
901 return (1);
902 }
903 }
904
905 /* See if we can prepend to the next entry. */
906 if (insert_idx <= physmap_idx &&
907 smap->base + smap->length == physmap[insert_idx]) {
908 physmap[insert_idx] = smap->base;
909 return (1);
910 }
911
912 /* See if we can append to the previous entry. */
913 if (insert_idx > 0 && smap->base == physmap[insert_idx - 1]) {
914 physmap[insert_idx - 1] += smap->length;
915 return (1);
916 }
917
918 physmap_idx += 2;
919 *physmap_idxp = physmap_idx;
920 if (physmap_idx == PHYSMAP_SIZE) {
921 printf(
922 "Too many segments in the physical address map, giving up\n");
923 return (0);
924 }
925
926 /*
927 * Move the last 'N' entries down to make room for the new
928 * entry if needed.
929 */
930 for (i = physmap_idx; i > insert_idx; i -= 2) {
931 physmap[i] = physmap[i - 2];
932 physmap[i + 1] = physmap[i - 1];
933 }
934
935 /* Insert the new entry. */
936 physmap[insert_idx] = smap->base;
937 physmap[insert_idx + 1] = smap->base + smap->length;
938 return (1);
939 }
940
941 /*
942 * Populate the (physmap) array with base/bound pairs describing the
943 * available physical memory in the system, then test this memory and
944 * build the phys_avail array describing the actually-available memory.
945 *
946 * If we cannot accurately determine the physical memory map, then use
947 * value from the 0xE801 call, and failing that, the RTC.
948 *
949 * Total memory size may be set by the kernel environment variable
950 * hw.physmem or the compile-time define MAXMEM.
951 *
952 * XXX first should be vm_paddr_t.
953 */
954 static void
955 getmemsize(caddr_t kmdp, u_int64_t first)
956 {
957 int i, off, physmap_idx, pa_indx, da_indx;
958 vm_paddr_t pa, physmap[PHYSMAP_SIZE];
959 u_long physmem_tunable, memtest;
960 pt_entry_t *pte;
961 struct bios_smap *smapbase, *smap, *smapend;
962 u_int32_t smapsize;
963 quad_t dcons_addr, dcons_size;
964
965 bzero(physmap, sizeof(physmap));
966 basemem = 0;
967 physmap_idx = 0;
968
969 /*
970 * get memory map from INT 15:E820, kindly supplied by the loader.
971 *
972 * subr_module.c says:
973 * "Consumer may safely assume that size value precedes data."
974 * ie: an int32_t immediately precedes smap.
975 */
976 smapbase = (struct bios_smap *)preload_search_info(kmdp,
977 MODINFO_METADATA | MODINFOMD_SMAP);
978 if (smapbase == NULL)
979 panic("No BIOS smap info from loader!");
980
981 smapsize = *((u_int32_t *)smapbase - 1);
982 smapend = (struct bios_smap *)((uintptr_t)smapbase + smapsize);
983
984 for (smap = smapbase; smap < smapend; smap++)
985 if (!add_smap_entry(smap, physmap, &physmap_idx))
986 break;
987
988 /*
989 * Find the 'base memory' segment for SMP
990 */
991 basemem = 0;
992 for (i = 0; i <= physmap_idx; i += 2) {
993 if (physmap[i] == 0x00000000) {
994 basemem = physmap[i + 1] / 1024;
995 break;
996 }
997 }
998 if (basemem == 0)
999 panic("BIOS smap did not include a basemem segment!");
1000
1001 #ifdef SMP
1002 /* make hole for AP bootstrap code */
1003 physmap[1] = mp_bootaddress(physmap[1] / 1024);
1004 #endif
1005
1006 /*
1007 * Maxmem isn't the "maximum memory", it's one larger than the
1008 * highest page of the physical address space. It should be
1009 * called something like "Maxphyspage". We may adjust this
1010 * based on ``hw.physmem'' and the results of the memory test.
1011 */
1012 Maxmem = atop(physmap[physmap_idx + 1]);
1013
1014 #ifdef MAXMEM
1015 Maxmem = MAXMEM / 4;
1016 #endif
1017
1018 if (TUNABLE_ULONG_FETCH("hw.physmem", &physmem_tunable))
1019 Maxmem = atop(physmem_tunable);
1020
1021 /*
1022 * By default keep the memtest enabled. Use a general name so that
1023 * one could eventually do more with the code than just disable it.
1024 */
1025 memtest = 1;
1026 TUNABLE_ULONG_FETCH("hw.memtest.tests", &memtest);
1027
1028 /*
1029 * Don't allow MAXMEM or hw.physmem to extend the amount of memory
1030 * in the system.
1031 */
1032 if (Maxmem > atop(physmap[physmap_idx + 1]))
1033 Maxmem = atop(physmap[physmap_idx + 1]);
1034
1035 if (atop(physmap[physmap_idx + 1]) != Maxmem &&
1036 (boothowto & RB_VERBOSE))
1037 printf("Physical memory use set to %ldK\n", Maxmem * 4);
1038
1039 /* call pmap initialization to make new kernel address space */
1040 pmap_bootstrap(&first);
1041
1042 /*
1043 * Size up each available chunk of physical memory.
1044 */
1045 physmap[0] = PAGE_SIZE; /* mask off page 0 */
1046 pa_indx = 0;
1047 da_indx = 1;
1048 phys_avail[pa_indx++] = physmap[0];
1049 phys_avail[pa_indx] = physmap[0];
1050 dump_avail[da_indx] = physmap[0];
1051 pte = CMAP1;
1052
1053 /*
1054 * Get dcons buffer address
1055 */
1056 if (getenv_quad("dcons.addr", &dcons_addr) == 0 ||
1057 getenv_quad("dcons.size", &dcons_size) == 0)
1058 dcons_addr = 0;
1059
1060 /*
1061 * physmap is in bytes, so when converting to page boundaries,
1062 * round up the start address and round down the end address.
1063 */
1064 for (i = 0; i <= physmap_idx; i += 2) {
1065 vm_paddr_t end;
1066
1067 end = ptoa((vm_paddr_t)Maxmem);
1068 if (physmap[i + 1] < end)
1069 end = trunc_page(physmap[i + 1]);
1070 for (pa = round_page(physmap[i]); pa < end; pa += PAGE_SIZE) {
1071 int tmp, page_bad, full;
1072 int *ptr = (int *)CADDR1;
1073
1074 full = FALSE;
1075 /*
1076 * block out kernel memory as not available.
1077 */
1078 if (pa >= 0x100000 && pa < first)
1079 goto do_dump_avail;
1080
1081 /*
1082 * block out dcons buffer
1083 */
1084 if (dcons_addr > 0
1085 && pa >= trunc_page(dcons_addr)
1086 && pa < dcons_addr + dcons_size)
1087 goto do_dump_avail;
1088
1089 page_bad = FALSE;
1090 if (memtest == 0)
1091 goto skip_memtest;
1092
1093 /*
1094 * map page into kernel: valid, read/write,non-cacheable
1095 */
1096 *pte = pa | PG_V | PG_RW | PG_N;
1097 invltlb();
1098
1099 tmp = *(int *)ptr;
1100 /*
1101 * Test for alternating 1's and 0's
1102 */
1103 *(volatile int *)ptr = 0xaaaaaaaa;
1104 if (*(volatile int *)ptr != 0xaaaaaaaa)
1105 page_bad = TRUE;
1106 /*
1107 * Test for alternating 0's and 1's
1108 */
1109 *(volatile int *)ptr = 0x55555555;
1110 if (*(volatile int *)ptr != 0x55555555)
1111 page_bad = TRUE;
1112 /*
1113 * Test for all 1's
1114 */
1115 *(volatile int *)ptr = 0xffffffff;
1116 if (*(volatile int *)ptr != 0xffffffff)
1117 page_bad = TRUE;
1118 /*
1119 * Test for all 0's
1120 */
1121 *(volatile int *)ptr = 0x0;
1122 if (*(volatile int *)ptr != 0x0)
1123 page_bad = TRUE;
1124 /*
1125 * Restore original value.
1126 */
1127 *(int *)ptr = tmp;
1128
1129 skip_memtest:
1130 /*
1131 * Adjust array of valid/good pages.
1132 */
1133 if (page_bad == TRUE)
1134 continue;
1135 /*
1136 * If this good page is a continuation of the
1137 * previous set of good pages, then just increase
1138 * the end pointer. Otherwise start a new chunk.
1139 * Note that "end" points one higher than end,
1140 * making the range >= start and < end.
1141 * If we're also doing a speculative memory
1142 * test and we at or past the end, bump up Maxmem
1143 * so that we keep going. The first bad page
1144 * will terminate the loop.
1145 */
1146 if (phys_avail[pa_indx] == pa) {
1147 phys_avail[pa_indx] += PAGE_SIZE;
1148 } else {
1149 pa_indx++;
1150 if (pa_indx == PHYS_AVAIL_ARRAY_END) {
1151 printf(
1152 "Too many holes in the physical address space, giving up\n");
1153 pa_indx--;
1154 full = TRUE;
1155 goto do_dump_avail;
1156 }
1157 phys_avail[pa_indx++] = pa; /* start */
1158 phys_avail[pa_indx] = pa + PAGE_SIZE; /* end */
1159 }
1160 physmem++;
1161 do_dump_avail:
1162 if (dump_avail[da_indx] == pa) {
1163 dump_avail[da_indx] += PAGE_SIZE;
1164 } else {
1165 da_indx++;
1166 if (da_indx == DUMP_AVAIL_ARRAY_END) {
1167 da_indx--;
1168 goto do_next;
1169 }
1170 dump_avail[da_indx++] = pa; /* start */
1171 dump_avail[da_indx] = pa + PAGE_SIZE; /* end */
1172 }
1173 do_next:
1174 if (full)
1175 break;
1176 }
1177 }
1178 *pte = 0;
1179 invltlb();
1180
1181 /*
1182 * XXX
1183 * The last chunk must contain at least one page plus the message
1184 * buffer to avoid complicating other code (message buffer address
1185 * calculation, etc.).
1186 */
1187 while (phys_avail[pa_indx - 1] + PAGE_SIZE +
1188 round_page(MSGBUF_SIZE) >= phys_avail[pa_indx]) {
1189 physmem -= atop(phys_avail[pa_indx] - phys_avail[pa_indx - 1]);
1190 phys_avail[pa_indx--] = 0;
1191 phys_avail[pa_indx--] = 0;
1192 }
1193
1194 Maxmem = atop(phys_avail[pa_indx]);
1195
1196 /* Trim off space for the message buffer. */
1197 phys_avail[pa_indx] -= round_page(MSGBUF_SIZE);
1198
1199 /* Map the message buffer. */
1200 for (off = 0; off < round_page(MSGBUF_SIZE); off += PAGE_SIZE)
1201 pmap_kenter((vm_offset_t)msgbufp + off, phys_avail[pa_indx] +
1202 off);
1203 }
1204
1205 u_int64_t
1206 hammer_time(u_int64_t modulep, u_int64_t physfree)
1207 {
1208 caddr_t kmdp;
1209 int gsel_tss, x;
1210 struct pcpu *pc;
1211 struct nmi_pcpu *np;
1212 u_int64_t msr;
1213 char *env;
1214
1215 thread0.td_kstack = physfree + KERNBASE;
1216 bzero((void *)thread0.td_kstack, KSTACK_PAGES * PAGE_SIZE);
1217 physfree += KSTACK_PAGES * PAGE_SIZE;
1218 thread0.td_pcb = (struct pcb *)
1219 (thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1;
1220
1221 /*
1222 * This may be done better later if it gets more high level
1223 * components in it. If so just link td->td_proc here.
1224 */
1225 proc_linkup0(&proc0, &thread0);
1226
1227 preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);
1228 preload_bootstrap_relocate(KERNBASE);
1229 kmdp = preload_search_by_type("elf kernel");
1230 if (kmdp == NULL)
1231 kmdp = preload_search_by_type("elf64 kernel");
1232 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
1233 kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *) + KERNBASE;
1234 #ifdef DDB
1235 ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
1236 ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
1237 #endif
1238
1239 /* Init basic tunables, hz etc */
1240 init_param1();
1241
1242 /*
1243 * make gdt memory segments
1244 */
1245 gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&common_tss[0];
1246
1247 for (x = 0; x < NGDT; x++) {
1248 if (x != GPROC0_SEL && x != (GPROC0_SEL + 1))
1249 ssdtosd(&gdt_segs[x], &gdt[x]);
1250 }
1251 ssdtosyssd(&gdt_segs[GPROC0_SEL],
1252 (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
1253
1254 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
1255 r_gdt.rd_base = (long) gdt;
1256 lgdt(&r_gdt);
1257 pc = &__pcpu[0];
1258
1259 wrmsr(MSR_FSBASE, 0); /* User value */
1260 wrmsr(MSR_GSBASE, (u_int64_t)pc);
1261 wrmsr(MSR_KGSBASE, 0); /* User value while in the kernel */
1262
1263 pcpu_init(pc, 0, sizeof(struct pcpu));
1264 PCPU_SET(prvspace, pc);
1265 PCPU_SET(curthread, &thread0);
1266 PCPU_SET(curpcb, thread0.td_pcb);
1267 PCPU_SET(tssp, &common_tss[0]);
1268 PCPU_SET(gs32p, &gdt[GUGS32_SEL]);
1269
1270 /*
1271 * Initialize mutexes.
1272 *
1273 * icu_lock: in order to allow an interrupt to occur in a critical
1274 * section, to set pcpu->ipending (etc...) properly, we
1275 * must be able to get the icu lock, so it can't be
1276 * under witness.
1277 */
1278 mutex_init();
1279 mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS);
1280
1281 /* exceptions */
1282 for (x = 0; x < NIDT; x++)
1283 setidt(x, &IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0);
1284 setidt(IDT_DE, &IDTVEC(div), SDT_SYSIGT, SEL_KPL, 0);
1285 setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 0);
1286 setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2);
1287 setidt(IDT_BP, &IDTVEC(bpt), SDT_SYSIGT, SEL_UPL, 0);
1288 setidt(IDT_OF, &IDTVEC(ofl), SDT_SYSIGT, SEL_KPL, 0);
1289 setidt(IDT_BR, &IDTVEC(bnd), SDT_SYSIGT, SEL_KPL, 0);
1290 setidt(IDT_UD, &IDTVEC(ill), SDT_SYSIGT, SEL_KPL, 0);
1291 setidt(IDT_NM, &IDTVEC(dna), SDT_SYSIGT, SEL_KPL, 0);
1292 setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);
1293 setidt(IDT_FPUGP, &IDTVEC(fpusegm), SDT_SYSIGT, SEL_KPL, 0);
1294 setidt(IDT_TS, &IDTVEC(tss), SDT_SYSIGT, SEL_KPL, 0);
1295 setidt(IDT_NP, &IDTVEC(missing), SDT_SYSIGT, SEL_KPL, 0);
1296 setidt(IDT_SS, &IDTVEC(stk), SDT_SYSIGT, SEL_KPL, 0);
1297 setidt(IDT_GP, &IDTVEC(prot), SDT_SYSIGT, SEL_KPL, 0);
1298 setidt(IDT_PF, &IDTVEC(page), SDT_SYSIGT, SEL_KPL, 0);
1299 setidt(IDT_MF, &IDTVEC(fpu), SDT_SYSIGT, SEL_KPL, 0);
1300 setidt(IDT_AC, &IDTVEC(align), SDT_SYSIGT, SEL_KPL, 0);
1301 setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 0);
1302 setidt(IDT_XF, &IDTVEC(xmm), SDT_SYSIGT, SEL_KPL, 0);
1303
1304 r_idt.rd_limit = sizeof(idt0) - 1;
1305 r_idt.rd_base = (long) idt;
1306 lidt(&r_idt);
1307
1308 /*
1309 * Initialize the i8254 before the console so that console
1310 * initialization can use DELAY().
1311 */
1312 i8254_init();
1313
1314 /*
1315 * Initialize the console before we print anything out.
1316 */
1317 cninit();
1318
1319 #ifdef DEV_ISA
1320 #ifdef DEV_ATPIC
1321 elcr_probe();
1322 atpic_startup();
1323 #else
1324 /* Reset and mask the atpics and leave them shut down. */
1325 atpic_reset();
1326
1327 /*
1328 * Point the ICU spurious interrupt vectors at the APIC spurious
1329 * interrupt handler.
1330 */
1331 setidt(IDT_IO_INTS + 7, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
1332 setidt(IDT_IO_INTS + 15, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
1333 #endif
1334 #else
1335 #error "have you forgotten the isa device?";
1336 #endif
1337
1338 kdb_init();
1339
1340 #ifdef KDB
1341 if (boothowto & RB_KDB)
1342 kdb_enter_why(KDB_WHY_BOOTFLAGS,
1343 "Boot flags requested debugger");
1344 #endif
1345
1346 identify_cpu(); /* Final stage of CPU initialization */
1347 initializecpu(); /* Initialize CPU registers */
1348 initializecpucache();
1349
1350 /* make an initial tss so cpu can get interrupt stack on syscall! */
1351 common_tss[0].tss_rsp0 = thread0.td_kstack + \
1352 KSTACK_PAGES * PAGE_SIZE - sizeof(struct pcb);
1353 /* Ensure the stack is aligned to 16 bytes */
1354 common_tss[0].tss_rsp0 &= ~0xFul;
1355 PCPU_SET(rsp0, common_tss[0].tss_rsp0);
1356
1357 /* doublefault stack space, runs on ist1 */
1358 common_tss[0].tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)];
1359
1360 /*
1361 * NMI stack, runs on ist2. The pcpu pointer is stored just
1362 * above the start of the ist2 stack.
1363 */
1364 np = ((struct nmi_pcpu *) &nmi0_stack[sizeof(nmi0_stack)]) - 1;
1365 np->np_pcpu = (register_t) pc;
1366 common_tss[0].tss_ist2 = (long) np;
1367
1368 /* Set the IO permission bitmap (empty due to tss seg limit) */
1369 common_tss[0].tss_iobase = sizeof(struct amd64tss);
1370
1371 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
1372 ltr(gsel_tss);
1373
1374 /* Set up the fast syscall stuff */
1375 msr = rdmsr(MSR_EFER) | EFER_SCE;
1376 wrmsr(MSR_EFER, msr);
1377 wrmsr(MSR_LSTAR, (u_int64_t)IDTVEC(fast_syscall));
1378 wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
1379 msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
1380 ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
1381 wrmsr(MSR_STAR, msr);
1382 wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);
1383
1384 getmemsize(kmdp, physfree);
1385 init_param2(physmem);
1386
1387 /* now running on new page tables, configured,and u/iom is accessible */
1388
1389 msgbufinit(msgbufp, MSGBUF_SIZE);
1390 fpuinit();
1391
1392 /* transfer to user mode */
1393
1394 _ucodesel = GSEL(GUCODE_SEL, SEL_UPL);
1395 _udatasel = GSEL(GUDATA_SEL, SEL_UPL);
1396 _ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL);
1397
1398 /* setup proc 0's pcb */
1399 thread0.td_pcb->pcb_flags = 0; /* XXXKSE */
1400 thread0.td_pcb->pcb_cr3 = KPML4phys;
1401 thread0.td_frame = &proc0_tf;
1402
1403 env = getenv("kernelname");
1404 if (env != NULL)
1405 strlcpy(kernelname, env, sizeof(kernelname));
1406
1407 /* Location of kernel stack for locore */
1408 return ((u_int64_t)thread0.td_pcb);
1409 }
1410
1411 void
1412 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
1413 {
1414
1415 pcpu->pc_acpi_id = 0xffffffff;
1416 }
1417
1418 void
1419 spinlock_enter(void)
1420 {
1421 struct thread *td;
1422 register_t flags;
1423
1424 td = curthread;
1425 if (td->td_md.md_spinlock_count == 0) {
1426 flags = intr_disable();
1427 td->td_md.md_spinlock_count = 1;
1428 td->td_md.md_saved_flags = flags;
1429 } else
1430 td->td_md.md_spinlock_count++;
1431 critical_enter();
1432 }
1433
1434 void
1435 spinlock_exit(void)
1436 {
1437 struct thread *td;
1438 register_t flags;
1439
1440 td = curthread;
1441 critical_exit();
1442 flags = td->td_md.md_saved_flags;
1443 td->td_md.md_spinlock_count--;
1444 if (td->td_md.md_spinlock_count == 0)
1445 intr_restore(flags);
1446 }
1447
1448 /*
1449 * Construct a PCB from a trapframe. This is called from kdb_trap() where
1450 * we want to start a backtrace from the function that caused us to enter
1451 * the debugger. We have the context in the trapframe, but base the trace
1452 * on the PCB. The PCB doesn't have to be perfect, as long as it contains
1453 * enough for a backtrace.
1454 */
1455 void
1456 makectx(struct trapframe *tf, struct pcb *pcb)
1457 {
1458
1459 pcb->pcb_r12 = tf->tf_r12;
1460 pcb->pcb_r13 = tf->tf_r13;
1461 pcb->pcb_r14 = tf->tf_r14;
1462 pcb->pcb_r15 = tf->tf_r15;
1463 pcb->pcb_rbp = tf->tf_rbp;
1464 pcb->pcb_rbx = tf->tf_rbx;
1465 pcb->pcb_rip = tf->tf_rip;
1466 pcb->pcb_rsp = tf->tf_rsp;
1467 }
1468
1469 int
1470 ptrace_set_pc(struct thread *td, unsigned long addr)
1471 {
1472 td->td_frame->tf_rip = addr;
1473 return (0);
1474 }
1475
1476 int
1477 ptrace_single_step(struct thread *td)
1478 {
1479 td->td_frame->tf_rflags |= PSL_T;
1480 return (0);
1481 }
1482
1483 int
1484 ptrace_clear_single_step(struct thread *td)
1485 {
1486 td->td_frame->tf_rflags &= ~PSL_T;
1487 return (0);
1488 }
1489
1490 int
1491 fill_regs(struct thread *td, struct reg *regs)
1492 {
1493 struct trapframe *tp;
1494
1495 tp = td->td_frame;
1496 regs->r_r15 = tp->tf_r15;
1497 regs->r_r14 = tp->tf_r14;
1498 regs->r_r13 = tp->tf_r13;
1499 regs->r_r12 = tp->tf_r12;
1500 regs->r_r11 = tp->tf_r11;
1501 regs->r_r10 = tp->tf_r10;
1502 regs->r_r9 = tp->tf_r9;
1503 regs->r_r8 = tp->tf_r8;
1504 regs->r_rdi = tp->tf_rdi;
1505 regs->r_rsi = tp->tf_rsi;
1506 regs->r_rbp = tp->tf_rbp;
1507 regs->r_rbx = tp->tf_rbx;
1508 regs->r_rdx = tp->tf_rdx;
1509 regs->r_rcx = tp->tf_rcx;
1510 regs->r_rax = tp->tf_rax;
1511 regs->r_rip = tp->tf_rip;
1512 regs->r_cs = tp->tf_cs;
1513 regs->r_rflags = tp->tf_rflags;
1514 regs->r_rsp = tp->tf_rsp;
1515 regs->r_ss = tp->tf_ss;
1516 return (0);
1517 }
1518
1519 int
1520 set_regs(struct thread *td, struct reg *regs)
1521 {
1522 struct trapframe *tp;
1523 register_t rflags;
1524
1525 tp = td->td_frame;
1526 rflags = regs->r_rflags & 0xffffffff;
1527 if (!EFL_SECURE(rflags, tp->tf_rflags) || !CS_SECURE(regs->r_cs))
1528 return (EINVAL);
1529 tp->tf_r15 = regs->r_r15;
1530 tp->tf_r14 = regs->r_r14;
1531 tp->tf_r13 = regs->r_r13;
1532 tp->tf_r12 = regs->r_r12;
1533 tp->tf_r11 = regs->r_r11;
1534 tp->tf_r10 = regs->r_r10;
1535 tp->tf_r9 = regs->r_r9;
1536 tp->tf_r8 = regs->r_r8;
1537 tp->tf_rdi = regs->r_rdi;
1538 tp->tf_rsi = regs->r_rsi;
1539 tp->tf_rbp = regs->r_rbp;
1540 tp->tf_rbx = regs->r_rbx;
1541 tp->tf_rdx = regs->r_rdx;
1542 tp->tf_rcx = regs->r_rcx;
1543 tp->tf_rax = regs->r_rax;
1544 tp->tf_rip = regs->r_rip;
1545 tp->tf_cs = regs->r_cs;
1546 tp->tf_rflags = rflags;
1547 tp->tf_rsp = regs->r_rsp;
1548 tp->tf_ss = regs->r_ss;
1549 td->td_pcb->pcb_flags |= PCB_FULLCTX;
1550 return (0);
1551 }
1552
1553 /* XXX check all this stuff! */
1554 /* externalize from sv_xmm */
1555 static void
1556 fill_fpregs_xmm(struct savefpu *sv_xmm, struct fpreg *fpregs)
1557 {
1558 struct envxmm *penv_fpreg = (struct envxmm *)&fpregs->fpr_env;
1559 struct envxmm *penv_xmm = &sv_xmm->sv_env;
1560 int i;
1561
1562 /* pcb -> fpregs */
1563 bzero(fpregs, sizeof(*fpregs));
1564
1565 /* FPU control/status */
1566 penv_fpreg->en_cw = penv_xmm->en_cw;
1567 penv_fpreg->en_sw = penv_xmm->en_sw;
1568 penv_fpreg->en_tw = penv_xmm->en_tw;
1569 penv_fpreg->en_opcode = penv_xmm->en_opcode;
1570 penv_fpreg->en_rip = penv_xmm->en_rip;
1571 penv_fpreg->en_rdp = penv_xmm->en_rdp;
1572 penv_fpreg->en_mxcsr = penv_xmm->en_mxcsr;
1573 penv_fpreg->en_mxcsr_mask = penv_xmm->en_mxcsr_mask;
1574
1575 /* FPU registers */
1576 for (i = 0; i < 8; ++i)
1577 bcopy(sv_xmm->sv_fp[i].fp_acc.fp_bytes, fpregs->fpr_acc[i], 10);
1578
1579 /* SSE registers */
1580 for (i = 0; i < 16; ++i)
1581 bcopy(sv_xmm->sv_xmm[i].xmm_bytes, fpregs->fpr_xacc[i], 16);
1582 }
1583
1584 /* internalize from fpregs into sv_xmm */
1585 static void
1586 set_fpregs_xmm(struct fpreg *fpregs, struct savefpu *sv_xmm)
1587 {
1588 struct envxmm *penv_xmm = &sv_xmm->sv_env;
1589 struct envxmm *penv_fpreg = (struct envxmm *)&fpregs->fpr_env;
1590 int i;
1591
1592 /* fpregs -> pcb */
1593 /* FPU control/status */
1594 penv_xmm->en_cw = penv_fpreg->en_cw;
1595 penv_xmm->en_sw = penv_fpreg->en_sw;
1596 penv_xmm->en_tw = penv_fpreg->en_tw;
1597 penv_xmm->en_opcode = penv_fpreg->en_opcode;
1598 penv_xmm->en_rip = penv_fpreg->en_rip;
1599 penv_xmm->en_rdp = penv_fpreg->en_rdp;
1600 penv_xmm->en_mxcsr = penv_fpreg->en_mxcsr;
1601 penv_xmm->en_mxcsr_mask = penv_fpreg->en_mxcsr_mask & cpu_mxcsr_mask;
1602
1603 /* FPU registers */
1604 for (i = 0; i < 8; ++i)
1605 bcopy(fpregs->fpr_acc[i], sv_xmm->sv_fp[i].fp_acc.fp_bytes, 10);
1606
1607 /* SSE registers */
1608 for (i = 0; i < 16; ++i)
1609 bcopy(fpregs->fpr_xacc[i], sv_xmm->sv_xmm[i].xmm_bytes, 16);
1610 }
1611
1612 /* externalize from td->pcb */
1613 int
1614 fill_fpregs(struct thread *td, struct fpreg *fpregs)
1615 {
1616
1617 fill_fpregs_xmm(&td->td_pcb->pcb_save, fpregs);
1618 return (0);
1619 }
1620
1621 /* internalize to td->pcb */
1622 int
1623 set_fpregs(struct thread *td, struct fpreg *fpregs)
1624 {
1625
1626 set_fpregs_xmm(fpregs, &td->td_pcb->pcb_save);
1627 return (0);
1628 }
1629
1630 /*
1631 * Get machine context.
1632 */
1633 int
1634 get_mcontext(struct thread *td, mcontext_t *mcp, int flags)
1635 {
1636 struct trapframe *tp;
1637
1638 tp = td->td_frame;
1639 PROC_LOCK(curthread->td_proc);
1640 mcp->mc_onstack = sigonstack(tp->tf_rsp);
1641 PROC_UNLOCK(curthread->td_proc);
1642 mcp->mc_r15 = tp->tf_r15;
1643 mcp->mc_r14 = tp->tf_r14;
1644 mcp->mc_r13 = tp->tf_r13;
1645 mcp->mc_r12 = tp->tf_r12;
1646 mcp->mc_r11 = tp->tf_r11;
1647 mcp->mc_r10 = tp->tf_r10;
1648 mcp->mc_r9 = tp->tf_r9;
1649 mcp->mc_r8 = tp->tf_r8;
1650 mcp->mc_rdi = tp->tf_rdi;
1651 mcp->mc_rsi = tp->tf_rsi;
1652 mcp->mc_rbp = tp->tf_rbp;
1653 mcp->mc_rbx = tp->tf_rbx;
1654 mcp->mc_rcx = tp->tf_rcx;
1655 mcp->mc_rflags = tp->tf_rflags;
1656 if (flags & GET_MC_CLEAR_RET) {
1657 mcp->mc_rax = 0;
1658 mcp->mc_rdx = 0;
1659 mcp->mc_rflags &= ~PSL_C;
1660 } else {
1661 mcp->mc_rax = tp->tf_rax;
1662 mcp->mc_rdx = tp->tf_rdx;
1663 }
1664 mcp->mc_rip = tp->tf_rip;
1665 mcp->mc_cs = tp->tf_cs;
1666 mcp->mc_rsp = tp->tf_rsp;
1667 mcp->mc_ss = tp->tf_ss;
1668 mcp->mc_len = sizeof(*mcp);
1669 get_fpcontext(td, mcp);
1670 bzero(mcp->mc_spare, sizeof(mcp->mc_spare));
1671 return (0);
1672 }
1673
1674 /*
1675 * Set machine context.
1676 *
1677 * However, we don't set any but the user modifiable flags, and we won't
1678 * touch the cs selector.
1679 */
1680 int
1681 set_mcontext(struct thread *td, const mcontext_t *mcp)
1682 {
1683 struct trapframe *tp;
1684 long rflags;
1685 int ret;
1686
1687 tp = td->td_frame;
1688 if (mcp->mc_len != sizeof(*mcp))
1689 return (EINVAL);
1690 rflags = (mcp->mc_rflags & PSL_USERCHANGE) |
1691 (tp->tf_rflags & ~PSL_USERCHANGE);
1692 ret = set_fpcontext(td, mcp);
1693 if (ret != 0)
1694 return (ret);
1695 tp->tf_r15 = mcp->mc_r15;
1696 tp->tf_r14 = mcp->mc_r14;
1697 tp->tf_r13 = mcp->mc_r13;
1698 tp->tf_r12 = mcp->mc_r12;
1699 tp->tf_r11 = mcp->mc_r11;
1700 tp->tf_r10 = mcp->mc_r10;
1701 tp->tf_r9 = mcp->mc_r9;
1702 tp->tf_r8 = mcp->mc_r8;
1703 tp->tf_rdi = mcp->mc_rdi;
1704 tp->tf_rsi = mcp->mc_rsi;
1705 tp->tf_rbp = mcp->mc_rbp;
1706 tp->tf_rbx = mcp->mc_rbx;
1707 tp->tf_rdx = mcp->mc_rdx;
1708 tp->tf_rcx = mcp->mc_rcx;
1709 tp->tf_rax = mcp->mc_rax;
1710 tp->tf_rip = mcp->mc_rip;
1711 tp->tf_rflags = rflags;
1712 tp->tf_rsp = mcp->mc_rsp;
1713 tp->tf_ss = mcp->mc_ss;
1714 td->td_pcb->pcb_flags |= PCB_FULLCTX;
1715 return (0);
1716 }
1717
1718 static void
1719 get_fpcontext(struct thread *td, mcontext_t *mcp)
1720 {
1721
1722 mcp->mc_ownedfp = fpugetregs(td, (struct savefpu *)&mcp->mc_fpstate);
1723 mcp->mc_fpformat = fpuformat();
1724 }
1725
1726 static int
1727 set_fpcontext(struct thread *td, const mcontext_t *mcp)
1728 {
1729 struct savefpu *fpstate;
1730
1731 if (mcp->mc_fpformat == _MC_FPFMT_NODEV)
1732 return (0);
1733 else if (mcp->mc_fpformat != _MC_FPFMT_XMM)
1734 return (EINVAL);
1735 else if (mcp->mc_ownedfp == _MC_FPOWNED_NONE)
1736 /* We don't care what state is left in the FPU or PCB. */
1737 fpstate_drop(td);
1738 else if (mcp->mc_ownedfp == _MC_FPOWNED_FPU ||
1739 mcp->mc_ownedfp == _MC_FPOWNED_PCB) {
1740 /*
1741 * XXX we violate the dubious requirement that fpusetregs()
1742 * be called with interrupts disabled.
1743 * XXX obsolete on trap-16 systems?
1744 */
1745 fpstate = (struct savefpu *)&mcp->mc_fpstate;
1746 fpstate->sv_env.en_mxcsr &= cpu_mxcsr_mask;
1747 fpusetregs(td, fpstate);
1748 } else
1749 return (EINVAL);
1750 return (0);
1751 }
1752
1753 void
1754 fpstate_drop(struct thread *td)
1755 {
1756 register_t s;
1757
1758 s = intr_disable();
1759 if (PCPU_GET(fpcurthread) == td)
1760 fpudrop();
1761 /*
1762 * XXX force a full drop of the fpu. The above only drops it if we
1763 * owned it.
1764 *
1765 * XXX I don't much like fpugetregs()'s semantics of doing a full
1766 * drop. Dropping only to the pcb matches fnsave's behaviour.
1767 * We only need to drop to !PCB_INITDONE in sendsig(). But
1768 * sendsig() is the only caller of fpugetregs()... perhaps we just
1769 * have too many layers.
1770 */
1771 curthread->td_pcb->pcb_flags &= ~PCB_FPUINITDONE;
1772 intr_restore(s);
1773 }
1774
1775 int
1776 fill_dbregs(struct thread *td, struct dbreg *dbregs)
1777 {
1778 struct pcb *pcb;
1779
1780 if (td == NULL) {
1781 dbregs->dr[0] = rdr0();
1782 dbregs->dr[1] = rdr1();
1783 dbregs->dr[2] = rdr2();
1784 dbregs->dr[3] = rdr3();
1785 dbregs->dr[6] = rdr6();
1786 dbregs->dr[7] = rdr7();
1787 } else {
1788 pcb = td->td_pcb;
1789 dbregs->dr[0] = pcb->pcb_dr0;
1790 dbregs->dr[1] = pcb->pcb_dr1;
1791 dbregs->dr[2] = pcb->pcb_dr2;
1792 dbregs->dr[3] = pcb->pcb_dr3;
1793 dbregs->dr[6] = pcb->pcb_dr6;
1794 dbregs->dr[7] = pcb->pcb_dr7;
1795 }
1796 dbregs->dr[4] = 0;
1797 dbregs->dr[5] = 0;
1798 dbregs->dr[8] = 0;
1799 dbregs->dr[9] = 0;
1800 dbregs->dr[10] = 0;
1801 dbregs->dr[11] = 0;
1802 dbregs->dr[12] = 0;
1803 dbregs->dr[13] = 0;
1804 dbregs->dr[14] = 0;
1805 dbregs->dr[15] = 0;
1806 return (0);
1807 }
1808
1809 int
1810 set_dbregs(struct thread *td, struct dbreg *dbregs)
1811 {
1812 struct pcb *pcb;
1813 int i;
1814
1815 if (td == NULL) {
1816 load_dr0(dbregs->dr[0]);
1817 load_dr1(dbregs->dr[1]);
1818 load_dr2(dbregs->dr[2]);
1819 load_dr3(dbregs->dr[3]);
1820 load_dr6(dbregs->dr[6]);
1821 load_dr7(dbregs->dr[7]);
1822 } else {
1823 /*
1824 * Don't let an illegal value for dr7 get set. Specifically,
1825 * check for undefined settings. Setting these bit patterns
1826 * result in undefined behaviour and can lead to an unexpected
1827 * TRCTRAP or a general protection fault right here.
1828 * Upper bits of dr6 and dr7 must not be set
1829 */
1830 for (i = 0; i < 4; i++) {
1831 if (DBREG_DR7_ACCESS(dbregs->dr[7], i) == 0x02)
1832 return (EINVAL);
1833 if (td->td_frame->tf_cs == _ucode32sel &&
1834 DBREG_DR7_LEN(dbregs->dr[7], i) == DBREG_DR7_LEN_8)
1835 return (EINVAL);
1836 }
1837 if ((dbregs->dr[6] & 0xffffffff00000000ul) != 0 ||
1838 (dbregs->dr[7] & 0xffffffff00000000ul) != 0)
1839 return (EINVAL);
1840
1841 pcb = td->td_pcb;
1842
1843 /*
1844 * Don't let a process set a breakpoint that is not within the
1845 * process's address space. If a process could do this, it
1846 * could halt the system by setting a breakpoint in the kernel
1847 * (if ddb was enabled). Thus, we need to check to make sure
1848 * that no breakpoints are being enabled for addresses outside
1849 * process's address space.
1850 *
1851 * XXX - what about when the watched area of the user's
1852 * address space is written into from within the kernel
1853 * ... wouldn't that still cause a breakpoint to be generated
1854 * from within kernel mode?
1855 */
1856
1857 if (DBREG_DR7_ENABLED(dbregs->dr[7], 0)) {
1858 /* dr0 is enabled */
1859 if (dbregs->dr[0] >= VM_MAXUSER_ADDRESS)
1860 return (EINVAL);
1861 }
1862 if (DBREG_DR7_ENABLED(dbregs->dr[7], 1)) {
1863 /* dr1 is enabled */
1864 if (dbregs->dr[1] >= VM_MAXUSER_ADDRESS)
1865 return (EINVAL);
1866 }
1867 if (DBREG_DR7_ENABLED(dbregs->dr[7], 2)) {
1868 /* dr2 is enabled */
1869 if (dbregs->dr[2] >= VM_MAXUSER_ADDRESS)
1870 return (EINVAL);
1871 }
1872 if (DBREG_DR7_ENABLED(dbregs->dr[7], 3)) {
1873 /* dr3 is enabled */
1874 if (dbregs->dr[3] >= VM_MAXUSER_ADDRESS)
1875 return (EINVAL);
1876 }
1877
1878 pcb->pcb_dr0 = dbregs->dr[0];
1879 pcb->pcb_dr1 = dbregs->dr[1];
1880 pcb->pcb_dr2 = dbregs->dr[2];
1881 pcb->pcb_dr3 = dbregs->dr[3];
1882 pcb->pcb_dr6 = dbregs->dr[6];
1883 pcb->pcb_dr7 = dbregs->dr[7];
1884
1885 pcb->pcb_flags |= PCB_DBREGS;
1886 }
1887
1888 return (0);
1889 }
1890
1891 void
1892 reset_dbregs(void)
1893 {
1894
1895 load_dr7(0); /* Turn off the control bits first */
1896 load_dr0(0);
1897 load_dr1(0);
1898 load_dr2(0);
1899 load_dr3(0);
1900 load_dr6(0);
1901 }
1902
1903 /*
1904 * Return > 0 if a hardware breakpoint has been hit, and the
1905 * breakpoint was in user space. Return 0, otherwise.
1906 */
1907 int
1908 user_dbreg_trap(void)
1909 {
1910 u_int64_t dr7, dr6; /* debug registers dr6 and dr7 */
1911 u_int64_t bp; /* breakpoint bits extracted from dr6 */
1912 int nbp; /* number of breakpoints that triggered */
1913 caddr_t addr[4]; /* breakpoint addresses */
1914 int i;
1915
1916 dr7 = rdr7();
1917 if ((dr7 & 0x000000ff) == 0) {
1918 /*
1919 * all GE and LE bits in the dr7 register are zero,
1920 * thus the trap couldn't have been caused by the
1921 * hardware debug registers
1922 */
1923 return 0;
1924 }
1925
1926 nbp = 0;
1927 dr6 = rdr6();
1928 bp = dr6 & 0x0000000f;
1929
1930 if (!bp) {
1931 /*
1932 * None of the breakpoint bits are set meaning this
1933 * trap was not caused by any of the debug registers
1934 */
1935 return 0;
1936 }
1937
1938 /*
1939 * at least one of the breakpoints were hit, check to see
1940 * which ones and if any of them are user space addresses
1941 */
1942
1943 if (bp & 0x01) {
1944 addr[nbp++] = (caddr_t)rdr0();
1945 }
1946 if (bp & 0x02) {
1947 addr[nbp++] = (caddr_t)rdr1();
1948 }
1949 if (bp & 0x04) {
1950 addr[nbp++] = (caddr_t)rdr2();
1951 }
1952 if (bp & 0x08) {
1953 addr[nbp++] = (caddr_t)rdr3();
1954 }
1955
1956 for (i = 0; i < nbp; i++) {
1957 if (addr[i] < (caddr_t)VM_MAXUSER_ADDRESS) {
1958 /*
1959 * addr[i] is in user space
1960 */
1961 return nbp;
1962 }
1963 }
1964
1965 /*
1966 * None of the breakpoints are in user space.
1967 */
1968 return 0;
1969 }
1970
1971 #ifdef KDB
1972
1973 /*
1974 * Provide inb() and outb() as functions. They are normally only
1975 * available as macros calling inlined functions, thus cannot be
1976 * called from the debugger.
1977 *
1978 * The actual code is stolen from <machine/cpufunc.h>, and de-inlined.
1979 */
1980
1981 #undef inb
1982 #undef outb
1983
1984 /* silence compiler warnings */
1985 u_char inb(u_int);
1986 void outb(u_int, u_char);
1987
1988 u_char
1989 inb(u_int port)
1990 {
1991 u_char data;
1992 /*
1993 * We use %%dx and not %1 here because i/o is done at %dx and not at
1994 * %edx, while gcc generates inferior code (movw instead of movl)
1995 * if we tell it to load (u_short) port.
1996 */
1997 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
1998 return (data);
1999 }
2000
2001 void
2002 outb(u_int port, u_char data)
2003 {
2004 u_char al;
2005 /*
2006 * Use an unnecessary assignment to help gcc's register allocator.
2007 * This make a large difference for gcc-1.40 and a tiny difference
2008 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for
2009 * best results. gcc-2.6.0 can't handle this.
2010 */
2011 al = data;
2012 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
2013 }
2014
2015 #endif /* KDB */
Cache object: 31bdb76f86da9abb54622b5854cc3401
|