Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/arm64/arm64/machdep.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28 #include "opt_platform.h" 29 #include "opt_ddb.h" 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD: releng/11.2/sys/arm64/arm64/machdep.c 331023 2018-03-15 20:09:24Z kevans $"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/buf.h> 37 #include <sys/bus.h> 38 #include <sys/cons.h> 39 #include <sys/cpu.h> 40 #include <sys/devmap.h> 41 #include <sys/efi.h> 42 #include <sys/exec.h> 43 #include <sys/imgact.h> 44 #include <sys/kdb.h> 45 #include <sys/kernel.h> 46 #include <sys/limits.h> 47 #include <sys/linker.h> 48 #include <sys/msgbuf.h> 49 #include <sys/pcpu.h> 50 #include <sys/proc.h> 51 #include <sys/ptrace.h> 52 #include <sys/reboot.h> 53 #include <sys/rwlock.h> 54 #include <sys/sched.h> 55 #include <sys/signalvar.h> 56 #include <sys/syscallsubr.h> 57 #include <sys/sysent.h> 58 #include <sys/sysproto.h> 59 #include <sys/ucontext.h> 60 #include <sys/vdso.h> 61 62 #include <vm/vm.h> 63 #include <vm/vm_kern.h> 64 #include <vm/vm_object.h> 65 #include <vm/vm_page.h> 66 #include <vm/pmap.h> 67 #include <vm/vm_map.h> 68 #include <vm/vm_pager.h> 69 70 #include <machine/armreg.h> 71 #include <machine/cpu.h> 72 #include <machine/debug_monitor.h> 73 #include <machine/kdb.h> 74 #include <machine/machdep.h> 75 #include <machine/metadata.h> 76 #include <machine/md_var.h> 77 #include <machine/pcb.h> 78 #include <machine/reg.h> 79 #include <machine/vmparam.h> 80 81 #ifdef VFP 82 #include <machine/vfp.h> 83 #endif 84 85 #ifdef FDT 86 #include <dev/fdt/fdt_common.h> 87 #include <dev/ofw/openfirm.h> 88 #endif 89 90 struct pcpu __pcpu[MAXCPU]; 91 92 static struct trapframe proc0_tf; 93 94 vm_paddr_t phys_avail[PHYS_AVAIL_SIZE + 2]; 95 vm_paddr_t dump_avail[PHYS_AVAIL_SIZE + 2]; 96 97 int early_boot = 1; 98 int cold = 1; 99 long realmem = 0; 100 long Maxmem = 0; 101 102 #define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1)) 103 vm_paddr_t physmap[PHYSMAP_SIZE]; 104 u_int physmap_idx; 105 106 struct kva_md_info kmi; 107 108 int64_t dcache_line_size; /* The minimum D cache line size */ 109 int64_t icache_line_size; /* The minimum I cache line size */ 110 int64_t idcache_line_size; /* The minimum cache line size */ 111 int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ 112 int has_pan; 113 114 /* pagezero_* implementations are provided in support.S */ 115 void pagezero_simple(void *); 116 void pagezero_cache(void *); 117 118 /* pagezero_simple is default pagezero */ 119 void (*pagezero)(void *p) = pagezero_simple; 120 121 static void 122 pan_setup(void) 123 { 124 uint64_t id_aa64mfr1; 125 126 id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 127 if (ID_AA64MMFR1_PAN(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE) 128 has_pan = 1; 129 } 130 131 void 132 pan_enable(void) 133 { 134 135 /* 136 * The LLVM integrated assembler doesn't understand the PAN 137 * PSTATE field. Because of this we need to manually create 138 * the instruction in an asm block. This is equivalent to: 139 * msr pan, #1 140 * 141 * This sets the PAN bit, stopping the kernel from accessing 142 * memory when userspace can also access it unless the kernel 143 * uses the userspace load/store instructions. 144 */ 145 if (has_pan) { 146 WRITE_SPECIALREG(sctlr_el1, 147 READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN); 148 __asm __volatile(".inst 0xd500409f | (0x1 << 8)"); 149 } 150 } 151 152 static void 153 cpu_startup(void *dummy) 154 { 155 156 identify_cpu(); 157 158 vm_ksubmap_init(&kmi); 159 bufinit(); 160 vm_pager_bufferinit(); 161 } 162 163 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); 164 165 int 166 cpu_idle_wakeup(int cpu) 167 { 168 169 return (0); 170 } 171 172 int 173 fill_regs(struct thread *td, struct reg *regs) 174 { 175 struct trapframe *frame; 176 177 frame = td->td_frame; 178 regs->sp = frame->tf_sp; 179 regs->lr = frame->tf_lr; 180 regs->elr = frame->tf_elr; 181 regs->spsr = frame->tf_spsr; 182 183 memcpy(regs->x, frame->tf_x, sizeof(regs->x)); 184 185 return (0); 186 } 187 188 int 189 set_regs(struct thread *td, struct reg *regs) 190 { 191 struct trapframe *frame; 192 193 frame = td->td_frame; 194 frame->tf_sp = regs->sp; 195 frame->tf_lr = regs->lr; 196 frame->tf_elr = regs->elr; 197 frame->tf_spsr &= ~PSR_FLAGS; 198 frame->tf_spsr |= regs->spsr & PSR_FLAGS; 199 200 memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x)); 201 202 return (0); 203 } 204 205 int 206 fill_fpregs(struct thread *td, struct fpreg *regs) 207 { 208 #ifdef VFP 209 struct pcb *pcb; 210 211 pcb = td->td_pcb; 212 if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 213 /* 214 * If we have just been running VFP instructions we will 215 * need to save the state to memcpy it below. 216 */ 217 if (td == curthread) 218 vfp_save_state(td, pcb); 219 220 memcpy(regs->fp_q, pcb->pcb_vfp, sizeof(regs->fp_q)); 221 regs->fp_cr = pcb->pcb_fpcr; 222 regs->fp_sr = pcb->pcb_fpsr; 223 } else 224 #endif 225 memset(regs->fp_q, 0, sizeof(regs->fp_q)); 226 return (0); 227 } 228 229 int 230 set_fpregs(struct thread *td, struct fpreg *regs) 231 { 232 #ifdef VFP 233 struct pcb *pcb; 234 235 pcb = td->td_pcb; 236 memcpy(pcb->pcb_vfp, regs->fp_q, sizeof(regs->fp_q)); 237 pcb->pcb_fpcr = regs->fp_cr; 238 pcb->pcb_fpsr = regs->fp_sr; 239 #endif 240 return (0); 241 } 242 243 int 244 fill_dbregs(struct thread *td, struct dbreg *regs) 245 { 246 247 printf("ARM64TODO: fill_dbregs"); 248 return (EDOOFUS); 249 } 250 251 int 252 set_dbregs(struct thread *td, struct dbreg *regs) 253 { 254 255 printf("ARM64TODO: set_dbregs"); 256 return (EDOOFUS); 257 } 258 259 int 260 ptrace_set_pc(struct thread *td, u_long addr) 261 { 262 263 printf("ARM64TODO: ptrace_set_pc"); 264 return (EDOOFUS); 265 } 266 267 int 268 ptrace_single_step(struct thread *td) 269 { 270 271 td->td_frame->tf_spsr |= PSR_SS; 272 td->td_pcb->pcb_flags |= PCB_SINGLE_STEP; 273 return (0); 274 } 275 276 int 277 ptrace_clear_single_step(struct thread *td) 278 { 279 280 td->td_frame->tf_spsr &= ~PSR_SS; 281 td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; 282 return (0); 283 } 284 285 void 286 exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) 287 { 288 struct trapframe *tf = td->td_frame; 289 290 memset(tf, 0, sizeof(struct trapframe)); 291 292 /* 293 * We need to set x0 for init as it doesn't call 294 * cpu_set_syscall_retval to copy the value. We also 295 * need to set td_retval for the cases where we do. 296 */ 297 tf->tf_x[0] = td->td_retval[0] = stack; 298 tf->tf_sp = STACKALIGN(stack); 299 tf->tf_lr = imgp->entry_addr; 300 tf->tf_elr = imgp->entry_addr; 301 } 302 303 /* Sanity check these are the same size, they will be memcpy'd to and fro */ 304 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == 305 sizeof((struct gpregs *)0)->gp_x); 306 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == 307 sizeof((struct reg *)0)->x); 308 309 int 310 get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) 311 { 312 struct trapframe *tf = td->td_frame; 313 314 if (clear_ret & GET_MC_CLEAR_RET) { 315 mcp->mc_gpregs.gp_x[0] = 0; 316 mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C; 317 } else { 318 mcp->mc_gpregs.gp_x[0] = tf->tf_x[0]; 319 mcp->mc_gpregs.gp_spsr = tf->tf_spsr; 320 } 321 322 memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1], 323 sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1)); 324 325 mcp->mc_gpregs.gp_sp = tf->tf_sp; 326 mcp->mc_gpregs.gp_lr = tf->tf_lr; 327 mcp->mc_gpregs.gp_elr = tf->tf_elr; 328 329 return (0); 330 } 331 332 int 333 set_mcontext(struct thread *td, mcontext_t *mcp) 334 { 335 struct trapframe *tf = td->td_frame; 336 uint32_t spsr; 337 338 spsr = mcp->mc_gpregs.gp_spsr; 339 if ((spsr & PSR_M_MASK) != PSR_M_EL0t || 340 (spsr & (PSR_F | PSR_I | PSR_A | PSR_D)) != 0) 341 return (EINVAL); 342 343 memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x)); 344 345 tf->tf_sp = mcp->mc_gpregs.gp_sp; 346 tf->tf_lr = mcp->mc_gpregs.gp_lr; 347 tf->tf_elr = mcp->mc_gpregs.gp_elr; 348 tf->tf_spsr = mcp->mc_gpregs.gp_spsr; 349 350 return (0); 351 } 352 353 static void 354 get_fpcontext(struct thread *td, mcontext_t *mcp) 355 { 356 #ifdef VFP 357 struct pcb *curpcb; 358 359 critical_enter(); 360 361 curpcb = curthread->td_pcb; 362 363 if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 364 /* 365 * If we have just been running VFP instructions we will 366 * need to save the state to memcpy it below. 367 */ 368 vfp_save_state(td, curpcb); 369 370 memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_vfp, 371 sizeof(mcp->mc_fpregs)); 372 mcp->mc_fpregs.fp_cr = curpcb->pcb_fpcr; 373 mcp->mc_fpregs.fp_sr = curpcb->pcb_fpsr; 374 mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags; 375 mcp->mc_flags |= _MC_FP_VALID; 376 } 377 378 critical_exit(); 379 #endif 380 } 381 382 static void 383 set_fpcontext(struct thread *td, mcontext_t *mcp) 384 { 385 #ifdef VFP 386 struct pcb *curpcb; 387 388 critical_enter(); 389 390 if ((mcp->mc_flags & _MC_FP_VALID) != 0) { 391 curpcb = curthread->td_pcb; 392 393 /* 394 * Discard any vfp state for the current thread, we 395 * are about to override it. 396 */ 397 vfp_discard(td); 398 399 memcpy(curpcb->pcb_vfp, mcp->mc_fpregs.fp_q, 400 sizeof(mcp->mc_fpregs)); 401 curpcb->pcb_fpcr = mcp->mc_fpregs.fp_cr; 402 curpcb->pcb_fpsr = mcp->mc_fpregs.fp_sr; 403 curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags; 404 } 405 406 critical_exit(); 407 #endif 408 } 409 410 void 411 cpu_idle(int busy) 412 { 413 414 spinlock_enter(); 415 if (!busy) 416 cpu_idleclock(); 417 if (!sched_runnable()) 418 __asm __volatile( 419 "dsb sy \n" 420 "wfi \n"); 421 if (!busy) 422 cpu_activeclock(); 423 spinlock_exit(); 424 } 425 426 void 427 cpu_halt(void) 428 { 429 430 /* We should have shutdown by now, if not enter a low power sleep */ 431 intr_disable(); 432 while (1) { 433 __asm __volatile("wfi"); 434 } 435 } 436 437 /* 438 * Flush the D-cache for non-DMA I/O so that the I-cache can 439 * be made coherent later. 440 */ 441 void 442 cpu_flush_dcache(void *ptr, size_t len) 443 { 444 445 /* ARM64TODO TBD */ 446 } 447 448 /* Get current clock frequency for the given CPU ID. */ 449 int 450 cpu_est_clockrate(int cpu_id, uint64_t *rate) 451 { 452 struct pcpu *pc; 453 454 pc = pcpu_find(cpu_id); 455 if (pc == NULL || rate == NULL) 456 return (EINVAL); 457 458 if (pc->pc_clock == 0) 459 return (EOPNOTSUPP); 460 461 *rate = pc->pc_clock; 462 return (0); 463 } 464 465 void 466 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 467 { 468 469 pcpu->pc_acpi_id = 0xffffffff; 470 } 471 472 void 473 spinlock_enter(void) 474 { 475 struct thread *td; 476 register_t daif; 477 478 td = curthread; 479 if (td->td_md.md_spinlock_count == 0) { 480 daif = intr_disable(); 481 td->td_md.md_spinlock_count = 1; 482 td->td_md.md_saved_daif = daif; 483 } else 484 td->td_md.md_spinlock_count++; 485 critical_enter(); 486 } 487 488 void 489 spinlock_exit(void) 490 { 491 struct thread *td; 492 register_t daif; 493 494 td = curthread; 495 critical_exit(); 496 daif = td->td_md.md_saved_daif; 497 td->td_md.md_spinlock_count--; 498 if (td->td_md.md_spinlock_count == 0) 499 intr_restore(daif); 500 } 501 502 #ifndef _SYS_SYSPROTO_H_ 503 struct sigreturn_args { 504 ucontext_t *ucp; 505 }; 506 #endif 507 508 int 509 sys_sigreturn(struct thread *td, struct sigreturn_args *uap) 510 { 511 ucontext_t uc; 512 int error; 513 514 if (uap == NULL) 515 return (EFAULT); 516 if (copyin(uap->sigcntxp, &uc, sizeof(uc))) 517 return (EFAULT); 518 519 error = set_mcontext(td, &uc.uc_mcontext); 520 if (error != 0) 521 return (error); 522 set_fpcontext(td, &uc.uc_mcontext); 523 524 /* Restore signal mask. */ 525 kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); 526 527 return (EJUSTRETURN); 528 } 529 530 /* 531 * Construct a PCB from a trapframe. This is called from kdb_trap() where 532 * we want to start a backtrace from the function that caused us to enter 533 * the debugger. We have the context in the trapframe, but base the trace 534 * on the PCB. The PCB doesn't have to be perfect, as long as it contains 535 * enough for a backtrace. 536 */ 537 void 538 makectx(struct trapframe *tf, struct pcb *pcb) 539 { 540 int i; 541 542 for (i = 0; i < PCB_LR; i++) 543 pcb->pcb_x[i] = tf->tf_x[i]; 544 545 pcb->pcb_x[PCB_LR] = tf->tf_lr; 546 pcb->pcb_pc = tf->tf_elr; 547 pcb->pcb_sp = tf->tf_sp; 548 } 549 550 void 551 sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) 552 { 553 struct thread *td; 554 struct proc *p; 555 struct trapframe *tf; 556 struct sigframe *fp, frame; 557 struct sigacts *psp; 558 struct sysentvec *sysent; 559 int code, onstack, sig; 560 561 td = curthread; 562 p = td->td_proc; 563 PROC_LOCK_ASSERT(p, MA_OWNED); 564 565 sig = ksi->ksi_signo; 566 code = ksi->ksi_code; 567 psp = p->p_sigacts; 568 mtx_assert(&psp->ps_mtx, MA_OWNED); 569 570 tf = td->td_frame; 571 onstack = sigonstack(tf->tf_sp); 572 573 CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, 574 catcher, sig); 575 576 /* Allocate and validate space for the signal handler context. */ 577 if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && 578 SIGISMEMBER(psp->ps_sigonstack, sig)) { 579 fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + 580 td->td_sigstk.ss_size); 581 #if defined(COMPAT_43) 582 td->td_sigstk.ss_flags |= SS_ONSTACK; 583 #endif 584 } else { 585 fp = (struct sigframe *)td->td_frame->tf_sp; 586 } 587 588 /* Make room, keeping the stack aligned */ 589 fp--; 590 fp = (struct sigframe *)STACKALIGN(fp); 591 592 /* Fill in the frame to copy out */ 593 get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); 594 get_fpcontext(td, &frame.sf_uc.uc_mcontext); 595 frame.sf_si = ksi->ksi_info; 596 frame.sf_uc.uc_sigmask = *mask; 597 frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ? 598 ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE; 599 frame.sf_uc.uc_stack = td->td_sigstk; 600 mtx_unlock(&psp->ps_mtx); 601 PROC_UNLOCK(td->td_proc); 602 603 /* Copy the sigframe out to the user's stack. */ 604 if (copyout(&frame, fp, sizeof(*fp)) != 0) { 605 /* Process has trashed its stack. Kill it. */ 606 CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); 607 PROC_LOCK(p); 608 sigexit(td, SIGILL); 609 } 610 611 tf->tf_x[0]= sig; 612 tf->tf_x[1] = (register_t)&fp->sf_si; 613 tf->tf_x[2] = (register_t)&fp->sf_uc; 614 615 tf->tf_elr = (register_t)catcher; 616 tf->tf_sp = (register_t)fp; 617 sysent = p->p_sysent; 618 if (sysent->sv_sigcode_base != 0) 619 tf->tf_lr = (register_t)sysent->sv_sigcode_base; 620 else 621 tf->tf_lr = (register_t)(sysent->sv_psstrings - 622 *(sysent->sv_szsigcode)); 623 624 CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr, 625 tf->tf_sp); 626 627 PROC_LOCK(p); 628 mtx_lock(&psp->ps_mtx); 629 } 630 631 static void 632 init_proc0(vm_offset_t kstack) 633 { 634 struct pcpu *pcpup = &__pcpu[0]; 635 636 proc_linkup0(&proc0, &thread0); 637 thread0.td_kstack = kstack; 638 thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1; 639 thread0.td_pcb->pcb_fpflags = 0; 640 thread0.td_pcb->pcb_vfpcpu = UINT_MAX; 641 thread0.td_frame = &proc0_tf; 642 pcpup->pc_curpcb = thread0.td_pcb; 643 } 644 645 typedef struct { 646 uint32_t type; 647 uint64_t phys_start; 648 uint64_t virt_start; 649 uint64_t num_pages; 650 uint64_t attr; 651 } EFI_MEMORY_DESCRIPTOR; 652 653 static int 654 add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap, 655 u_int *physmap_idxp) 656 { 657 u_int i, insert_idx, _physmap_idx; 658 659 _physmap_idx = *physmap_idxp; 660 661 if (length == 0) 662 return (1); 663 664 /* 665 * Find insertion point while checking for overlap. Start off by 666 * assuming the new entry will be added to the end. 667 */ 668 insert_idx = _physmap_idx; 669 for (i = 0; i <= _physmap_idx; i += 2) { 670 if (base < physmap[i + 1]) { 671 if (base + length <= physmap[i]) { 672 insert_idx = i; 673 break; 674 } 675 if (boothowto & RB_VERBOSE) 676 printf( 677 "Overlapping memory regions, ignoring second region\n"); 678 return (1); 679 } 680 } 681 682 /* See if we can prepend to the next entry. */ 683 if (insert_idx <= _physmap_idx && 684 base + length == physmap[insert_idx]) { 685 physmap[insert_idx] = base; 686 return (1); 687 } 688 689 /* See if we can append to the previous entry. */ 690 if (insert_idx > 0 && base == physmap[insert_idx - 1]) { 691 physmap[insert_idx - 1] += length; 692 return (1); 693 } 694 695 _physmap_idx += 2; 696 *physmap_idxp = _physmap_idx; 697 if (_physmap_idx == PHYSMAP_SIZE) { 698 printf( 699 "Too many segments in the physical address map, giving up\n"); 700 return (0); 701 } 702 703 /* 704 * Move the last 'N' entries down to make room for the new 705 * entry if needed. 706 */ 707 for (i = _physmap_idx; i > insert_idx; i -= 2) { 708 physmap[i] = physmap[i - 2]; 709 physmap[i + 1] = physmap[i - 1]; 710 } 711 712 /* Insert the new entry. */ 713 physmap[insert_idx] = base; 714 physmap[insert_idx + 1] = base + length; 715 return (1); 716 } 717 718 #ifdef FDT 719 static void 720 add_fdt_mem_regions(struct mem_region *mr, int mrcnt, vm_paddr_t *physmap, 721 u_int *physmap_idxp) 722 { 723 724 for (int i = 0; i < mrcnt; i++) { 725 if (!add_physmap_entry(mr[i].mr_start, mr[i].mr_size, physmap, 726 physmap_idxp)) 727 break; 728 } 729 } 730 #endif 731 732 static void 733 add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap, 734 u_int *physmap_idxp) 735 { 736 struct efi_md *map, *p; 737 const char *type; 738 size_t efisz; 739 int ndesc, i; 740 741 static const char *types[] = { 742 "Reserved", 743 "LoaderCode", 744 "LoaderData", 745 "BootServicesCode", 746 "BootServicesData", 747 "RuntimeServicesCode", 748 "RuntimeServicesData", 749 "ConventionalMemory", 750 "UnusableMemory", 751 "ACPIReclaimMemory", 752 "ACPIMemoryNVS", 753 "MemoryMappedIO", 754 "MemoryMappedIOPortSpace", 755 "PalCode", 756 "PersistentMemory" 757 }; 758 759 /* 760 * Memory map data provided by UEFI via the GetMemoryMap 761 * Boot Services API. 762 */ 763 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 764 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 765 766 if (efihdr->descriptor_size == 0) 767 return; 768 ndesc = efihdr->memory_size / efihdr->descriptor_size; 769 770 if (boothowto & RB_VERBOSE) 771 printf("%23s %12s %12s %8s %4s\n", 772 "Type", "Physical", "Virtual", "#Pages", "Attr"); 773 774 for (i = 0, p = map; i < ndesc; i++, 775 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 776 if (boothowto & RB_VERBOSE) { 777 if (p->md_type < nitems(types)) 778 type = types[p->md_type]; 779 else 780 type = "<INVALID>"; 781 printf("%23s %012lx %12p %08lx ", type, p->md_phys, 782 p->md_virt, p->md_pages); 783 if (p->md_attr & EFI_MD_ATTR_UC) 784 printf("UC "); 785 if (p->md_attr & EFI_MD_ATTR_WC) 786 printf("WC "); 787 if (p->md_attr & EFI_MD_ATTR_WT) 788 printf("WT "); 789 if (p->md_attr & EFI_MD_ATTR_WB) 790 printf("WB "); 791 if (p->md_attr & EFI_MD_ATTR_UCE) 792 printf("UCE "); 793 if (p->md_attr & EFI_MD_ATTR_WP) 794 printf("WP "); 795 if (p->md_attr & EFI_MD_ATTR_RP) 796 printf("RP "); 797 if (p->md_attr & EFI_MD_ATTR_XP) 798 printf("XP "); 799 if (p->md_attr & EFI_MD_ATTR_NV) 800 printf("NV "); 801 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE) 802 printf("MORE_RELIABLE "); 803 if (p->md_attr & EFI_MD_ATTR_RO) 804 printf("RO "); 805 if (p->md_attr & EFI_MD_ATTR_RT) 806 printf("RUNTIME"); 807 printf("\n"); 808 } 809 810 switch (p->md_type) { 811 case EFI_MD_TYPE_CODE: 812 case EFI_MD_TYPE_DATA: 813 case EFI_MD_TYPE_BS_CODE: 814 case EFI_MD_TYPE_BS_DATA: 815 case EFI_MD_TYPE_FREE: 816 /* 817 * We're allowed to use any entry with these types. 818 */ 819 break; 820 default: 821 continue; 822 } 823 824 if (!add_physmap_entry(p->md_phys, (p->md_pages * PAGE_SIZE), 825 physmap, physmap_idxp)) 826 break; 827 } 828 } 829 830 #ifdef FDT 831 static void 832 try_load_dtb(caddr_t kmdp) 833 { 834 vm_offset_t dtbp; 835 836 dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); 837 if (dtbp == (vm_offset_t)NULL) { 838 printf("ERROR loading DTB\n"); 839 return; 840 } 841 842 if (OF_install(OFW_FDT, 0) == FALSE) 843 panic("Cannot install FDT"); 844 845 if (OF_init((void *)dtbp) != 0) 846 panic("OF_init failed with the found device tree"); 847 } 848 #endif 849 850 static void 851 cache_setup(void) 852 { 853 int dcache_line_shift, icache_line_shift, dczva_line_shift; 854 uint32_t ctr_el0; 855 uint32_t dczid_el0; 856 857 ctr_el0 = READ_SPECIALREG(ctr_el0); 858 859 /* Read the log2 words in each D cache line */ 860 dcache_line_shift = CTR_DLINE_SIZE(ctr_el0); 861 /* Get the D cache line size */ 862 dcache_line_size = sizeof(int) << dcache_line_shift; 863 864 /* And the same for the I cache */ 865 icache_line_shift = CTR_ILINE_SIZE(ctr_el0); 866 icache_line_size = sizeof(int) << icache_line_shift; 867 868 idcache_line_size = MIN(dcache_line_size, icache_line_size); 869 870 dczid_el0 = READ_SPECIALREG(dczid_el0); 871 872 /* Check if dc zva is not prohibited */ 873 if (dczid_el0 & DCZID_DZP) 874 dczva_line_size = 0; 875 else { 876 /* Same as with above calculations */ 877 dczva_line_shift = DCZID_BS_SIZE(dczid_el0); 878 dczva_line_size = sizeof(int) << dczva_line_shift; 879 880 /* Change pagezero function */ 881 pagezero = pagezero_cache; 882 } 883 } 884 885 void 886 initarm(struct arm64_bootparams *abp) 887 { 888 struct efi_map_header *efihdr; 889 struct pcpu *pcpup; 890 #ifdef FDT 891 struct mem_region mem_regions[FDT_MEM_REGIONS]; 892 int mem_regions_sz; 893 #endif 894 vm_offset_t lastaddr; 895 caddr_t kmdp; 896 vm_paddr_t mem_len; 897 int i; 898 899 /* Set the module data location */ 900 preload_metadata = (caddr_t)(uintptr_t)(abp->modulep); 901 902 /* Find the kernel address */ 903 kmdp = preload_search_by_type("elf kernel"); 904 if (kmdp == NULL) 905 kmdp = preload_search_by_type("elf64 kernel"); 906 907 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); 908 init_static_kenv(MD_FETCH(kmdp, MODINFOMD_ENVP, char *), 0); 909 910 #ifdef FDT 911 try_load_dtb(kmdp); 912 #endif 913 914 /* Find the address to start allocating from */ 915 lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); 916 917 /* Load the physical memory ranges */ 918 physmap_idx = 0; 919 efihdr = (struct efi_map_header *)preload_search_info(kmdp, 920 MODINFO_METADATA | MODINFOMD_EFI_MAP); 921 if (efihdr != NULL) 922 add_efi_map_entries(efihdr, physmap, &physmap_idx); 923 #ifdef FDT 924 else { 925 /* Grab physical memory regions information from device tree. */ 926 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, 927 NULL) != 0) 928 panic("Cannot get physical memory regions"); 929 add_fdt_mem_regions(mem_regions, mem_regions_sz, physmap, 930 &physmap_idx); 931 } 932 #endif 933 934 /* Print the memory map */ 935 mem_len = 0; 936 for (i = 0; i < physmap_idx; i += 2) { 937 dump_avail[i] = physmap[i]; 938 dump_avail[i + 1] = physmap[i + 1]; 939 mem_len += physmap[i + 1] - physmap[i]; 940 } 941 dump_avail[i] = 0; 942 dump_avail[i + 1] = 0; 943 944 /* Set the pcpu data, this is needed by pmap_bootstrap */ 945 pcpup = &__pcpu[0]; 946 pcpu_init(pcpup, 0, sizeof(struct pcpu)); 947 948 /* 949 * Set the pcpu pointer with a backup in tpidr_el1 to be 950 * loaded when entering the kernel from userland. 951 */ 952 __asm __volatile( 953 "mov x18, %0 \n" 954 "msr tpidr_el1, %0" :: "r"(pcpup)); 955 956 PCPU_SET(curthread, &thread0); 957 958 /* Do basic tuning, hz etc */ 959 init_param1(); 960 961 cache_setup(); 962 pan_setup(); 963 964 /* Bootstrap enough of pmap to enter the kernel proper */ 965 pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt, 966 KERNBASE - abp->kern_delta, lastaddr - KERNBASE); 967 968 devmap_bootstrap(0, NULL); 969 970 cninit(); 971 972 init_proc0(abp->kern_stack); 973 msgbufinit(msgbufp, msgbufsize); 974 mutex_init(); 975 init_param2(physmem); 976 977 dbg_init(); 978 kdb_init(); 979 pan_enable(); 980 981 early_boot = 0; 982 } 983 984 void 985 dbg_init(void) 986 { 987 988 /* Clear OS lock */ 989 WRITE_SPECIALREG(OSLAR_EL1, 0); 990 991 /* This permits DDB to use debug registers for watchpoints. */ 992 dbg_monitor_init(); 993 994 /* TODO: Eventually will need to initialize debug registers here. */ 995 } 996 997 #ifdef DDB 998 #include <ddb/ddb.h> 999 1000 DB_SHOW_COMMAND(specialregs, db_show_spregs) 1001 { 1002 #define PRINT_REG(reg) \ 1003 db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg)) 1004 1005 PRINT_REG(actlr_el1); 1006 PRINT_REG(afsr0_el1); 1007 PRINT_REG(afsr1_el1); 1008 PRINT_REG(aidr_el1); 1009 PRINT_REG(amair_el1); 1010 PRINT_REG(ccsidr_el1); 1011 PRINT_REG(clidr_el1); 1012 PRINT_REG(contextidr_el1); 1013 PRINT_REG(cpacr_el1); 1014 PRINT_REG(csselr_el1); 1015 PRINT_REG(ctr_el0); 1016 PRINT_REG(currentel); 1017 PRINT_REG(daif); 1018 PRINT_REG(dczid_el0); 1019 PRINT_REG(elr_el1); 1020 PRINT_REG(esr_el1); 1021 PRINT_REG(far_el1); 1022 #if 0 1023 /* ARM64TODO: Enable VFP before reading floating-point registers */ 1024 PRINT_REG(fpcr); 1025 PRINT_REG(fpsr); 1026 #endif 1027 PRINT_REG(id_aa64afr0_el1); 1028 PRINT_REG(id_aa64afr1_el1); 1029 PRINT_REG(id_aa64dfr0_el1); 1030 PRINT_REG(id_aa64dfr1_el1); 1031 PRINT_REG(id_aa64isar0_el1); 1032 PRINT_REG(id_aa64isar1_el1); 1033 PRINT_REG(id_aa64pfr0_el1); 1034 PRINT_REG(id_aa64pfr1_el1); 1035 PRINT_REG(id_afr0_el1); 1036 PRINT_REG(id_dfr0_el1); 1037 PRINT_REG(id_isar0_el1); 1038 PRINT_REG(id_isar1_el1); 1039 PRINT_REG(id_isar2_el1); 1040 PRINT_REG(id_isar3_el1); 1041 PRINT_REG(id_isar4_el1); 1042 PRINT_REG(id_isar5_el1); 1043 PRINT_REG(id_mmfr0_el1); 1044 PRINT_REG(id_mmfr1_el1); 1045 PRINT_REG(id_mmfr2_el1); 1046 PRINT_REG(id_mmfr3_el1); 1047 #if 0 1048 /* Missing from llvm */ 1049 PRINT_REG(id_mmfr4_el1); 1050 #endif 1051 PRINT_REG(id_pfr0_el1); 1052 PRINT_REG(id_pfr1_el1); 1053 PRINT_REG(isr_el1); 1054 PRINT_REG(mair_el1); 1055 PRINT_REG(midr_el1); 1056 PRINT_REG(mpidr_el1); 1057 PRINT_REG(mvfr0_el1); 1058 PRINT_REG(mvfr1_el1); 1059 PRINT_REG(mvfr2_el1); 1060 PRINT_REG(revidr_el1); 1061 PRINT_REG(sctlr_el1); 1062 PRINT_REG(sp_el0); 1063 PRINT_REG(spsel); 1064 PRINT_REG(spsr_el1); 1065 PRINT_REG(tcr_el1); 1066 PRINT_REG(tpidr_el0); 1067 PRINT_REG(tpidr_el1); 1068 PRINT_REG(tpidrro_el0); 1069 PRINT_REG(ttbr0_el1); 1070 PRINT_REG(ttbr1_el1); 1071 PRINT_REG(vbar_el1); 1072 #undef PRINT_REG 1073 } 1074 1075 DB_SHOW_COMMAND(vtop, db_show_vtop) 1076 { 1077 uint64_t phys; 1078 1079 if (have_addr) { 1080 phys = arm64_address_translate_s1e1r(addr); 1081 db_printf("Physical address reg (read): 0x%016lx\n", phys); 1082 phys = arm64_address_translate_s1e1w(addr); 1083 db_printf("Physical address reg (write): 0x%016lx\n", phys); 1084 } else 1085 db_printf("show vtop <virt_addr>\n"); 1086 } 1087 #endif
Cache object: c0ceecedbf9ad9b858e369cbf60419c8
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]