The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/arm/arm/vm_machdep.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1982, 1986 The Regents of the University of California.
    3  * Copyright (c) 1989, 1990 William Jolitz
    4  * Copyright (c) 1994 John Dyson
    5  * All rights reserved.
    6  *
    7  * This code is derived from software contributed to Berkeley by
    8  * the Systems Programming Group of the University of Utah Computer
    9  * Science Department, and William Jolitz.
   10  *
   11  * Redistribution and use in source and binary :forms, with or without
   12  * modification, are permitted provided that the following conditions
   13  * are met:
   14  * 1. Redistributions of source code must retain the above copyright
   15  *    notice, this list of conditions and the following disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  * 3. All advertising materials mentioning features or use of this software
   20  *    must display the following acknowledgement:
   21  *      This product includes software developed by the University of
   22  *      California, Berkeley and its contributors.
   23  * 4. Neither the name of the University nor the names of its contributors
   24  *    may be used to endorse or promote products derived from this software
   25  *    without specific prior written permission.
   26  *
   27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   37  * SUCH DAMAGE.
   38  *
   39  *      from: @(#)vm_machdep.c  7.3 (Berkeley) 5/13/91
   40  *      Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$
   41  */
   42 
   43 #include <sys/cdefs.h>
   44 __FBSDID("$FreeBSD: releng/10.0/sys/arm/arm/vm_machdep.c 255786 2013-09-22 13:36:52Z glebius $");
   45 
   46 #include <sys/param.h>
   47 #include <sys/systm.h>
   48 #include <sys/kernel.h>
   49 #include <sys/malloc.h>
   50 #include <sys/mbuf.h>
   51 #include <sys/proc.h>
   52 #include <sys/socketvar.h>
   53 #include <sys/sf_buf.h>
   54 #include <sys/syscall.h>
   55 #include <sys/sysctl.h>
   56 #include <sys/sysent.h>
   57 #include <sys/unistd.h>
   58 #include <machine/cpu.h>
   59 #include <machine/pcb.h>
   60 #include <machine/sysarch.h>
   61 #include <sys/lock.h>
   62 #include <sys/mutex.h>
   63 
   64 #include <vm/vm.h>
   65 #include <vm/pmap.h>
   66 #include <vm/vm_extern.h>
   67 #include <vm/vm_kern.h>
   68 #include <vm/vm_page.h>
   69 #include <vm/vm_map.h>
   70 #include <vm/vm_param.h>
   71 #include <vm/vm_pageout.h>
   72 #include <vm/uma.h>
   73 #include <vm/uma_int.h>
   74 
   75 #include <machine/md_var.h>
   76 
   77 /*
   78  * struct switchframe and trapframe must both be a multiple of 8
   79  * for correct stack alignment.
   80  */
   81 CTASSERT(sizeof(struct switchframe) == 24);
   82 CTASSERT(sizeof(struct trapframe) == 80);
   83 
   84 #ifndef ARM_USE_SMALL_ALLOC
   85 
   86 #ifndef NSFBUFS
   87 #define NSFBUFS         (512 + maxusers * 16)
   88 #endif
   89 
   90 static int nsfbufs;
   91 static int nsfbufspeak;
   92 static int nsfbufsused;
   93 
   94 SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufs, CTLFLAG_RDTUN, &nsfbufs, 0,
   95     "Maximum number of sendfile(2) sf_bufs available");
   96 SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufspeak, CTLFLAG_RD, &nsfbufspeak, 0,
   97     "Number of sendfile(2) sf_bufs at peak usage");
   98 SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufsused, CTLFLAG_RD, &nsfbufsused, 0,
   99     "Number of sendfile(2) sf_bufs in use");
  100 
  101 static void     sf_buf_init(void *arg);
  102 SYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL);
  103 
  104 LIST_HEAD(sf_head, sf_buf);
  105 
  106 /*
  107  * A hash table of active sendfile(2) buffers
  108  */
  109 static struct sf_head *sf_buf_active;
  110 static u_long sf_buf_hashmask;
  111 
  112 #define SF_BUF_HASH(m)  (((m) - vm_page_array) & sf_buf_hashmask)
  113 
  114 static TAILQ_HEAD(, sf_buf) sf_buf_freelist;
  115 static u_int    sf_buf_alloc_want;
  116 
  117 /*
  118  * A lock used to synchronize access to the hash table and free list
  119  */
  120 static struct mtx sf_buf_lock;
  121 #endif /* !ARM_USE_SMALL_ALLOC */
  122 
  123 /*
  124  * Finish a fork operation, with process p2 nearly set up.
  125  * Copy and update the pcb, set up the stack so that the child
  126  * ready to run and return to user mode.
  127  */
  128 void
  129 cpu_fork(register struct thread *td1, register struct proc *p2,
  130     struct thread *td2, int flags)
  131 {
  132         struct pcb *pcb2;
  133         struct trapframe *tf;
  134         struct switchframe *sf;
  135         struct mdproc *mdp2;
  136 
  137         if ((flags & RFPROC) == 0)
  138                 return;
  139         pcb2 = (struct pcb *)(td2->td_kstack + td2->td_kstack_pages * PAGE_SIZE) - 1;
  140 #ifdef __XSCALE__
  141 #ifndef CPU_XSCALE_CORE3
  142         pmap_use_minicache(td2->td_kstack, td2->td_kstack_pages * PAGE_SIZE);
  143 #endif
  144 #endif
  145         td2->td_pcb = pcb2;
  146         bcopy(td1->td_pcb, pcb2, sizeof(*pcb2));
  147         mdp2 = &p2->p_md;
  148         bcopy(&td1->td_proc->p_md, mdp2, sizeof(*mdp2));
  149         pcb2->un_32.pcb32_und_sp = td2->td_kstack + USPACE_UNDEF_STACK_TOP;
  150         pcb2->un_32.pcb32_sp = td2->td_kstack +
  151             USPACE_SVC_STACK_TOP - sizeof(*pcb2);
  152         pmap_activate(td2);
  153         td2->td_frame = tf = (struct trapframe *)STACKALIGN(
  154             pcb2->un_32.pcb32_sp - sizeof(struct trapframe));
  155         *tf = *td1->td_frame;
  156         sf = (struct switchframe *)tf - 1;
  157         sf->sf_r4 = (u_int)fork_return;
  158         sf->sf_r5 = (u_int)td2;
  159         sf->sf_pc = (u_int)fork_trampoline;
  160         tf->tf_spsr &= ~PSR_C_bit;
  161         tf->tf_r0 = 0;
  162         tf->tf_r1 = 0;
  163         pcb2->un_32.pcb32_sp = (u_int)sf;
  164         KASSERT((pcb2->un_32.pcb32_sp & 7) == 0,
  165             ("cpu_fork: Incorrect stack alignment"));
  166 
  167         /* Setup to release spin count in fork_exit(). */
  168         td2->td_md.md_spinlock_count = 1;
  169         td2->td_md.md_saved_cspr = 0;
  170 #ifdef ARM_TP_ADDRESS
  171         td2->td_md.md_tp = *(register_t *)ARM_TP_ADDRESS;
  172 #else
  173         td2->td_md.md_tp = (register_t) get_tls();
  174 #endif
  175 }
  176                                 
  177 void
  178 cpu_thread_swapin(struct thread *td)
  179 {
  180 }
  181 
  182 void
  183 cpu_thread_swapout(struct thread *td)
  184 {
  185 }
  186 
  187 /*
  188  * Detatch mapped page and release resources back to the system.
  189  */
  190 void
  191 sf_buf_free(struct sf_buf *sf)
  192 {
  193 #ifndef ARM_USE_SMALL_ALLOC
  194          mtx_lock(&sf_buf_lock);
  195          sf->ref_count--;
  196          if (sf->ref_count == 0) {
  197                  TAILQ_INSERT_TAIL(&sf_buf_freelist, sf, free_entry);
  198                  nsfbufsused--;
  199                  pmap_kremove(sf->kva);
  200                  sf->m = NULL;
  201                  LIST_REMOVE(sf, list_entry);
  202                  if (sf_buf_alloc_want > 0)
  203                          wakeup(&sf_buf_freelist);
  204          }
  205          mtx_unlock(&sf_buf_lock);
  206 #endif
  207 }
  208 
  209 #ifndef ARM_USE_SMALL_ALLOC
  210 /*
  211  * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
  212  */
  213 static void
  214 sf_buf_init(void *arg)
  215 {
  216         struct sf_buf *sf_bufs;
  217         vm_offset_t sf_base;
  218         int i;
  219 
  220         nsfbufs = NSFBUFS;
  221         TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs);
  222                 
  223         sf_buf_active = hashinit(nsfbufs, M_TEMP, &sf_buf_hashmask);
  224         TAILQ_INIT(&sf_buf_freelist);
  225         sf_base = kva_alloc(nsfbufs * PAGE_SIZE);
  226         sf_bufs = malloc(nsfbufs * sizeof(struct sf_buf), M_TEMP,
  227             M_NOWAIT | M_ZERO);
  228         for (i = 0; i < nsfbufs; i++) {
  229                 sf_bufs[i].kva = sf_base + i * PAGE_SIZE;
  230                 TAILQ_INSERT_TAIL(&sf_buf_freelist, &sf_bufs[i], free_entry);
  231         }
  232         sf_buf_alloc_want = 0;
  233         mtx_init(&sf_buf_lock, "sf_buf", NULL, MTX_DEF);
  234 }
  235 #endif
  236 
  237 /*
  238  * Get an sf_buf from the freelist. Will block if none are available.
  239  */
  240 struct sf_buf *
  241 sf_buf_alloc(struct vm_page *m, int flags)
  242 {
  243 #ifdef ARM_USE_SMALL_ALLOC
  244         return ((struct sf_buf *)m);
  245 #else
  246         struct sf_head *hash_list;
  247         struct sf_buf *sf;
  248         int error;
  249 
  250         hash_list = &sf_buf_active[SF_BUF_HASH(m)];
  251         mtx_lock(&sf_buf_lock);
  252         LIST_FOREACH(sf, hash_list, list_entry) {
  253                 if (sf->m == m) {
  254                         sf->ref_count++;
  255                         if (sf->ref_count == 1) {
  256                                 TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
  257                                 nsfbufsused++;
  258                                 nsfbufspeak = imax(nsfbufspeak, nsfbufsused);
  259                         }
  260                         goto done;
  261                 }
  262         }
  263         while ((sf = TAILQ_FIRST(&sf_buf_freelist)) == NULL) {
  264                 if (flags & SFB_NOWAIT)
  265                         goto done;
  266                 sf_buf_alloc_want++;
  267                 SFSTAT_INC(sf_allocwait);
  268                 error = msleep(&sf_buf_freelist, &sf_buf_lock,
  269                     (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
  270                 sf_buf_alloc_want--;
  271         
  272 
  273                 /*
  274                  * If we got a signal, don't risk going back to sleep.
  275                  */
  276                 if (error)
  277                         goto done;
  278         }
  279         TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
  280         if (sf->m != NULL)
  281                 LIST_REMOVE(sf, list_entry);
  282         LIST_INSERT_HEAD(hash_list, sf, list_entry);
  283         sf->ref_count = 1;
  284         sf->m = m;
  285         nsfbufsused++;
  286         nsfbufspeak = imax(nsfbufspeak, nsfbufsused);
  287         pmap_kenter(sf->kva, VM_PAGE_TO_PHYS(sf->m));
  288 done:
  289         mtx_unlock(&sf_buf_lock);
  290         return (sf);
  291 #endif
  292 }
  293 
  294 void
  295 cpu_set_syscall_retval(struct thread *td, int error)
  296 {
  297         trapframe_t *frame;
  298         int fixup;
  299 #ifdef __ARMEB__
  300         uint32_t insn;
  301 #endif
  302 
  303         frame = td->td_frame;
  304         fixup = 0;
  305 
  306 #ifdef __ARMEB__
  307         insn = *(u_int32_t *)(frame->tf_pc - INSN_SIZE);
  308         if ((insn & 0x000fffff) == SYS___syscall) {
  309                 register_t *ap = &frame->tf_r0;
  310                 register_t code = ap[_QUAD_LOWWORD];
  311                 if (td->td_proc->p_sysent->sv_mask)
  312                         code &= td->td_proc->p_sysent->sv_mask;
  313                 fixup = (code != SYS_freebsd6_lseek && code != SYS_lseek)
  314                     ? 1 : 0;
  315         }
  316 #endif
  317 
  318         switch (error) {
  319         case 0:
  320                 if (fixup) {
  321                         frame->tf_r0 = 0;
  322                         frame->tf_r1 = td->td_retval[0];
  323                 } else {
  324                         frame->tf_r0 = td->td_retval[0];
  325                         frame->tf_r1 = td->td_retval[1];
  326                 }
  327                 frame->tf_spsr &= ~PSR_C_bit;   /* carry bit */
  328                 break;
  329         case ERESTART:
  330                 /*
  331                  * Reconstruct the pc to point at the swi.
  332                  */
  333                 frame->tf_pc -= INSN_SIZE;
  334                 break;
  335         case EJUSTRETURN:
  336                 /* nothing to do */
  337                 break;
  338         default:
  339                 frame->tf_r0 = error;
  340                 frame->tf_spsr |= PSR_C_bit;    /* carry bit */
  341                 break;
  342         }
  343 }
  344 
  345 /*
  346  * Initialize machine state (pcb and trap frame) for a new thread about to
  347  * upcall. Put enough state in the new thread's PCB to get it to go back
  348  * userret(), where we can intercept it again to set the return (upcall)
  349  * Address and stack, along with those from upcals that are from other sources
  350  * such as those generated in thread_userret() itself.
  351  */
  352 void
  353 cpu_set_upcall(struct thread *td, struct thread *td0)
  354 {
  355         struct trapframe *tf;
  356         struct switchframe *sf;
  357 
  358         bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe));
  359         bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb));
  360         tf = td->td_frame;
  361         sf = (struct switchframe *)tf - 1;
  362         sf->sf_r4 = (u_int)fork_return;
  363         sf->sf_r5 = (u_int)td;
  364         sf->sf_pc = (u_int)fork_trampoline;
  365         tf->tf_spsr &= ~PSR_C_bit;
  366         tf->tf_r0 = 0;
  367         td->td_pcb->un_32.pcb32_sp = (u_int)sf;
  368         td->td_pcb->un_32.pcb32_und_sp = td->td_kstack + USPACE_UNDEF_STACK_TOP;
  369         KASSERT((td->td_pcb->un_32.pcb32_sp & 7) == 0,
  370             ("cpu_set_upcall: Incorrect stack alignment"));
  371 
  372         /* Setup to release spin count in fork_exit(). */
  373         td->td_md.md_spinlock_count = 1;
  374         td->td_md.md_saved_cspr = 0;
  375 }
  376 
  377 /*
  378  * Set that machine state for performing an upcall that has to
  379  * be done in thread_userret() so that those upcalls generated
  380  * in thread_userret() itself can be done as well.
  381  */
  382 void
  383 cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
  384         stack_t *stack)
  385 {
  386         struct trapframe *tf = td->td_frame;
  387 
  388         tf->tf_usr_sp = STACKALIGN((int)stack->ss_sp + stack->ss_size
  389             - sizeof(struct trapframe));
  390         tf->tf_pc = (int)entry;
  391         tf->tf_r0 = (int)arg;
  392         tf->tf_spsr = PSR_USR32_MODE;
  393 }
  394 
  395 int
  396 cpu_set_user_tls(struct thread *td, void *tls_base)
  397 {
  398 
  399         td->td_md.md_tp = (register_t)tls_base;
  400         if (td == curthread) {
  401                 critical_enter();
  402 #ifdef ARM_TP_ADDRESS
  403                 *(register_t *)ARM_TP_ADDRESS = (register_t)tls_base;
  404 #else
  405                 set_tls((void *)tls_base);
  406 #endif
  407                 critical_exit();
  408         }
  409         return (0);
  410 }
  411 
  412 void
  413 cpu_thread_exit(struct thread *td)
  414 {
  415 }
  416 
  417 void
  418 cpu_thread_alloc(struct thread *td)
  419 {
  420         td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_pages *
  421             PAGE_SIZE) - 1;
  422         /*
  423          * Ensure td_frame is aligned to an 8 byte boundary as it will be
  424          * placed into the stack pointer which must be 8 byte aligned in
  425          * the ARM EABI.
  426          */
  427         td->td_frame = (struct trapframe *)STACKALIGN((u_int)td->td_kstack +
  428             USPACE_SVC_STACK_TOP - sizeof(struct pcb) -
  429             sizeof(struct trapframe));
  430 #ifdef __XSCALE__
  431 #ifndef CPU_XSCALE_CORE3
  432         pmap_use_minicache(td->td_kstack, td->td_kstack_pages * PAGE_SIZE);
  433 #endif
  434 #endif
  435 }
  436 
  437 void
  438 cpu_thread_free(struct thread *td)
  439 {
  440 }
  441 
  442 void
  443 cpu_thread_clean(struct thread *td)
  444 {
  445 }
  446 
  447 /*
  448  * Intercept the return address from a freshly forked process that has NOT
  449  * been scheduled yet.
  450  *
  451  * This is needed to make kernel threads stay in kernel mode.
  452  */
  453 void
  454 cpu_set_fork_handler(struct thread *td, void (*func)(void *), void *arg)
  455 {
  456         struct switchframe *sf;
  457         struct trapframe *tf;
  458         
  459         tf = td->td_frame;
  460         sf = (struct switchframe *)tf - 1;
  461         sf->sf_r4 = (u_int)func;
  462         sf->sf_r5 = (u_int)arg;
  463         td->td_pcb->un_32.pcb32_sp = (u_int)sf;
  464         KASSERT((td->td_pcb->un_32.pcb32_sp & 7) == 0,
  465             ("cpu_set_fork_handler: Incorrect stack alignment"));
  466 }
  467 
  468 /*
  469  * Software interrupt handler for queued VM system processing.
  470  */
  471 void
  472 swi_vm(void *dummy)
  473 {
  474         
  475         if (busdma_swi_pending)
  476                 busdma_swi();
  477 }
  478 
  479 void
  480 cpu_exit(struct thread *td)
  481 {
  482 }
  483 
  484 #define BITS_PER_INT    (8 * sizeof(int))
  485 vm_offset_t arm_nocache_startaddr;
  486 static int arm_nocache_allocated[ARM_NOCACHE_KVA_SIZE / (PAGE_SIZE *
  487     BITS_PER_INT)];
  488 
  489 /*
  490  * Functions to map and unmap memory non-cached into KVA the kernel won't try
  491  * to allocate. The goal is to provide uncached memory to busdma, to honor
  492  * BUS_DMA_COHERENT.
  493  * We can allocate at most ARM_NOCACHE_KVA_SIZE bytes.
  494  * The allocator is rather dummy, each page is represented by a bit in
  495  * a bitfield, 0 meaning the page is not allocated, 1 meaning it is.
  496  * As soon as it finds enough contiguous pages to satisfy the request,
  497  * it returns the address.
  498  */
  499 void *
  500 arm_remap_nocache(void *addr, vm_size_t size)
  501 {
  502         int i, j;
  503 
  504         size = round_page(size);
  505         for (i = 0; i < ARM_NOCACHE_KVA_SIZE / PAGE_SIZE; i++) {
  506                 if (!(arm_nocache_allocated[i / BITS_PER_INT] & (1 << (i %
  507                     BITS_PER_INT)))) {
  508                         for (j = i; j < i + (size / (PAGE_SIZE)); j++)
  509                                 if (arm_nocache_allocated[j / BITS_PER_INT] &
  510                                     (1 << (j % BITS_PER_INT)))
  511                                         break;
  512                         if (j == i + (size / (PAGE_SIZE)))
  513                                 break;
  514                 }
  515         }
  516         if (i < ARM_NOCACHE_KVA_SIZE / PAGE_SIZE) {
  517                 vm_offset_t tomap = arm_nocache_startaddr + i * PAGE_SIZE;
  518                 void *ret = (void *)tomap;
  519                 vm_paddr_t physaddr = vtophys((vm_offset_t)addr);
  520                 vm_offset_t vaddr = (vm_offset_t) addr;
  521                 
  522                 vaddr = vaddr & ~PAGE_MASK;
  523                 for (; tomap < (vm_offset_t)ret + size; tomap += PAGE_SIZE,
  524                     vaddr += PAGE_SIZE, physaddr += PAGE_SIZE, i++) {
  525                         cpu_idcache_wbinv_range(vaddr, PAGE_SIZE);
  526 #ifdef ARM_L2_PIPT
  527                         cpu_l2cache_wbinv_range(physaddr, PAGE_SIZE);
  528 #else
  529                         cpu_l2cache_wbinv_range(vaddr, PAGE_SIZE);
  530 #endif
  531                         pmap_kenter_nocache(tomap, physaddr);
  532                         cpu_tlb_flushID_SE(vaddr);
  533                         arm_nocache_allocated[i / BITS_PER_INT] |= 1 << (i %
  534                             BITS_PER_INT);
  535                 }
  536                 return (ret);
  537         }
  538 
  539         return (NULL);
  540 }
  541 
  542 void
  543 arm_unmap_nocache(void *addr, vm_size_t size)
  544 {
  545         vm_offset_t raddr = (vm_offset_t)addr;
  546         int i;
  547 
  548         size = round_page(size);
  549         i = (raddr - arm_nocache_startaddr) / (PAGE_SIZE);
  550         for (; size > 0; size -= PAGE_SIZE, i++) {
  551                 arm_nocache_allocated[i / BITS_PER_INT] &= ~(1 << (i %
  552                     BITS_PER_INT));
  553                 pmap_kremove(raddr);
  554                 raddr += PAGE_SIZE;
  555         }
  556 }
  557 
  558 #ifdef ARM_USE_SMALL_ALLOC
  559 
  560 static TAILQ_HEAD(,arm_small_page) pages_normal =
  561         TAILQ_HEAD_INITIALIZER(pages_normal);
  562 static TAILQ_HEAD(,arm_small_page) pages_wt =
  563         TAILQ_HEAD_INITIALIZER(pages_wt);
  564 static TAILQ_HEAD(,arm_small_page) free_pgdesc =
  565         TAILQ_HEAD_INITIALIZER(free_pgdesc);
  566 
  567 extern uma_zone_t l2zone;
  568 
  569 struct mtx smallalloc_mtx;
  570 
  571 vm_offset_t alloc_firstaddr;
  572 
  573 #ifdef ARM_HAVE_SUPERSECTIONS
  574 #define S_FRAME L1_SUP_FRAME
  575 #define S_SIZE  L1_SUP_SIZE
  576 #else
  577 #define S_FRAME L1_S_FRAME
  578 #define S_SIZE  L1_S_SIZE
  579 #endif
  580 
  581 vm_offset_t
  582 arm_ptovirt(vm_paddr_t pa)
  583 {
  584         int i;
  585         vm_offset_t addr = alloc_firstaddr;
  586 
  587         KASSERT(alloc_firstaddr != 0, ("arm_ptovirt called too early ?"));
  588         for (i = 0; dump_avail[i + 1]; i += 2) {
  589                 if (pa >= dump_avail[i] && pa < dump_avail[i + 1])
  590                         break;
  591                 addr += (dump_avail[i + 1] & S_FRAME) + S_SIZE -
  592                     (dump_avail[i] & S_FRAME);
  593         }
  594         KASSERT(dump_avail[i + 1] != 0, ("Trying to access invalid physical address"));
  595         return (addr + (pa - (dump_avail[i] & S_FRAME)));
  596 }
  597 
  598 void
  599 arm_init_smallalloc(void)
  600 {
  601         vm_offset_t to_map = 0, mapaddr;
  602         int i;
  603         
  604         /*
  605          * We need to use dump_avail and not phys_avail, since we want to
  606          * map the whole memory and not just the memory available to the VM
  607          * to be able to do a pa => va association for any address.
  608          */
  609 
  610         for (i = 0; dump_avail[i + 1]; i+= 2) {
  611                 to_map += (dump_avail[i + 1] & S_FRAME) + S_SIZE -
  612                     (dump_avail[i] & S_FRAME);
  613         }
  614         alloc_firstaddr = mapaddr = KERNBASE - to_map;
  615         for (i = 0; dump_avail[i + 1]; i+= 2) {
  616                 vm_offset_t size = (dump_avail[i + 1] & S_FRAME) +
  617                     S_SIZE - (dump_avail[i] & S_FRAME);
  618                 vm_offset_t did = 0;
  619                 while (size > 0) {
  620 #ifdef ARM_HAVE_SUPERSECTIONS
  621                         pmap_kenter_supersection(mapaddr,
  622                             (dump_avail[i] & L1_SUP_FRAME) + did,
  623                             SECTION_CACHE);
  624 #else
  625                         pmap_kenter_section(mapaddr,
  626                             (dump_avail[i] & L1_S_FRAME) + did, SECTION_CACHE);
  627 #endif
  628                         mapaddr += S_SIZE;
  629                         did += S_SIZE;
  630                         size -= S_SIZE;
  631                 }
  632         }
  633 }
  634 
  635 void
  636 arm_add_smallalloc_pages(void *list, void *mem, int bytes, int pagetable)
  637 {
  638         struct arm_small_page *pg;
  639         
  640         bytes &= ~PAGE_MASK;
  641         while (bytes > 0) {
  642                 pg = (struct arm_small_page *)list;
  643                 pg->addr = mem;
  644                 if (pagetable)
  645                         TAILQ_INSERT_HEAD(&pages_wt, pg, pg_list);
  646                 else
  647                         TAILQ_INSERT_HEAD(&pages_normal, pg, pg_list);
  648                 list = (char *)list + sizeof(*pg);
  649                 mem = (char *)mem + PAGE_SIZE;
  650                 bytes -= PAGE_SIZE;
  651         }
  652 }
  653 
  654 void *
  655 uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
  656 {
  657         void *ret;
  658         struct arm_small_page *sp;
  659         TAILQ_HEAD(,arm_small_page) *head;
  660         vm_page_t m;
  661 
  662         *flags = UMA_SLAB_PRIV;
  663         /*
  664          * For CPUs where we setup page tables as write back, there's no
  665          * need to maintain two separate pools.
  666          */
  667         if (zone == l2zone && pte_l1_s_cache_mode != pte_l1_s_cache_mode_pt)
  668                 head = (void *)&pages_wt;
  669         else
  670                 head = (void *)&pages_normal;
  671 
  672         mtx_lock(&smallalloc_mtx);
  673         sp = TAILQ_FIRST(head);
  674 
  675         if (!sp) {
  676                 int pflags;
  677 
  678                 mtx_unlock(&smallalloc_mtx);
  679                 if (zone == l2zone &&
  680                     pte_l1_s_cache_mode != pte_l1_s_cache_mode_pt) {
  681                         *flags = UMA_SLAB_KMEM;
  682                         ret = ((void *)kmem_malloc(kmem_arena, bytes,
  683                             M_NOWAIT));
  684                         return (ret);
  685                 }
  686                 pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
  687                 for (;;) {
  688                         m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
  689                         if (m == NULL) {
  690                                 if (wait & M_NOWAIT)
  691                                         return (NULL);
  692                                 VM_WAIT;
  693                         } else
  694                                 break;
  695                 }
  696                 ret = (void *)arm_ptovirt(VM_PAGE_TO_PHYS(m));
  697                 if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
  698                         bzero(ret, PAGE_SIZE);
  699                 return (ret);
  700         }
  701         TAILQ_REMOVE(head, sp, pg_list);
  702         TAILQ_INSERT_HEAD(&free_pgdesc, sp, pg_list);
  703         ret = sp->addr;
  704         mtx_unlock(&smallalloc_mtx);
  705         if ((wait & M_ZERO))
  706                 bzero(ret, bytes);
  707         return (ret);
  708 }
  709 
  710 void
  711 uma_small_free(void *mem, int size, u_int8_t flags)
  712 {
  713         pd_entry_t *pd;
  714         pt_entry_t *pt;
  715 
  716         if (flags & UMA_SLAB_KMEM)
  717                 kmem_free(kmem_arena, (vm_offset_t)mem, size);
  718         else {
  719                 struct arm_small_page *sp;
  720 
  721                 if ((vm_offset_t)mem >= KERNBASE) {
  722                         mtx_lock(&smallalloc_mtx);
  723                         sp = TAILQ_FIRST(&free_pgdesc);
  724                         KASSERT(sp != NULL, ("No more free page descriptor ?"));
  725                         TAILQ_REMOVE(&free_pgdesc, sp, pg_list);
  726                         sp->addr = mem;
  727                         pmap_get_pde_pte(kernel_pmap, (vm_offset_t)mem, &pd,
  728                             &pt);
  729                         if ((*pd & pte_l1_s_cache_mask) ==
  730                             pte_l1_s_cache_mode_pt &&
  731                             pte_l1_s_cache_mode_pt != pte_l1_s_cache_mode)
  732                                 TAILQ_INSERT_HEAD(&pages_wt, sp, pg_list);
  733                         else
  734                                 TAILQ_INSERT_HEAD(&pages_normal, sp, pg_list);
  735                         mtx_unlock(&smallalloc_mtx);
  736                 } else {
  737                         vm_page_t m;
  738                         vm_paddr_t pa = vtophys((vm_offset_t)mem);
  739 
  740                         m = PHYS_TO_VM_PAGE(pa);
  741                         m->wire_count--;
  742                         vm_page_free(m);
  743                         atomic_subtract_int(&cnt.v_wire_count, 1);
  744                 }
  745         }
  746 }
  747 
  748 #endif

Cache object: 2385e48fd71f3279bb97fbf863b08d95


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.