FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_physmap.c

     /*      $NetBSD: subr_physmap.c,v 1.5 2021/09/06 20:55:08 andvar Exp $  */
     
     /*-
      * Copyright (c) 2013 The NetBSD Foundation, Inc.
      * All rights reserved.
      *
      * This code is derived from software contributed to The NetBSD Foundation
      * by Matt Thomas of 3am Software Foundry.
      *
     * Redistribution and use in source and binary forms, with or without
     * modification, are permitted provided that the following conditions
     * are met:
     * 1. Redistributions of source code must retain the above copyright
     *    notice, this list of conditions and the following disclaimer.
     * 2. Redistributions in binary form must reproduce the above copyright
     *    notice, this list of conditions and the following disclaimer in the
     *    documentation and/or other materials provided with the distribution.
     *
     * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     * POSSIBILITY OF SUCH DAMAGE.
     */
    
    #include <sys/cdefs.h>
    __KERNEL_RCSID(1, "$NetBSD: subr_physmap.c,v 1.5 2021/09/06 20:55:08 andvar Exp $");
    
    #include <sys/param.h>
    #include <sys/physmap.h>
    #include <sys/kmem.h>
    
    #include <uvm/uvm_extern.h>
    #include <uvm/uvm_page.h>
    
    #include <dev/mm.h>
    
    /*
     * This file contain support routines used to create and destroy lists of
     * physical pages from lists of pages or ranges of virtual address.  By using
     * these physical maps, the kernel can avoid mapping physical I/O in the 
     * kernel's address space in most cases.
     */
    
    typedef struct {
            physmap_t *pc_physmap;
            physmap_segment_t *pc_segs;
            vsize_t pc_offset;
            vsize_t pc_klen;
            vaddr_t pc_kva;
            u_int pc_nsegs;
            vm_prot_t pc_prot;
            bool pc_direct_mapped;
    } physmap_cookie_t;
    
    /*
     * Allocate a physmap structure that requires "maxsegs" segments.
     */
    static physmap_t *
    physmap_alloc(size_t maxsegs)
    {
            const size_t mapsize = offsetof(physmap_t, pm_segs[maxsegs]);
    
            KASSERT(maxsegs > 0);
    
            physmap_t * const map = kmem_zalloc(mapsize, KM_SLEEP);
            map->pm_maxsegs = maxsegs;
    
            return map;
    }
    
    static int
    physmap_fill(physmap_t *map, pmap_t pmap, vaddr_t va, vsize_t len)
    {
            size_t nsegs = map->pm_nsegs;
            physmap_segment_t *ps = &map->pm_segs[nsegs];
            vsize_t offset = va - trunc_page(va);
    
            if (nsegs == 0) {
                    if (!pmap_extract(pmap, va, &ps->ps_addr)) {
                            return EFAULT;
                    }
                    ps->ps_len = MIN(len, PAGE_SIZE - offset);
                    if (ps->ps_len == len) {
                            map->pm_nsegs = 1;
                            return 0;
                    }
                    offset = 0;
            } else {
                    /*
                     * Backup to the last segment since we have to see if we can
                     * merge virtual addresses that are physically contiguous into
                     * as few segments as possible.
                    */
                   ps--;
                   nsegs--;
           }
   
           paddr_t lastaddr = ps->ps_addr + ps->ps_len;
           for (;;) {
                   paddr_t curaddr;
                   if (!pmap_extract(pmap, va, &curaddr)) {
                           return EFAULT;
                   }
                   if (curaddr != lastaddr) {
                           ps++;
                           nsegs++;
                           KASSERT(nsegs < map->pm_maxsegs);
                           ps->ps_addr = curaddr;
                           lastaddr = curaddr;
                   }
                   if (offset + len > PAGE_SIZE) {
                           ps->ps_len += PAGE_SIZE - offset;
                           lastaddr = ps->ps_addr + ps->ps_len;
                           len -= PAGE_SIZE - offset;
                           lastaddr += PAGE_SIZE - offset;
                           offset = 0;
                   } else {
                           ps->ps_len += len;
                           map->pm_nsegs = nsegs + 1;
                           return 0;
                   }
           }
   }
   
   /*
    * Create a physmap and populate it with the pages that are used to mapped
    * linear range of virtual addresses.  It is assumed that uvm_vslock has been
    * called to lock these pages into memory.
    */
   int
   physmap_create_linear(physmap_t **map_p, const struct vmspace *vs, vaddr_t va,
           vsize_t len)
   {
           const size_t maxsegs = atop(round_page(va + len) - trunc_page(va));
           physmap_t * const map = physmap_alloc(maxsegs);
           int error = physmap_fill(map, vs->vm_map.pmap, va, len);
           if (error) {
                   physmap_destroy(map);
                   *map_p = NULL;
                   return error;
           }
           *map_p = map;
           return 0;
   }
   
   /*
    * Create a physmap and populate it with the pages that are contained in an
    * iovec array.  It is assumed that uvm_vslock has been called to lock these
    * pages into memory.
    */
   int
   physmap_create_iov(physmap_t **map_p, const struct vmspace *vs,
           struct iovec *iov, size_t iovlen)
   {
           size_t maxsegs = 0;
           for (size_t i = 0; i < iovlen; i++) {
                   const vaddr_t start = (vaddr_t) iov[i].iov_base;
                   const vaddr_t end = start + iov[i].iov_len;
                   maxsegs += atop(round_page(end) - trunc_page(start));
           }
           physmap_t * const map = physmap_alloc(maxsegs);
   
           for (size_t i = 0; i < iovlen; i++) {
                   int error = physmap_fill(map, vs->vm_map.pmap,
                       (vaddr_t) iov[i].iov_base, iov[i].iov_len);
                   if (error) {
                           physmap_destroy(map);
                           *map_p = NULL;
                           return error;
                   }
           }
           *map_p = map;
           return 0;
   }
   
   /*
    * This uses a list of vm_page structure to create a physmap.
    */
   physmap_t *
   physmap_create_pagelist(struct vm_page **pgs, size_t npgs)
   {
           physmap_t * const map = physmap_alloc(npgs);
   
           physmap_segment_t *ps = map->pm_segs;
   
           /*
            * Initialize the first segment.
            */
           paddr_t lastaddr = VM_PAGE_TO_PHYS(pgs[0]);
           ps->ps_addr = lastaddr;
           ps->ps_len = PAGE_SIZE;
   
           for (pgs++; npgs-- > 1; pgs++) {
                   /*
                    * lastaddr needs to be increased by a page.
                    */
                   lastaddr += PAGE_SIZE;
                   paddr_t curaddr = VM_PAGE_TO_PHYS(*pgs);
                   if (curaddr != lastaddr) {
                           /*
                            * If the addresses are not the same, we need to use
                            * a new segment.  Set its address and update lastaddr.
                            */
                           ps++;
                           ps->ps_addr = curaddr;
                           lastaddr = curaddr;
                   }
                   /*
                    * Increase this segment's length by a page
                    */
                   ps->ps_len += PAGE_SIZE;
           }
   
           map->pm_nsegs = ps + 1 - map->pm_segs;
           return map;
   }
   
   void
   physmap_destroy(physmap_t *map)
   {
           const size_t mapsize = offsetof(physmap_t, pm_segs[map->pm_maxsegs]);
   
           kmem_free(map, mapsize);
   }
   
   void *
   physmap_map_init(physmap_t *map, size_t offset, vm_prot_t prot)
   {
           physmap_cookie_t * const pc = kmem_zalloc(sizeof(*pc), KM_SLEEP);
   
           KASSERT(prot == VM_PROT_READ || prot == (VM_PROT_READ|VM_PROT_WRITE));
   
           pc->pc_physmap = map;
           pc->pc_segs = map->pm_segs;
           pc->pc_nsegs = map->pm_nsegs;
           pc->pc_prot = prot;
           pc->pc_klen = 0;
           pc->pc_kva = 0;
           pc->pc_direct_mapped = false;
   
           /*
            * Skip to the first segment we are interested in.
            */
           while (offset >= pc->pc_segs->ps_len) {
                   offset -= pc->pc_segs->ps_len;
                   pc->pc_segs++;
                   pc->pc_nsegs--;
           }
   
           pc->pc_offset = offset;
   
           return pc;
   }
   
   size_t
   physmap_map(void *cookie, vaddr_t *kvap)
   {
           physmap_cookie_t * const pc = cookie;
   
           /*
            * If there is currently a non-direct mapped KVA region allocated,
            * free it now.
            */
           if (pc->pc_kva != 0 && !pc->pc_direct_mapped) {
                   pmap_kremove(pc->pc_kva, pc->pc_klen);
                   pmap_update(pmap_kernel());
                   uvm_km_free(kernel_map, pc->pc_kva, pc->pc_klen,
                       UVM_KMF_VAONLY);
           }
   
           /*
            * If there are no more segments to process, return 0 indicating
            * we are done.
            */
           if (pc->pc_nsegs == 0) {
                   return 0;
           }
   
           /*
            * Get starting physical address of this segment and its length.
            */
           paddr_t pa = pc->pc_segs->ps_addr + pc->pc_offset;
           const size_t koff = pa & PAGE_MASK;
           const size_t len = pc->pc_segs->ps_len - pc->pc_offset;
   
           /*
            * Now that we have the starting offset in the page, reset to the
            * beginning of the page.
            */
           pa = trunc_page(pa);
   
           /*
            * We are now done with this segment; advance to the next one.
            */
           pc->pc_segs++;
           pc->pc_nsegs--;
           pc->pc_offset = 0;
   
           /*
            * Find out how many pages we are mapping.
            */
           pc->pc_klen = round_page(len);
   #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
           /*
            * Always try to direct map it since that's nearly zero cost.
            */
           pc->pc_direct_mapped = mm_md_direct_mapped_phys(pa, &pc->pc_kva);
   #endif
           if (!pc->pc_direct_mapped) {
                   /*
                    * If we can't direct map it, we have to allocate some KVA
                    * so we map it via the kernel_map.
                    */
                   pc->pc_kva = uvm_km_alloc(kernel_map, pc->pc_klen,
                       atop(pa) & uvmexp.colormask,
                       UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH);
                   KASSERT(pc->pc_kva != 0);
   
                   /*
                    * Setup mappings for this segment.
                    */
                   for (size_t poff = 0; poff < pc->pc_klen; poff += PAGE_SIZE) {
                           pmap_kenter_pa(pc->pc_kva + poff, pa + poff,
                               pc->pc_prot, 0);
                   }
                   /*
                    * Make them real.
                    */
                   pmap_update(pmap_kernel());
           }
           /*
            * Return the starting KVA (including offset into the page) and
            * the length of this segment.
            */
           *kvap = pc->pc_kva + koff;
           return len;
   }
   
   void
   physmap_map_fini(void *cookie)
   {
           physmap_cookie_t * const pc = cookie;
   
           /*
            * If there is currently a non-direct mapped KVA region allocated,
            * free it now.
            */
           if (pc->pc_kva != 0 && !pc->pc_direct_mapped) {
                   pmap_kremove(pc->pc_kva, pc->pc_klen);
                   pmap_update(pmap_kernel());
                   uvm_km_free(kernel_map, pc->pc_kva, pc->pc_klen,
                       UVM_KMF_VAONLY);
           }
   
           /*
            * Free the cookie.
            */
           kmem_free(pc, sizeof(*pc));
   }
   
   /*
    * genio needs to zero pages past the EOF or without backing storage (think
    * sparse files).  But since we are using physmaps, there is no kva to use with
    * memset so we need a helper to obtain a kva and memset the desired memory.
    */
   void
   physmap_zero(physmap_t *map, size_t offset, size_t len)
   {
           void * const cookie = physmap_map_init(map, offset,
               VM_PROT_READ|VM_PROT_WRITE);
   
           for (;;) {
                   vaddr_t kva;
                   size_t seglen = physmap_map(cookie, &kva);
                   KASSERT(seglen != 0);
                   if (seglen > len)
                           seglen = len;
                   memset((void *)kva, 0, seglen);
                   if (seglen == len)
                           break;
           }
   
           physmap_map_fini(cookie);
   }

Cache object: df4f93bf2d66923fdc015110ddfabc30