mm.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*      $NetBSD: mm.c,v 1.24 2019/02/05 11:33:13 mrg Exp $      */
    2 
    3 /*-
    4  * Copyright (c) 2002, 2008, 2010 The NetBSD Foundation, Inc.
    5  * All rights reserved.
    6  *
    7  * This code is derived from software contributed to The NetBSD Foundation
    8  * by Christos Zoulas, Joerg Sonnenberger and Mindaugas Rasiukevicius.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  *
   19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
   20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
   21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
   23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   29  * POSSIBILITY OF SUCH DAMAGE.
   30  */
   31 
   32 /*
   33  * Special /dev/{mem,kmem,zero,null} memory devices.
   34  */
   35 
   36 #include <sys/cdefs.h>
   37 __KERNEL_RCSID(0, "$NetBSD: mm.c,v 1.24 2019/02/05 11:33:13 mrg Exp $");
   38 
   39 #include "opt_compat_netbsd.h"
   40 
   41 #include <sys/param.h>
   42 #include <sys/conf.h>
   43 #include <sys/ioctl.h>
   44 #include <sys/mman.h>
   45 #include <sys/uio.h>
   46 #include <sys/termios.h>
   47 
   48 #include <dev/mm.h>
   49 
   50 #include <uvm/uvm_extern.h>
   51 
   52 static void *           dev_zero_page   __read_mostly;
   53 static kmutex_t         dev_mem_lock    __cacheline_aligned;
   54 static vaddr_t          dev_mem_addr    __read_mostly;
   55 
   56 static dev_type_open(mm_open);
   57 static dev_type_read(mm_readwrite);
   58 static dev_type_ioctl(mm_ioctl);
   59 static dev_type_mmap(mm_mmap);
   60 static dev_type_ioctl(mm_ioctl);
   61 
   62 const struct cdevsw mem_cdevsw = {
   63         .d_open = mm_open,
   64         .d_close = nullclose,
   65         .d_read = mm_readwrite,
   66         .d_write = mm_readwrite,
   67         .d_ioctl = mm_ioctl,
   68         .d_stop = nostop,
   69         .d_tty = notty,
   70         .d_poll = nopoll,
   71         .d_mmap = mm_mmap,
   72         .d_kqfilter = nokqfilter,
   73         .d_discard = nodiscard,
   74         .d_flag = D_MPSAFE
   75 };
   76 
   77 #ifdef pmax     /* XXX */
   78 const struct cdevsw mem_ultrix_cdevsw = {
   79         .d_open = nullopen,
   80         .d_close = nullclose,
   81         .d_read = mm_readwrite,
   82         .d_write = mm_readwrite,
   83         .d_ioctl = mm_ioctl,
   84         .d_stop = nostop,
   85         .d_tty = notty,
   86         .d_poll = nopoll,
   87         .d_mmap = mm_mmap,
   88         .d_kqfilter = nokqfilter,
   89         .d_discard = nodiscard,
   90         .d_flag = D_MPSAFE
   91 };
   92 #endif
   93 
   94 static int
   95 mm_open(dev_t dev, int flag, int mode, struct lwp *l)
   96 {
   97 #ifdef __HAVE_MM_MD_OPEN
   98         int error;
   99         if ((error = mm_md_open(dev, flag, mode, l)) != 0)
  100                 return error;
  101 #endif
  102         l->l_proc->p_flag |= PK_KMEM;
  103         return 0;
  104 }
  105 
  106 /*
  107  * mm_init: initialize memory device driver.
  108  */
  109 void
  110 mm_init(void)
  111 {
  112         vaddr_t pg;
  113 
  114         mutex_init(&dev_mem_lock, MUTEX_DEFAULT, IPL_NONE);
  115 
  116         /* Read-only zero-page. */
  117         pg = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
  118         KASSERT(pg != 0);
  119         pmap_protect(pmap_kernel(), pg, pg + PAGE_SIZE, VM_PROT_READ);
  120         pmap_update(pmap_kernel());
  121         dev_zero_page = (void *)pg;
  122 
  123 #ifndef __HAVE_MM_MD_CACHE_ALIASING
  124         /* KVA for mappings during I/O. */
  125         dev_mem_addr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
  126             UVM_KMF_VAONLY|UVM_KMF_WAITVA);
  127         KASSERT(dev_mem_addr != 0);
  128 #else
  129         dev_mem_addr = 0;
  130 #endif
  131 }
  132 
  133 
  134 /*
  135  * dev_mem_getva: get a special virtual address.  If architecture requires,
  136  * allocate VA according to PA, which avoids cache-aliasing issues.  Use a
  137  * constant, general mapping address otherwise.
  138  */
  139 static inline vaddr_t
  140 dev_mem_getva(paddr_t pa, int color)
  141 {
  142 #ifdef __HAVE_MM_MD_CACHE_ALIASING
  143         return uvm_km_alloc(kernel_map, PAGE_SIZE,
  144             color & uvmexp.colormask,
  145             UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH);
  146 #else
  147         return dev_mem_addr;
  148 #endif
  149 }
  150 
  151 static inline void
  152 dev_mem_relva(paddr_t pa, vaddr_t va)
  153 {
  154 #ifdef __HAVE_MM_MD_CACHE_ALIASING
  155         uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY);
  156 #else
  157         KASSERT(dev_mem_addr == va);
  158 #endif
  159 }
  160 
  161 /*
  162  * dev_kmem_readwrite: helper for DEV_MEM (/dev/mem) case of R/W.
  163  */
  164 static int
  165 dev_mem_readwrite(struct uio *uio, struct iovec *iov)
  166 {
  167         paddr_t paddr;
  168         vaddr_t vaddr;
  169         vm_prot_t prot;
  170         size_t len, offset;
  171         bool have_direct;
  172         int error;
  173         int color = 0;
  174 
  175         /* Check for wrap around. */
  176         if ((uintptr_t)uio->uio_offset != uio->uio_offset) {
  177                 return EFAULT;
  178         }
  179         paddr = uio->uio_offset & ~PAGE_MASK;
  180         prot = (uio->uio_rw == UIO_WRITE) ? VM_PROT_WRITE : VM_PROT_READ;
  181         error = mm_md_physacc(paddr, prot);
  182         if (error) {
  183                 return error;
  184         }
  185         offset = uio->uio_offset & PAGE_MASK;
  186         len = MIN(uio->uio_resid, PAGE_SIZE - offset);
  187 
  188 #ifdef __HAVE_MM_MD_CACHE_ALIASING
  189         have_direct = mm_md_page_color(paddr, &color);
  190 #else
  191         have_direct = true;
  192         color = 0;
  193 #endif
  194 
  195 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
  196         /* Is physical address directly mapped?  Return VA. */
  197         if (have_direct)
  198                 have_direct = mm_md_direct_mapped_phys(paddr, &vaddr);
  199 #else
  200         vaddr = 0;
  201         have_direct = false;
  202 #endif
  203         if (!have_direct) {
  204                 /* Get a special virtual address. */
  205                 const vaddr_t va = dev_mem_getva(paddr, color);
  206 
  207                 /* Map selected KVA to physical address. */
  208                 mutex_enter(&dev_mem_lock);
  209                 pmap_kenter_pa(va, paddr, prot, 0);
  210                 pmap_update(pmap_kernel());
  211 
  212                 /* Perform I/O. */
  213                 vaddr = va + offset;
  214                 error = uiomove((void *)vaddr, len, uio);
  215 
  216                 /* Unmap, flush before unlock. */
  217                 pmap_kremove(va, PAGE_SIZE);
  218                 pmap_update(pmap_kernel());
  219                 mutex_exit(&dev_mem_lock);
  220 
  221                 /* "Release" the virtual address. */
  222                 dev_mem_relva(paddr, va);
  223         } else {
  224                 /* Direct map, just perform I/O. */
  225                 vaddr += offset;
  226                 error = uiomove((void *)vaddr, len, uio);
  227         }
  228         return error;
  229 }
  230 
  231 /*
  232  * dev_kmem_readwrite: helper for DEV_KMEM (/dev/kmem) case of R/W.
  233  */
  234 static int
  235 dev_kmem_readwrite(struct uio *uio, struct iovec *iov)
  236 {
  237         void *addr;
  238         size_t len, offset;
  239         vm_prot_t prot;
  240         int error;
  241         bool md_kva;
  242 
  243         /* Check for wrap around. */
  244         addr = (void *)(intptr_t)uio->uio_offset;
  245         if ((uintptr_t)addr != uio->uio_offset) {
  246                 return EFAULT;
  247         }
  248         /*
  249          * Handle non-page aligned offset.
  250          * Otherwise, we operate in page-by-page basis.
  251          */
  252         offset = uio->uio_offset & PAGE_MASK;
  253         len = MIN(uio->uio_resid, PAGE_SIZE - offset);
  254         prot = (uio->uio_rw == UIO_WRITE) ? VM_PROT_WRITE : VM_PROT_READ;
  255 
  256         md_kva = false;
  257 
  258 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_IO
  259         paddr_t paddr;
  260         /* MD case: is this is a directly mapped address? */
  261         if (mm_md_direct_mapped_io(addr, &paddr)) {
  262                 /* If so, validate physical address. */
  263                 error = mm_md_physacc(paddr, prot);
  264                 if (error) {
  265                         return error;
  266                 }
  267                 md_kva = true;
  268         }
  269 #endif
  270         if (!md_kva) {
  271                 bool checked = false;
  272 
  273 #ifdef __HAVE_MM_MD_KERNACC
  274                 /* MD check for the address. */
  275                 error = mm_md_kernacc(addr, prot, &checked);
  276                 if (error) {
  277                         return error;
  278                 }
  279 #endif
  280                 /* UVM check for the address (unless MD indicated to not). */
  281                 if (!checked && !uvm_kernacc(addr, len, prot)) {
  282                         return EFAULT;
  283                 }
  284         }
  285         error = uiomove(addr, len, uio);
  286         return error;
  287 }
  288 
  289 /*
  290  * dev_zero_readwrite: helper for DEV_ZERO (/dev/null) case of R/W.
  291  */
  292 static inline int
  293 dev_zero_readwrite(struct uio *uio, struct iovec *iov)
  294 {
  295         size_t len;
  296 
  297         /* Nothing to do for the write case. */
  298         if (uio->uio_rw == UIO_WRITE) {
  299                 uio->uio_resid = 0;
  300                 return 0;
  301         }
  302         /*
  303          * Read in page-by-page basis, caller will continue.
  304          * Cut appropriately for a single/last-iteration cases.
  305          */
  306         len = MIN(iov->iov_len, PAGE_SIZE);
  307         return uiomove(dev_zero_page, len, uio);
  308 }
  309 
  310 /*
  311  * mm_readwrite: general memory R/W function.
  312  */
  313 static int
  314 mm_readwrite(dev_t dev, struct uio *uio, int flags)
  315 {
  316         struct iovec *iov;
  317         int error;
  318 
  319 #ifdef __HAVE_MM_MD_READWRITE
  320         /* If defined - there are extra MD cases. */
  321         switch (minor(dev)) {
  322         case DEV_MEM:
  323         case DEV_KMEM:
  324         case DEV_NULL:
  325         case DEV_ZERO:
  326 #if defined(COMPAT_16) && defined(__arm)
  327         case _DEV_ZERO_oARM:
  328 #endif
  329                 break;
  330         default:
  331                 return mm_md_readwrite(dev, uio);
  332         }
  333 #endif
  334         error = 0;
  335         while (uio->uio_resid > 0 && error == 0) {
  336                 iov = uio->uio_iov;
  337                 if (iov->iov_len == 0) {
  338                         /* Processed; next I/O vector. */
  339                         uio->uio_iov++;
  340                         uio->uio_iovcnt--;
  341                         KASSERT(uio->uio_iovcnt >= 0);
  342                         continue;
  343                 }
  344                 /* Helper functions will process in page-by-page basis. */
  345                 switch (minor(dev)) {
  346                 case DEV_MEM:
  347                         error = dev_mem_readwrite(uio, iov);
  348                         break;
  349                 case DEV_KMEM:
  350                         error = dev_kmem_readwrite(uio, iov);
  351                         break;
  352                 case DEV_NULL:
  353                         if (uio->uio_rw == UIO_WRITE) {
  354                                 uio->uio_resid = 0;
  355                         }
  356                         /* Break directly out of the loop. */
  357                         return 0;
  358                 case DEV_FULL:
  359                         if (uio->uio_rw == UIO_WRITE) {
  360                                 return ENOSPC;
  361                         }
  362 #if defined(COMPAT_16) && defined(__arm)
  363                         /* FALLTHROUGH */
  364                 case _DEV_ZERO_oARM:
  365 #endif
  366                 /* FALLTHROUGH */
  367                 case DEV_ZERO:
  368                         error = dev_zero_readwrite(uio, iov);
  369                         break;
  370                 default:
  371                         error = ENXIO;
  372                         break;
  373                 }
  374         }
  375         return error;
  376 }
  377 
  378 /*
  379  * mm_mmap: general mmap() handler.
  380  */
  381 static paddr_t
  382 mm_mmap(dev_t dev, off_t off, int acc)
  383 {
  384         vm_prot_t prot;
  385 
  386 #ifdef __HAVE_MM_MD_MMAP
  387         /* If defined - there are extra mmap() MD cases. */
  388         switch (minor(dev)) {
  389         case DEV_MEM:
  390         case DEV_KMEM:
  391         case DEV_NULL:
  392 #if defined(COMPAT_16) && defined(__arm)
  393         case _DEV_ZERO_oARM:
  394 #endif
  395         case DEV_ZERO:
  396                 break;
  397         default:
  398                 return mm_md_mmap(dev, off, acc);
  399         }
  400 #endif
  401         /*
  402          * /dev/null does not make sense, /dev/kmem is volatile and
  403          * /dev/zero is handled in mmap already.
  404          */
  405         if (minor(dev) != DEV_MEM) {
  406                 return -1;
  407         }
  408 
  409         prot = 0;
  410         if (acc & PROT_EXEC)
  411                 prot |= VM_PROT_EXECUTE;
  412         if (acc & PROT_READ)
  413                 prot |= VM_PROT_READ;
  414         if (acc & PROT_WRITE)
  415                 prot |= VM_PROT_WRITE;
  416 
  417         /* Validate the physical address. */
  418         if (mm_md_physacc(off, prot) != 0) {
  419                 return -1;
  420         }
  421         return off >> PGSHIFT;
  422 }
  423 
  424 static int
  425 mm_ioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
  426 {
  427 
  428         switch (cmd) {
  429         case FIONBIO:
  430                 /* We never block anyway. */
  431                 return 0;
  432 
  433         case FIOSETOWN:
  434         case FIOGETOWN:
  435         case TIOCGPGRP:
  436         case TIOCSPGRP:
  437         case TIOCGETA:
  438                 return ENOTTY;
  439 
  440         case FIOASYNC:
  441                 if ((*(int *)data) == 0) {
  442                         return 0;
  443                 }
  444                 /* FALLTHROUGH */
  445         default:
  446                 return EOPNOTSUPP;
  447         }
  448 }
Cache object: 4ac0e46818a25b4f796d55c74c7969ee
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/mm.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/mm.c
