The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_uio.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1982, 1986, 1991, 1993
    3  *      The Regents of the University of California.  All rights reserved.
    4  * (c) UNIX System Laboratories, Inc.
    5  * All or some portions of this file are derived from material licensed
    6  * to the University of California by American Telephone and Telegraph
    7  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
    8  * the permission of UNIX System Laboratories, Inc.
    9  *
   10  * Copyright (c) 2014 The FreeBSD Foundation
   11  *
   12  * Portions of this software were developed by Konstantin Belousov
   13  * under sponsorship from the FreeBSD Foundation.
   14  *
   15  * Redistribution and use in source and binary forms, with or without
   16  * modification, are permitted provided that the following conditions
   17  * are met:
   18  * 1. Redistributions of source code must retain the above copyright
   19  *    notice, this list of conditions and the following disclaimer.
   20  * 2. Redistributions in binary form must reproduce the above copyright
   21  *    notice, this list of conditions and the following disclaimer in the
   22  *    documentation and/or other materials provided with the distribution.
   23  * 4. Neither the name of the University nor the names of its contributors
   24  *    may be used to endorse or promote products derived from this software
   25  *    without specific prior written permission.
   26  *
   27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   37  * SUCH DAMAGE.
   38  *
   39  *      @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
   40  */
   41 
   42 #include <sys/cdefs.h>
   43 __FBSDID("$FreeBSD: stable/10/sys/kern/subr_uio.c 308103 2016-10-30 11:45:01Z kib $");
   44 
   45 #include <sys/param.h>
   46 #include <sys/systm.h>
   47 #include <sys/kernel.h>
   48 #include <sys/limits.h>
   49 #include <sys/lock.h>
   50 #include <sys/mman.h>
   51 #include <sys/proc.h>
   52 #include <sys/resourcevar.h>
   53 #include <sys/rwlock.h>
   54 #include <sys/sched.h>
   55 #include <sys/sysctl.h>
   56 #include <sys/vnode.h>
   57 
   58 #include <vm/vm.h>
   59 #include <vm/vm_param.h>
   60 #include <vm/vm_extern.h>
   61 #include <vm/vm_page.h>
   62 #include <vm/vm_pageout.h>
   63 #include <vm/vm_map.h>
   64 
   65 #include <machine/bus.h>
   66 
   67 SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, UIO_MAXIOV,
   68         "Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
   69 
   70 static int uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault);
   71 
   72 int
   73 copyin_nofault(const void *udaddr, void *kaddr, size_t len)
   74 {
   75         int error, save;
   76 
   77         save = vm_fault_disable_pagefaults();
   78         error = copyin(udaddr, kaddr, len);
   79         vm_fault_enable_pagefaults(save);
   80         return (error);
   81 }
   82 
   83 int
   84 copyout_nofault(const void *kaddr, void *udaddr, size_t len)
   85 {
   86         int error, save;
   87 
   88         save = vm_fault_disable_pagefaults();
   89         error = copyout(kaddr, udaddr, len);
   90         vm_fault_enable_pagefaults(save);
   91         return (error);
   92 }
   93 
   94 #define PHYS_PAGE_COUNT(len)    (howmany(len, PAGE_SIZE) + 1)
   95 
   96 int
   97 physcopyin(void *src, vm_paddr_t dst, size_t len)
   98 {
   99         vm_page_t m[PHYS_PAGE_COUNT(len)];
  100         struct iovec iov[1];
  101         struct uio uio;
  102         int i;
  103 
  104         iov[0].iov_base = src;
  105         iov[0].iov_len = len;
  106         uio.uio_iov = iov;
  107         uio.uio_iovcnt = 1;
  108         uio.uio_offset = 0;
  109         uio.uio_resid = len;
  110         uio.uio_segflg = UIO_SYSSPACE;
  111         uio.uio_rw = UIO_WRITE;
  112         for (i = 0; i < PHYS_PAGE_COUNT(len); i++, dst += PAGE_SIZE)
  113                 m[i] = PHYS_TO_VM_PAGE(dst);
  114         return (uiomove_fromphys(m, dst & PAGE_MASK, len, &uio));
  115 }
  116 
  117 int
  118 physcopyout(vm_paddr_t src, void *dst, size_t len)
  119 {
  120         vm_page_t m[PHYS_PAGE_COUNT(len)];
  121         struct iovec iov[1];
  122         struct uio uio;
  123         int i;
  124 
  125         iov[0].iov_base = dst;
  126         iov[0].iov_len = len;
  127         uio.uio_iov = iov;
  128         uio.uio_iovcnt = 1;
  129         uio.uio_offset = 0;
  130         uio.uio_resid = len;
  131         uio.uio_segflg = UIO_SYSSPACE;
  132         uio.uio_rw = UIO_READ;
  133         for (i = 0; i < PHYS_PAGE_COUNT(len); i++, src += PAGE_SIZE)
  134                 m[i] = PHYS_TO_VM_PAGE(src);
  135         return (uiomove_fromphys(m, src & PAGE_MASK, len, &uio));
  136 }
  137 
  138 #undef PHYS_PAGE_COUNT
  139 
  140 int
  141 physcopyin_vlist(bus_dma_segment_t *src, off_t offset, vm_paddr_t dst,
  142     size_t len)
  143 {
  144         size_t seg_len;
  145         int error;
  146 
  147         error = 0;
  148         while (offset >= src->ds_len) {
  149                 offset -= src->ds_len;
  150                 src++;
  151         }
  152 
  153         while (len > 0 && error == 0) {
  154                 seg_len = MIN(src->ds_len - offset, len);
  155                 error = physcopyin((void *)(uintptr_t)(src->ds_addr + offset),
  156                     dst, seg_len);
  157                 offset = 0;
  158                 src++;
  159                 len -= seg_len;
  160                 dst += seg_len;
  161         }
  162 
  163         return (error);
  164 }
  165 
  166 int
  167 physcopyout_vlist(vm_paddr_t src, bus_dma_segment_t *dst, off_t offset,
  168     size_t len)
  169 {
  170         size_t seg_len;
  171         int error;
  172 
  173         error = 0;
  174         while (offset >= dst->ds_len) {
  175                 offset -= dst->ds_len;
  176                 dst++;
  177         }
  178 
  179         while (len > 0 && error == 0) {
  180                 seg_len = MIN(dst->ds_len - offset, len);
  181                 error = physcopyout(src, (void *)(uintptr_t)(dst->ds_addr +
  182                     offset), seg_len);
  183                 offset = 0;
  184                 dst++;
  185                 len -= seg_len;
  186                 src += seg_len;
  187         }
  188 
  189         return (error);
  190 }
  191 
  192 int
  193 uiomove(void *cp, int n, struct uio *uio)
  194 {
  195 
  196         return (uiomove_faultflag(cp, n, uio, 0));
  197 }
  198 
  199 int
  200 uiomove_nofault(void *cp, int n, struct uio *uio)
  201 {
  202 
  203         return (uiomove_faultflag(cp, n, uio, 1));
  204 }
  205 
  206 static int
  207 uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
  208 {
  209         struct thread *td;
  210         struct iovec *iov;
  211         size_t cnt;
  212         int error, newflags, save;
  213 
  214         td = curthread;
  215         error = 0;
  216 
  217         KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
  218             ("uiomove: mode"));
  219         KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == td,
  220             ("uiomove proc"));
  221         if (!nofault)
  222                 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
  223                     "Calling uiomove()");
  224 
  225         /* XXX does it make a sense to set TDP_DEADLKTREAT for UIO_SYSSPACE ? */
  226         newflags = TDP_DEADLKTREAT;
  227         if (uio->uio_segflg == UIO_USERSPACE && nofault) {
  228                 /*
  229                  * Fail if a non-spurious page fault occurs.
  230                  */
  231                 newflags |= TDP_NOFAULTING | TDP_RESETSPUR;
  232         }
  233         save = curthread_pflags_set(newflags);
  234 
  235         while (n > 0 && uio->uio_resid) {
  236                 iov = uio->uio_iov;
  237                 cnt = iov->iov_len;
  238                 if (cnt == 0) {
  239                         uio->uio_iov++;
  240                         uio->uio_iovcnt--;
  241                         continue;
  242                 }
  243                 if (cnt > n)
  244                         cnt = n;
  245 
  246                 switch (uio->uio_segflg) {
  247 
  248                 case UIO_USERSPACE:
  249                         maybe_yield();
  250                         if (uio->uio_rw == UIO_READ)
  251                                 error = copyout(cp, iov->iov_base, cnt);
  252                         else
  253                                 error = copyin(iov->iov_base, cp, cnt);
  254                         if (error)
  255                                 goto out;
  256                         break;
  257 
  258                 case UIO_SYSSPACE:
  259                         if (uio->uio_rw == UIO_READ)
  260                                 bcopy(cp, iov->iov_base, cnt);
  261                         else
  262                                 bcopy(iov->iov_base, cp, cnt);
  263                         break;
  264                 case UIO_NOCOPY:
  265                         break;
  266                 }
  267                 iov->iov_base = (char *)iov->iov_base + cnt;
  268                 iov->iov_len -= cnt;
  269                 uio->uio_resid -= cnt;
  270                 uio->uio_offset += cnt;
  271                 cp = (char *)cp + cnt;
  272                 n -= cnt;
  273         }
  274 out:
  275         curthread_pflags_restore(save);
  276         return (error);
  277 }
  278 
  279 /*
  280  * Wrapper for uiomove() that validates the arguments against a known-good
  281  * kernel buffer.  Currently, uiomove accepts a signed (n) argument, which
  282  * is almost definitely a bad thing, so we catch that here as well.  We
  283  * return a runtime failure, but it might be desirable to generate a runtime
  284  * assertion failure instead.
  285  */
  286 int
  287 uiomove_frombuf(void *buf, int buflen, struct uio *uio)
  288 {
  289         size_t offset, n;
  290 
  291         if (uio->uio_offset < 0 || uio->uio_resid < 0 ||
  292             (offset = uio->uio_offset) != uio->uio_offset)
  293                 return (EINVAL);
  294         if (buflen <= 0 || offset >= buflen)
  295                 return (0);
  296         if ((n = buflen - offset) > IOSIZE_MAX)
  297                 return (EINVAL);
  298         return (uiomove((char *)buf + offset, n, uio));
  299 }
  300 
  301 /*
  302  * Give next character to user as result of read.
  303  */
  304 int
  305 ureadc(int c, struct uio *uio)
  306 {
  307         struct iovec *iov;
  308         char *iov_base;
  309 
  310         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
  311             "Calling ureadc()");
  312 
  313 again:
  314         if (uio->uio_iovcnt == 0 || uio->uio_resid == 0)
  315                 panic("ureadc");
  316         iov = uio->uio_iov;
  317         if (iov->iov_len == 0) {
  318                 uio->uio_iovcnt--;
  319                 uio->uio_iov++;
  320                 goto again;
  321         }
  322         switch (uio->uio_segflg) {
  323 
  324         case UIO_USERSPACE:
  325                 if (subyte(iov->iov_base, c) < 0)
  326                         return (EFAULT);
  327                 break;
  328 
  329         case UIO_SYSSPACE:
  330                 iov_base = iov->iov_base;
  331                 *iov_base = c;
  332                 break;
  333 
  334         case UIO_NOCOPY:
  335                 break;
  336         }
  337         iov->iov_base = (char *)iov->iov_base + 1;
  338         iov->iov_len--;
  339         uio->uio_resid--;
  340         uio->uio_offset++;
  341         return (0);
  342 }
  343 
  344 int
  345 copyinfrom(const void * __restrict src, void * __restrict dst, size_t len,
  346     int seg)
  347 {
  348         int error = 0;
  349 
  350         switch (seg) {
  351         case UIO_USERSPACE:
  352                 error = copyin(src, dst, len);
  353                 break;
  354         case UIO_SYSSPACE:
  355                 bcopy(src, dst, len);
  356                 break;
  357         default:
  358                 panic("copyinfrom: bad seg %d\n", seg);
  359         }
  360         return (error);
  361 }
  362 
  363 int
  364 copyinstrfrom(const void * __restrict src, void * __restrict dst, size_t len,
  365     size_t * __restrict copied, int seg)
  366 {
  367         int error = 0;
  368 
  369         switch (seg) {
  370         case UIO_USERSPACE:
  371                 error = copyinstr(src, dst, len, copied);
  372                 break;
  373         case UIO_SYSSPACE:
  374                 error = copystr(src, dst, len, copied);
  375                 break;
  376         default:
  377                 panic("copyinstrfrom: bad seg %d\n", seg);
  378         }
  379         return (error);
  380 }
  381 
  382 int
  383 copyiniov(const struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error)
  384 {
  385         u_int iovlen;
  386 
  387         *iov = NULL;
  388         if (iovcnt > UIO_MAXIOV)
  389                 return (error);
  390         iovlen = iovcnt * sizeof (struct iovec);
  391         *iov = malloc(iovlen, M_IOV, M_WAITOK);
  392         error = copyin(iovp, *iov, iovlen);
  393         if (error) {
  394                 free(*iov, M_IOV);
  395                 *iov = NULL;
  396         }
  397         return (error);
  398 }
  399 
  400 int
  401 copyinuio(const struct iovec *iovp, u_int iovcnt, struct uio **uiop)
  402 {
  403         struct iovec *iov;
  404         struct uio *uio;
  405         u_int iovlen;
  406         int error, i;
  407 
  408         *uiop = NULL;
  409         if (iovcnt > UIO_MAXIOV)
  410                 return (EINVAL);
  411         iovlen = iovcnt * sizeof (struct iovec);
  412         uio = malloc(iovlen + sizeof *uio, M_IOV, M_WAITOK);
  413         iov = (struct iovec *)(uio + 1);
  414         error = copyin(iovp, iov, iovlen);
  415         if (error) {
  416                 free(uio, M_IOV);
  417                 return (error);
  418         }
  419         uio->uio_iov = iov;
  420         uio->uio_iovcnt = iovcnt;
  421         uio->uio_segflg = UIO_USERSPACE;
  422         uio->uio_offset = -1;
  423         uio->uio_resid = 0;
  424         for (i = 0; i < iovcnt; i++) {
  425                 if (iov->iov_len > IOSIZE_MAX - uio->uio_resid) {
  426                         free(uio, M_IOV);
  427                         return (EINVAL);
  428                 }
  429                 uio->uio_resid += iov->iov_len;
  430                 iov++;
  431         }
  432         *uiop = uio;
  433         return (0);
  434 }
  435 
  436 struct uio *
  437 cloneuio(struct uio *uiop)
  438 {
  439         struct uio *uio;
  440         int iovlen;
  441 
  442         iovlen = uiop->uio_iovcnt * sizeof (struct iovec);
  443         uio = malloc(iovlen + sizeof *uio, M_IOV, M_WAITOK);
  444         *uio = *uiop;
  445         uio->uio_iov = (struct iovec *)(uio + 1);
  446         bcopy(uiop->uio_iov, uio->uio_iov, iovlen);
  447         return (uio);
  448 }
  449 
  450 /*
  451  * Map some anonymous memory in user space of size sz, rounded up to the page
  452  * boundary.
  453  */
  454 int
  455 copyout_map(struct thread *td, vm_offset_t *addr, size_t sz)
  456 {
  457         struct vmspace *vms;
  458         int error;
  459         vm_size_t size;
  460 
  461         vms = td->td_proc->p_vmspace;
  462 
  463         /*
  464          * Map somewhere after heap in process memory.
  465          */
  466         PROC_LOCK(td->td_proc);
  467         *addr = round_page((vm_offset_t)vms->vm_daddr +
  468             lim_max(td->td_proc, RLIMIT_DATA));
  469         PROC_UNLOCK(td->td_proc);
  470 
  471         /* round size up to page boundary */
  472         size = (vm_size_t)round_page(sz);
  473 
  474         error = vm_mmap(&vms->vm_map, addr, size, PROT_READ | PROT_WRITE,
  475             VM_PROT_ALL, MAP_PRIVATE | MAP_ANON, OBJT_DEFAULT, NULL, 0);
  476 
  477         return (error);
  478 }
  479 
  480 /*
  481  * Unmap memory in user space.
  482  */
  483 int
  484 copyout_unmap(struct thread *td, vm_offset_t addr, size_t sz)
  485 {
  486         vm_map_t map;
  487         vm_size_t size;
  488 
  489         if (sz == 0)
  490                 return (0);
  491 
  492         map = &td->td_proc->p_vmspace->vm_map;
  493         size = (vm_size_t)round_page(sz);
  494 
  495         if (vm_map_remove(map, addr, addr + size) != KERN_SUCCESS)
  496                 return (EINVAL);
  497 
  498         return (0);
  499 }
  500 
  501 #ifdef NO_FUEWORD
  502 /*
  503  * XXXKIB The temporal implementation of fue*() functions which do not
  504  * handle usermode -1 properly, mixing it with the fault code.  Keep
  505  * this until MD code is written.  Currently sparc64, mips and arm do
  506  * not have proper implementation.
  507  */
  508 
  509 int
  510 fueword(volatile const void *base, long *val)
  511 {
  512         long res;
  513 
  514         res = fuword(base);
  515         if (res == -1)
  516                 return (-1);
  517         *val = res;
  518         return (0);
  519 }
  520 
  521 int
  522 fueword32(volatile const void *base, int32_t *val)
  523 {
  524         int32_t res;
  525 
  526         res = fuword32(base);
  527         if (res == -1)
  528                 return (-1);
  529         *val = res;
  530         return (0);
  531 }
  532 
  533 #ifdef _LP64
  534 int
  535 fueword64(volatile const void *base, int64_t *val)
  536 {
  537         int64_t res;
  538 
  539         res = fuword64(base);
  540         if (res == -1)
  541                 return (-1);
  542         *val = res;
  543         return (0);
  544 }
  545 #endif
  546 
  547 int
  548 casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
  549     uint32_t newval)
  550 {
  551         int32_t ov;
  552 
  553         ov = casuword32(base, oldval, newval);
  554         if (ov == -1)
  555                 return (-1);
  556         *oldvalp = ov;
  557         return (0);
  558 }
  559 
  560 int
  561 casueword(volatile u_long *p, u_long oldval, u_long *oldvalp, u_long newval)
  562 {
  563         u_long ov;
  564 
  565         ov = casuword(p, oldval, newval);
  566         if (ov == -1)
  567                 return (-1);
  568         *oldvalp = ov;
  569         return (0);
  570 }
  571 #else /* NO_FUEWORD */
  572 int32_t
  573 fuword32(volatile const void *addr)
  574 {
  575         int rv;
  576         int32_t val;
  577 
  578         rv = fueword32(addr, &val);
  579         return (rv == -1 ? -1 : val);
  580 }
  581 
  582 #ifdef _LP64
  583 int64_t
  584 fuword64(volatile const void *addr)
  585 {
  586         int rv;
  587         int64_t val;
  588 
  589         rv = fueword64(addr, &val);
  590         return (rv == -1 ? -1 : val);
  591 }
  592 #endif /* _LP64 */
  593 
  594 long
  595 fuword(volatile const void *addr)
  596 {
  597         long val;
  598         int rv;
  599 
  600         rv = fueword(addr, &val);
  601         return (rv == -1 ? -1 : val);
  602 }
  603 
  604 uint32_t
  605 casuword32(volatile uint32_t *addr, uint32_t old, uint32_t new)
  606 {
  607         int rv;
  608         uint32_t val;
  609 
  610         rv = casueword32(addr, old, &val, new);
  611         return (rv == -1 ? -1 : val);
  612 }
  613 
  614 u_long
  615 casuword(volatile u_long *addr, u_long old, u_long new)
  616 {
  617         int rv;
  618         u_long val;
  619 
  620         rv = casueword(addr, old, &val, new);
  621         return (rv == -1 ? -1 : val);
  622 }
  623 
  624 #endif /* NO_FUEWORD */

Cache object: 8da706a9c4a9f6b27f88ca23203fb15c


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.