zfs_uio.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*
    2  * CDDL HEADER START
    3  *
    4  * The contents of this file are subject to the terms of the
    5  * Common Development and Distribution License (the "License").
    6  * You may not use this file except in compliance with the License.
    7  *
    8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
    9  * or https://opensource.org/licenses/CDDL-1.0.
   10  * See the License for the specific language governing permissions
   11  * and limitations under the License.
   12  *
   13  * When distributing Covered Code, include this CDDL HEADER in each
   14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
   15  * If applicable, add the following below this CDDL HEADER, with the
   16  * fields enclosed by brackets "[]" replaced with your own identifying
   17  * information: Portions Copyright [yyyy] [name of copyright owner]
   18  *
   19  * CDDL HEADER END
   20  */
   21 /*
   22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
   23  * Use is subject to license terms.
   24  */
   25 
   26 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
   27 /*        All Rights Reserved   */
   28 
   29 /*
   30  * University Copyright- Copyright (c) 1982, 1986, 1988
   31  * The Regents of the University of California
   32  * All Rights Reserved
   33  *
   34  * University Acknowledgment- Portions of this document are derived from
   35  * software developed by the University of California, Berkeley, and its
   36  * contributors.
   37  */
   38 /*
   39  * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
   40  */
   41 
   42 #ifdef _KERNEL
   43 
   44 #include <sys/types.h>
   45 #include <sys/uio_impl.h>
   46 #include <sys/sysmacros.h>
   47 #include <sys/string.h>
   48 #include <linux/kmap_compat.h>
   49 #include <linux/uaccess.h>
   50 
   51 /*
   52  * Move "n" bytes at byte address "p"; "rw" indicates the direction
   53  * of the move, and the I/O parameters are provided in "uio", which is
   54  * update to reflect the data which was moved.  Returns 0 on success or
   55  * a non-zero errno on failure.
   56  */
   57 static int
   58 zfs_uiomove_iov(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
   59 {
   60         const struct iovec *iov = uio->uio_iov;
   61         size_t skip = uio->uio_skip;
   62         ulong_t cnt;
   63 
   64         while (n && uio->uio_resid) {
   65                 cnt = MIN(iov->iov_len - skip, n);
   66                 switch (uio->uio_segflg) {
   67                 case UIO_USERSPACE:
   68                         /*
   69                          * p = kernel data pointer
   70                          * iov->iov_base = user data pointer
   71                          */
   72                         if (rw == UIO_READ) {
   73                                 if (copy_to_user(iov->iov_base+skip, p, cnt))
   74                                         return (EFAULT);
   75                         } else {
   76                                 unsigned long b_left = 0;
   77                                 if (uio->uio_fault_disable) {
   78                                         if (!zfs_access_ok(VERIFY_READ,
   79                                             (iov->iov_base + skip), cnt)) {
   80                                                 return (EFAULT);
   81                                         }
   82                                         pagefault_disable();
   83                                         b_left =
   84                                             __copy_from_user_inatomic(p,
   85                                             (iov->iov_base + skip), cnt);
   86                                         pagefault_enable();
   87                                 } else {
   88                                         b_left =
   89                                             copy_from_user(p,
   90                                             (iov->iov_base + skip), cnt);
   91                                 }
   92                                 if (b_left > 0) {
   93                                         unsigned long c_bytes =
   94                                             cnt - b_left;
   95                                         uio->uio_skip += c_bytes;
   96                                         ASSERT3U(uio->uio_skip, <,
   97                                             iov->iov_len);
   98                                         uio->uio_resid -= c_bytes;
   99                                         uio->uio_loffset += c_bytes;
  100                                         return (EFAULT);
  101                                 }
  102                         }
  103                         break;
  104                 case UIO_SYSSPACE:
  105                         if (rw == UIO_READ)
  106                                 memcpy(iov->iov_base + skip, p, cnt);
  107                         else
  108                                 memcpy(p, iov->iov_base + skip, cnt);
  109                         break;
  110                 default:
  111                         ASSERT(0);
  112                 }
  113                 skip += cnt;
  114                 if (skip == iov->iov_len) {
  115                         skip = 0;
  116                         uio->uio_iov = (++iov);
  117                         uio->uio_iovcnt--;
  118                 }
  119                 uio->uio_skip = skip;
  120                 uio->uio_resid -= cnt;
  121                 uio->uio_loffset += cnt;
  122                 p = (caddr_t)p + cnt;
  123                 n -= cnt;
  124         }
  125         return (0);
  126 }
  127 
  128 static int
  129 zfs_uiomove_bvec_impl(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
  130 {
  131         const struct bio_vec *bv = uio->uio_bvec;
  132         size_t skip = uio->uio_skip;
  133         ulong_t cnt;
  134 
  135         while (n && uio->uio_resid) {
  136                 void *paddr;
  137                 cnt = MIN(bv->bv_len - skip, n);
  138 
  139                 paddr = zfs_kmap_atomic(bv->bv_page);
  140                 if (rw == UIO_READ) {
  141                         /* Copy from buffer 'p' to the bvec data */
  142                         memcpy(paddr + bv->bv_offset + skip, p, cnt);
  143                 } else {
  144                         /* Copy from bvec data to buffer 'p' */
  145                         memcpy(p, paddr + bv->bv_offset + skip, cnt);
  146                 }
  147                 zfs_kunmap_atomic(paddr);
  148 
  149                 skip += cnt;
  150                 if (skip == bv->bv_len) {
  151                         skip = 0;
  152                         uio->uio_bvec = (++bv);
  153                         uio->uio_iovcnt--;
  154                 }
  155                 uio->uio_skip = skip;
  156                 uio->uio_resid -= cnt;
  157                 uio->uio_loffset += cnt;
  158                 p = (caddr_t)p + cnt;
  159                 n -= cnt;
  160         }
  161         return (0);
  162 }
  163 
  164 #ifdef HAVE_BLK_MQ
  165 static void
  166 zfs_copy_bvec(void *p, size_t skip, size_t cnt, zfs_uio_rw_t rw,
  167     struct bio_vec *bv)
  168 {
  169         void *paddr;
  170 
  171         paddr = zfs_kmap_atomic(bv->bv_page);
  172         if (rw == UIO_READ) {
  173                 /* Copy from buffer 'p' to the bvec data */
  174                 memcpy(paddr + bv->bv_offset + skip, p, cnt);
  175         } else {
  176                 /* Copy from bvec data to buffer 'p' */
  177                 memcpy(p, paddr + bv->bv_offset + skip, cnt);
  178         }
  179         zfs_kunmap_atomic(paddr);
  180 }
  181 
  182 /*
  183  * Copy 'n' bytes of data between the buffer p[] and the data represented
  184  * by the request in the uio.
  185  */
  186 static int
  187 zfs_uiomove_bvec_rq(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
  188 {
  189         struct request *rq = uio->rq;
  190         struct bio_vec bv;
  191         struct req_iterator iter;
  192         size_t this_seg_start;  /* logical offset */
  193         size_t this_seg_end;            /* logical offset */
  194         size_t skip_in_seg;
  195         size_t copy_from_seg;
  196         size_t orig_loffset;
  197         int copied = 0;
  198 
  199         /*
  200          * Get the original logical offset of this entire request (because
  201          * uio->uio_loffset will be modified over time).
  202          */
  203         orig_loffset = io_offset(NULL, rq);
  204         this_seg_start = orig_loffset;
  205 
  206         rq_for_each_segment(bv, rq, iter) {
  207                 if (uio->iter.bio) {
  208                         /*
  209                          * If uio->iter.bio is present, then we know we've saved
  210                          * uio->iter from a previous call to this function, and
  211                          * we can skip ahead in this rq_for_each_segment() loop
  212                          * to where we last left off.  That way, we don't need
  213                          * to iterate over tons of segments we've already
  214                          * processed - we can just restore the "saved state".
  215                          */
  216                         iter = uio->iter;
  217                         bv = uio->bv;
  218                         this_seg_start = uio->uio_loffset;
  219                         memset(&uio->iter, 0, sizeof (uio->iter));
  220                         continue;
  221                 }
  222 
  223                 /*
  224                  * Lookup what the logical offset of the last byte of this
  225                  * segment is.
  226                  */
  227                 this_seg_end = this_seg_start + bv.bv_len - 1;
  228 
  229                 /*
  230                  * We only need to operate on segments that have data we're
  231                  * copying.
  232                  */
  233                 if (uio->uio_loffset >= this_seg_start &&
  234                     uio->uio_loffset <= this_seg_end) {
  235                         /*
  236                          * Some, or all, of the data in this segment needs to be
  237                          * copied.
  238                          */
  239 
  240                         /*
  241                          * We may be not be copying from the first byte in the
  242                          * segment.  Figure out how many bytes to skip copying
  243                          * from the beginning of this segment.
  244                          */
  245                         skip_in_seg = uio->uio_loffset - this_seg_start;
  246 
  247                         /*
  248                          * Calculate the total number of bytes from this
  249                          * segment that we will be copying.
  250                          */
  251                         copy_from_seg = MIN(bv.bv_len - skip_in_seg, n);
  252 
  253                         /* Copy the bytes */
  254                         zfs_copy_bvec(p, skip_in_seg, copy_from_seg, rw, &bv);
  255                         p = ((char *)p) + copy_from_seg;
  256 
  257                         n -= copy_from_seg;
  258                         uio->uio_resid -= copy_from_seg;
  259                         uio->uio_loffset += copy_from_seg;
  260                         copied = 1;     /* We copied some data */
  261                 }
  262 
  263                 if (n == 0) {
  264                         /*
  265                          * All done copying.  Save our 'iter' value to the uio.
  266                          * This allows us to "save our state" and skip ahead in
  267                          * the rq_for_each_segment() loop the next time we call
  268                          * call zfs_uiomove_bvec_rq() on this uio (which we
  269                          * will be doing for any remaining data in the uio).
  270                          */
  271                         uio->iter = iter; /* make a copy of the struct data */
  272                         uio->bv = bv;
  273                         return (0);
  274                 }
  275 
  276                 this_seg_start = this_seg_end + 1;
  277         }
  278 
  279         if (!copied) {
  280                 /* Didn't copy anything */
  281                 uio->uio_resid = 0;
  282         }
  283         return (0);
  284 }
  285 #endif
  286 
  287 static int
  288 zfs_uiomove_bvec(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
  289 {
  290 #ifdef HAVE_BLK_MQ
  291         if (uio->rq != NULL)
  292                 return (zfs_uiomove_bvec_rq(p, n, rw, uio));
  293 #else
  294         ASSERT3P(uio->rq, ==, NULL);
  295 #endif
  296         return (zfs_uiomove_bvec_impl(p, n, rw, uio));
  297 }
  298 
  299 #if defined(HAVE_VFS_IOV_ITER)
  300 static int
  301 zfs_uiomove_iter(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio,
  302     boolean_t revert)
  303 {
  304         size_t cnt = MIN(n, uio->uio_resid);
  305 
  306         if (uio->uio_skip)
  307                 iov_iter_advance(uio->uio_iter, uio->uio_skip);
  308 
  309         if (rw == UIO_READ)
  310                 cnt = copy_to_iter(p, cnt, uio->uio_iter);
  311         else
  312                 cnt = copy_from_iter(p, cnt, uio->uio_iter);
  313 
  314         /*
  315          * When operating on a full pipe no bytes are processed.
  316          * In which case return EFAULT which is converted to EAGAIN
  317          * by the kernel's generic_file_splice_read() function.
  318          */
  319         if (cnt == 0)
  320                 return (EFAULT);
  321 
  322         /*
  323          * Revert advancing the uio_iter.  This is set by zfs_uiocopy()
  324          * to avoid consuming the uio and its iov_iter structure.
  325          */
  326         if (revert)
  327                 iov_iter_revert(uio->uio_iter, cnt);
  328 
  329         uio->uio_resid -= cnt;
  330         uio->uio_loffset += cnt;
  331 
  332         return (0);
  333 }
  334 #endif
  335 
  336 int
  337 zfs_uiomove(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
  338 {
  339         if (uio->uio_segflg == UIO_BVEC)
  340                 return (zfs_uiomove_bvec(p, n, rw, uio));
  341 #if defined(HAVE_VFS_IOV_ITER)
  342         else if (uio->uio_segflg == UIO_ITER)
  343                 return (zfs_uiomove_iter(p, n, rw, uio, B_FALSE));
  344 #endif
  345         else
  346                 return (zfs_uiomove_iov(p, n, rw, uio));
  347 }
  348 EXPORT_SYMBOL(zfs_uiomove);
  349 
  350 /*
  351  * Fault in the pages of the first n bytes specified by the uio structure.
  352  * 1 byte in each page is touched and the uio struct is unmodified. Any
  353  * error will terminate the process as this is only a best attempt to get
  354  * the pages resident.
  355  */
  356 int
  357 zfs_uio_prefaultpages(ssize_t n, zfs_uio_t *uio)
  358 {
  359         if (uio->uio_segflg == UIO_SYSSPACE || uio->uio_segflg == UIO_BVEC) {
  360                 /* There's never a need to fault in kernel pages */
  361                 return (0);
  362 #if defined(HAVE_VFS_IOV_ITER)
  363         } else if (uio->uio_segflg == UIO_ITER) {
  364                 /*
  365                  * At least a Linux 4.9 kernel, iov_iter_fault_in_readable()
  366                  * can be relied on to fault in user pages when referenced.
  367                  */
  368                 if (iov_iter_fault_in_readable(uio->uio_iter, n))
  369                         return (EFAULT);
  370 #endif
  371         } else {
  372                 /* Fault in all user pages */
  373                 ASSERT3S(uio->uio_segflg, ==, UIO_USERSPACE);
  374                 const struct iovec *iov = uio->uio_iov;
  375                 int iovcnt = uio->uio_iovcnt;
  376                 size_t skip = uio->uio_skip;
  377                 uint8_t tmp;
  378                 caddr_t p;
  379 
  380                 for (; n > 0 && iovcnt > 0; iov++, iovcnt--, skip = 0) {
  381                         ulong_t cnt = MIN(iov->iov_len - skip, n);
  382                         /* empty iov */
  383                         if (cnt == 0)
  384                                 continue;
  385                         n -= cnt;
  386                         /* touch each page in this segment. */
  387                         p = iov->iov_base + skip;
  388                         while (cnt) {
  389                                 if (copy_from_user(&tmp, p, 1))
  390                                         return (EFAULT);
  391                                 ulong_t incr = MIN(cnt, PAGESIZE);
  392                                 p += incr;
  393                                 cnt -= incr;
  394                         }
  395                         /* touch the last byte in case it straddles a page. */
  396                         p--;
  397                         if (copy_from_user(&tmp, p, 1))
  398                                 return (EFAULT);
  399                 }
  400         }
  401 
  402         return (0);
  403 }
  404 EXPORT_SYMBOL(zfs_uio_prefaultpages);
  405 
  406 /*
  407  * The same as zfs_uiomove() but doesn't modify uio structure.
  408  * return in cbytes how many bytes were copied.
  409  */
  410 int
  411 zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes)
  412 {
  413         zfs_uio_t uio_copy;
  414         int ret;
  415 
  416         memcpy(&uio_copy, uio, sizeof (zfs_uio_t));
  417 
  418         if (uio->uio_segflg == UIO_BVEC)
  419                 ret = zfs_uiomove_bvec(p, n, rw, &uio_copy);
  420 #if defined(HAVE_VFS_IOV_ITER)
  421         else if (uio->uio_segflg == UIO_ITER)
  422                 ret = zfs_uiomove_iter(p, n, rw, &uio_copy, B_TRUE);
  423 #endif
  424         else
  425                 ret = zfs_uiomove_iov(p, n, rw, &uio_copy);
  426 
  427         *cbytes = uio->uio_resid - uio_copy.uio_resid;
  428 
  429         return (ret);
  430 }
  431 EXPORT_SYMBOL(zfs_uiocopy);
  432 
  433 /*
  434  * Drop the next n chars out of *uio.
  435  */
  436 void
  437 zfs_uioskip(zfs_uio_t *uio, size_t n)
  438 {
  439         if (n > uio->uio_resid)
  440                 return;
  441         /*
  442          * When using a uio with a struct request, we simply
  443          * use uio_loffset as a pointer to the next logical byte to
  444          * copy in the request.  We don't have to do any fancy
  445          * accounting with uio_bvec/uio_iovcnt since we don't use
  446          * them.
  447          */
  448         if (uio->uio_segflg == UIO_BVEC && uio->rq == NULL) {
  449                 uio->uio_skip += n;
  450                 while (uio->uio_iovcnt &&
  451                     uio->uio_skip >= uio->uio_bvec->bv_len) {
  452                         uio->uio_skip -= uio->uio_bvec->bv_len;
  453                         uio->uio_bvec++;
  454                         uio->uio_iovcnt--;
  455                 }
  456 #if defined(HAVE_VFS_IOV_ITER)
  457         } else if (uio->uio_segflg == UIO_ITER) {
  458                 iov_iter_advance(uio->uio_iter, n);
  459 #endif
  460         } else {
  461                 uio->uio_skip += n;
  462                 while (uio->uio_iovcnt &&
  463                     uio->uio_skip >= uio->uio_iov->iov_len) {
  464                         uio->uio_skip -= uio->uio_iov->iov_len;
  465                         uio->uio_iov++;
  466                         uio->uio_iovcnt--;
  467                 }
  468         }
  469         uio->uio_loffset += n;
  470         uio->uio_resid -= n;
  471 }
  472 EXPORT_SYMBOL(zfs_uioskip);
  473 
  474 #endif /* _KERNEL */
Cache object: 26f25c4d03482a75e63367ef3bfe8a8c
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/contrib/openzfs/module/os/linux/zfs/zfs_uio.c

FreeBSD/Linux Kernel Cross Reference
sys/contrib/openzfs/module/os/linux/zfs/zfs_uio.c
