copyout.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2018 The FreeBSD Foundation
    5  *
    6  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
    7  * under sponsorship from the FreeBSD Foundation.
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  */
   30 
   31 #include <sys/cdefs.h>
   32 __FBSDID("$FreeBSD$");
   33 
   34 #include <sys/param.h>
   35 #include <sys/lock.h>
   36 #include <sys/mutex.h>
   37 #include <sys/pcpu.h>
   38 #include <sys/proc.h>
   39 #include <sys/sched.h>
   40 #include <sys/sysctl.h>
   41 #include <sys/systm.h>
   42 #include <vm/vm.h>
   43 #include <vm/vm_param.h>
   44 #include <vm/vm_extern.h>
   45 #include <vm/pmap.h>
   46 #include <vm/vm_map.h>
   47 #include <vm/vm_page.h>
   48 
   49 int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int);
   50 static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int);
   51 int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int);
   52 static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int);
   53 int fubyte_fast(volatile const void *base, u_int kcr3);
   54 static int (*fubyte_fast_tramp)(volatile const void *, u_int);
   55 int fuword16_fast(volatile const void *base, u_int kcr3);
   56 static int (*fuword16_fast_tramp)(volatile const void *, u_int);
   57 int fueword_fast(volatile const void *base, long *val, u_int kcr3);
   58 static int (*fueword_fast_tramp)(volatile const void *, long *, u_int);
   59 int subyte_fast(volatile void *base, int val, u_int kcr3);
   60 static int (*subyte_fast_tramp)(volatile void *, int, u_int);
   61 int suword16_fast(volatile void *base, int val, u_int kcr3);
   62 static int (*suword16_fast_tramp)(volatile void *, int, u_int);
   63 int suword_fast(volatile void *base, long val, u_int kcr3);
   64 static int (*suword_fast_tramp)(volatile void *, long, u_int);
   65 
   66 static int fast_copyout = 1;
   67 SYSCTL_INT(_machdep, OID_AUTO, fast_copyout, CTLFLAG_RWTUN,
   68     &fast_copyout, 0,
   69     "");
   70 
   71 void
   72 copyout_init_tramp(void)
   73 {
   74 
   75         copyin_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
   76             (uintptr_t)copyin_fast + setidt_disp);
   77         copyout_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
   78             (uintptr_t)copyout_fast + setidt_disp);
   79         fubyte_fast_tramp = (int (*)(volatile const void *, u_int))(
   80             (uintptr_t)fubyte_fast + setidt_disp);
   81         fuword16_fast_tramp = (int (*)(volatile const void *, u_int))(
   82             (uintptr_t)fuword16_fast + setidt_disp);
   83         fueword_fast_tramp = (int (*)(volatile const void *, long *, u_int))(
   84             (uintptr_t)fueword_fast + setidt_disp);
   85         subyte_fast_tramp = (int (*)(volatile void *, int, u_int))(
   86             (uintptr_t)subyte_fast + setidt_disp);
   87         suword16_fast_tramp = (int (*)(volatile void *, int, u_int))(
   88             (uintptr_t)suword16_fast + setidt_disp);
   89         suword_fast_tramp = (int (*)(volatile void *, long, u_int))(
   90             (uintptr_t)suword_fast + setidt_disp);
   91 }
   92 
   93 int
   94 cp_slow0(vm_offset_t uva, size_t len, bool write,
   95     void (*f)(vm_offset_t, void *), void *arg)
   96 {
   97         struct pcpu *pc;
   98         vm_page_t m[2];
   99         vm_offset_t kaddr;
  100         int error, i, plen;
  101         bool sleepable;
  102 
  103         plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE);
  104         MPASS(plen <= nitems(m));
  105         error = 0;
  106         i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len,
  107             (write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT,
  108             m, nitems(m));
  109         if (i != plen)
  110                 return (EFAULT);
  111         sched_pin();
  112         pc = get_pcpu();
  113         if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 ||
  114             (curthread->td_pflags & TDP_NOFAULTING) != 0) {
  115                 sleepable = false;
  116                 mtx_lock(&pc->pc_copyout_mlock);
  117                 kaddr = pc->pc_copyout_maddr;
  118         } else {
  119                 sleepable = true;
  120                 sx_xlock(&pc->pc_copyout_slock);
  121                 kaddr = pc->pc_copyout_saddr;
  122         }
  123         pmap_cp_slow0_map(kaddr, plen, m);
  124         kaddr += uva - trunc_page(uva);
  125         f(kaddr, arg);
  126         sched_unpin();
  127         if (sleepable)
  128                 sx_xunlock(&pc->pc_copyout_slock);
  129         else
  130                 mtx_unlock(&pc->pc_copyout_mlock);
  131         vm_page_unhold_pages(m, plen);
  132         return (error);
  133 }
  134 
  135 struct copyinstr_arg0 {
  136         vm_offset_t kc;
  137         size_t len;
  138         size_t alen;
  139         bool end;
  140 };
  141 
  142 static void
  143 copyinstr_slow0(vm_offset_t kva, void *arg)
  144 {
  145         struct copyinstr_arg0 *ca;
  146         char c;
  147 
  148         ca = arg;
  149         MPASS(ca->alen == 0 && ca->len > 0 && !ca->end);
  150         while (ca->alen < ca->len && !ca->end) {
  151                 c = *(char *)(kva + ca->alen);
  152                 *(char *)ca->kc = c;
  153                 ca->alen++;
  154                 ca->kc++;
  155                 if (c == '\0')
  156                         ca->end = true;
  157         }
  158 }
  159 
  160 int
  161 copyinstr(const void *udaddr, void *kaddr, size_t maxlen, size_t *lencopied)
  162 {
  163         struct copyinstr_arg0 ca;
  164         vm_offset_t uc;
  165         size_t plen;
  166         int error;
  167 
  168         error = 0;
  169         ca.end = false;
  170         for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
  171             plen < maxlen && !ca.end; uc += ca.alen, plen += ca.alen) {
  172                 ca.len = round_page(uc) - uc;
  173                 if (ca.len == 0)
  174                         ca.len = PAGE_SIZE;
  175                 if (plen + ca.len > maxlen)
  176                         ca.len = maxlen - plen;
  177                 ca.alen = 0;
  178                 if (cp_slow0(uc, ca.len, false, copyinstr_slow0, &ca) != 0) {
  179                         error = EFAULT;
  180                         break;
  181                 }
  182         }
  183         if (!ca.end && plen == maxlen && error == 0)
  184                 error = ENAMETOOLONG;
  185         if (lencopied != NULL)
  186                 *lencopied = plen;
  187         return (error);
  188 }
  189 
  190 struct copyin_arg0 {
  191         vm_offset_t kc;
  192         size_t len;
  193 };
  194 
  195 static void
  196 copyin_slow0(vm_offset_t kva, void *arg)
  197 {
  198         struct copyin_arg0 *ca;
  199 
  200         ca = arg;
  201         bcopy((void *)kva, (void *)ca->kc, ca->len);
  202 }
  203 
  204 int
  205 copyin(const void *udaddr, void *kaddr, size_t len)
  206 {
  207         struct copyin_arg0 ca;
  208         vm_offset_t uc;
  209         size_t plen;
  210 
  211         if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
  212             (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
  213                 return (EFAULT);
  214         if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
  215             copyin_fast_tramp(udaddr, kaddr, len, pmap_get_kcr3()) == 0))
  216                 return (0);
  217         for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
  218             plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
  219                 ca.len = round_page(uc) - uc;
  220                 if (ca.len == 0)
  221                         ca.len = PAGE_SIZE;
  222                 if (plen + ca.len > len)
  223                         ca.len = len - plen;
  224                 if (cp_slow0(uc, ca.len, false, copyin_slow0, &ca) != 0)
  225                         return (EFAULT);
  226         }
  227         return (0);
  228 }
  229 
  230 static void
  231 copyout_slow0(vm_offset_t kva, void *arg)
  232 {
  233         struct copyin_arg0 *ca;
  234 
  235         ca = arg;
  236         bcopy((void *)ca->kc, (void *)kva, ca->len);
  237 }
  238 
  239 int
  240 copyout(const void *kaddr, void *udaddr, size_t len)
  241 {
  242         struct copyin_arg0 ca;
  243         vm_offset_t uc;
  244         size_t plen;
  245 
  246         if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
  247             (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
  248                 return (EFAULT);
  249         if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
  250             copyout_fast_tramp(kaddr, udaddr, len, pmap_get_kcr3()) == 0))
  251                 return (0);
  252         for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
  253             plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
  254                 ca.len = round_page(uc) - uc;
  255                 if (ca.len == 0)
  256                         ca.len = PAGE_SIZE;
  257                 if (plen + ca.len > len)
  258                         ca.len = len - plen;
  259                 if (cp_slow0(uc, ca.len, true, copyout_slow0, &ca) != 0)
  260                         return (EFAULT);
  261         }
  262         return (0);
  263 }
  264 
  265 /*
  266  * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user
  267  * memory.
  268  */
  269 
  270 static void
  271 fubyte_slow0(vm_offset_t kva, void *arg)
  272 {
  273 
  274         *(int *)arg = *(u_char *)kva;
  275 }
  276 
  277 int
  278 fubyte(volatile const void *base)
  279 {
  280         int res;
  281 
  282         if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
  283             (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
  284                 return (-1);
  285         if (fast_copyout) {
  286                 res = fubyte_fast_tramp(base, pmap_get_kcr3());
  287                 if (res != -1)
  288                         return (res);
  289         }
  290         if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0,
  291             &res) != 0)
  292                 return (-1);
  293         return (res);
  294 }
  295 
  296 static void
  297 fuword16_slow0(vm_offset_t kva, void *arg)
  298 {
  299 
  300         *(int *)arg = *(uint16_t *)kva;
  301 }
  302 
  303 int
  304 fuword16(volatile const void *base)
  305 {
  306         int res;
  307 
  308         if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
  309             (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
  310                 return (-1);
  311         if (fast_copyout) {
  312                 res = fuword16_fast_tramp(base, pmap_get_kcr3());
  313                 if (res != -1)
  314                         return (res);
  315         }
  316         if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false,
  317             fuword16_slow0, &res) != 0)
  318                 return (-1);
  319         return (res);
  320 }
  321 
  322 static void
  323 fueword_slow0(vm_offset_t kva, void *arg)
  324 {
  325 
  326         *(uint32_t *)arg = *(uint32_t *)kva;
  327 }
  328 
  329 int
  330 fueword(volatile const void *base, long *val)
  331 {
  332         uint32_t res;
  333 
  334         if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base ||
  335             (uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS)
  336                 return (-1);
  337         if (fast_copyout) {
  338                 if (fueword_fast_tramp(base, val, pmap_get_kcr3()) == 0)
  339                         return (0);
  340         }
  341         if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0,
  342             &res) != 0)
  343                 return (-1);
  344         *val = res;
  345         return (0);
  346 }
  347 
  348 int
  349 fueword32(volatile const void *base, int32_t *val)
  350 {
  351 
  352         return (fueword(base, (long *)val));
  353 }
  354 
  355 /*
  356  * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory.
  357  */
  358 
  359 static void
  360 subyte_slow0(vm_offset_t kva, void *arg)
  361 {
  362 
  363         *(u_char *)kva = *(int *)arg;
  364 }
  365 
  366 int
  367 subyte(volatile void *base, int byte)
  368 {
  369 
  370         if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
  371             (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
  372                 return (-1);
  373         if (fast_copyout && subyte_fast_tramp(base, byte, pmap_get_kcr3()) == 0)
  374                 return (0);
  375         return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0,
  376             &byte) != 0 ? -1 : 0);
  377 }
  378 
  379 static void
  380 suword16_slow0(vm_offset_t kva, void *arg)
  381 {
  382 
  383         *(int *)kva = *(uint16_t *)arg;
  384 }
  385 
  386 int
  387 suword16(volatile void *base, int word)
  388 {
  389 
  390         if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
  391             (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
  392                 return (-1);
  393         if (fast_copyout && suword16_fast_tramp(base, word, pmap_get_kcr3())
  394             == 0)
  395                 return (0);
  396         return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true,
  397             suword16_slow0, &word) != 0 ? -1 : 0);
  398 }
  399 
  400 static void
  401 suword_slow0(vm_offset_t kva, void *arg)
  402 {
  403 
  404         *(int *)kva = *(uint32_t *)arg;
  405 }
  406 
  407 int
  408 suword(volatile void *base, long word)
  409 {
  410 
  411         if ((uintptr_t)base + sizeof(word) < (uintptr_t)base ||
  412             (uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS)
  413                 return (-1);
  414         if (fast_copyout && suword_fast_tramp(base, word, pmap_get_kcr3()) == 0)
  415                 return (0);
  416         return (cp_slow0((vm_offset_t)base, sizeof(long), true,
  417             suword_slow0, &word) != 0 ? -1 : 0);
  418 }
  419 
  420 int
  421 suword32(volatile void *base, int32_t word)
  422 {
  423 
  424         return (suword(base, word));
  425 }
  426 
  427 struct casueword_arg0 {
  428         uint32_t oldval;
  429         uint32_t newval;
  430         int res;
  431 };
  432 
  433 static void
  434 casueword_slow0(vm_offset_t kva, void *arg)
  435 {
  436         struct casueword_arg0 *ca;
  437 
  438         ca = arg;
  439         ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval,
  440             ca->newval);
  441 }
  442 
  443 int
  444 casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
  445     uint32_t newval)
  446 {
  447         struct casueword_arg0 ca;
  448         int res;
  449 
  450         ca.oldval = oldval;
  451         ca.newval = newval;
  452         res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
  453             casueword_slow0, &ca);
  454         if (res == 0) {
  455                 *oldvalp = ca.oldval;
  456                 return (ca.res);
  457         }
  458         return (-1);
  459 }
  460 
  461 int
  462 casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval)
  463 {
  464         struct casueword_arg0 ca;
  465         int res;
  466 
  467         ca.oldval = oldval;
  468         ca.newval = newval;
  469         res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
  470             casueword_slow0, &ca);
  471         if (res == 0) {
  472                 *oldvalp = ca.oldval;
  473                 return (ca.res);
  474         }
  475         return (-1);
  476 }
Cache object: 0c2680f28f4a6c052faa39114fe833d8
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/i386/i386/copyout.c

FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/copyout.c
