copyinout.S

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10

    1 /*-
    2  * Copyright (c) 2015 The FreeBSD Foundation
    3  * All rights reserved.
    4  *
    5  * This software was developed by Andrew Turner under
    6  * sponsorship from the FreeBSD Foundation.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  *
   29  */
   30 
   31 #include <machine/asm.h>
   32 __FBSDID("$FreeBSD: releng/11.2/sys/arm64/arm64/copyinout.S 319202 2017-05-30 12:26:36Z andrew $");
   33 
   34 #include <sys/errno.h>
   35 
   36 #include <machine/vmparam.h>
   37 
   38 #include "assym.s"
   39 
   40 /*
   41  * Fault handler for the copy{in,out} functions below.
   42  */
   43 ENTRY(copyio_fault)
   44         SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */
   45         EXIT_USER_ACCESS_CHECK(w0, x1)
   46 copyio_fault_nopcb:
   47         mov     x0, #EFAULT
   48         ret
   49 END(copyio_fault)
   50 
   51 /*
   52  * Copies from a kernel to user address
   53  *
   54  * int copyout(const void *kaddr, void *udaddr, size_t len)
   55  */
   56 ENTRY(copyout)
   57         cbz     x2, 1f
   58         adds    x3, x1, x2
   59         b.cs    copyio_fault_nopcb
   60         ldr     x4, =VM_MAXUSER_ADDRESS
   61         cmp     x3, x4
   62         b.hi    copyio_fault_nopcb
   63 
   64         b       copycommon
   65 
   66 1:      mov     x0, xzr         /* return 0 */
   67         ret
   68 
   69 END(copyout)
   70 
   71 /*
   72  * Copies from a user to kernel address
   73  *
   74  * int copyin(const void *uaddr, void *kdaddr, size_t len)
   75  */
   76 ENTRY(copyin)
   77         cbz     x2, 1f
   78         adds    x3, x0, x2
   79         b.cs    copyio_fault_nopcb
   80         ldr     x4, =VM_MAXUSER_ADDRESS
   81         cmp     x3, x4
   82         b.hi    copyio_fault_nopcb
   83 
   84         b       copycommon
   85 
   86 1:      mov     x0, xzr         /* return 0 */
   87         ret
   88 
   89 END(copyin)
   90 
   91 /*
   92  * Copies a string from a user to kernel address
   93  *
   94  * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
   95  */
   96 ENTRY(copyinstr)
   97         mov     x5, xzr         /* count = 0 */
   98         mov     w4, #1          /* If zero return faulure */
   99         cbz     x2, 3f          /* If len == 0 then skip loop */
  100 
  101         adr     x6, copyio_fault /* Get the handler address */
  102         SET_FAULT_HANDLER(x6, x7) /* Set the handler */
  103         ENTER_USER_ACCESS(w6, x7)
  104 
  105         ldr     x7, =VM_MAXUSER_ADDRESS
  106 1:      cmp     x0, x7
  107         b.cs    copyio_fault
  108         ldtrb   w4, [x0]        /* Load from uaddr */
  109         add     x0, x0, #1      /* Next char */
  110         strb    w4, [x1], #1    /* Store in kaddr */
  111         add     x5, x5, #1      /* count++ */
  112         cbz     w4, 2f          /* Break when NUL-terminated */
  113         sub     x2, x2, #1      /* len-- */
  114         cbnz    x2, 1b
  115 
  116 2:      EXIT_USER_ACCESS(w6)
  117         SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */
  118 
  119 
  120 3:      cbz     x3, 4f          /* Check if done != NULL */
  121         str     x5, [x3]        /* done = count */
  122 
  123 4:      mov     w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */
  124         cmp     w4, #0          /* Check if we saved the NUL-terminator */
  125         csel    w0, wzr, w1, eq /* If so return success, else failure */
  126         ret
  127 END(copyinstr)
  128 
  129 /*
  130  * Local helper
  131  *
  132  * x0 - src pointer
  133  * x1 - dst pointer
  134  * x2 - size
  135  * lr - the return address, so jump here instead of calling
  136  *
  137  * This function is optimized to minimize concurrent memory accesses. In
  138  * present form it is suited for cores with a single memory prefetching
  139  * unit.
  140  * ARM64TODO: 
  141  *   Consider using separate functions for each ARM64 core. Adding memory
  142  *   access interleaving might increase a total throughput on A57 or A72.
  143  */
  144         .text
  145         .align  4
  146         .local  copycommon
  147         .type   copycommon,@function
  148 
  149 copycommon:
  150         adr     x6, copyio_fault /* Get the handler address */
  151         SET_FAULT_HANDLER(x6, x7) /* Set the handler */
  152         ENTER_USER_ACCESS(w6, x7)
  153 
  154         /* Check alignment */
  155         orr     x3, x0, x1
  156         ands    x3, x3, 0x07
  157         b.eq    aligned
  158 
  159         /* Unaligned is byte by byte copy */
  160 byte_by_byte:
  161         ldrb    w3, [x0], #0x01
  162         strb    w3, [x1], #0x01
  163         subs    x2, x2, #0x01
  164         b.ne    byte_by_byte
  165         b       ending
  166 
  167 aligned:
  168         cmp     x2, #0x10
  169         b.lt    lead_out
  170         cmp     x2, #0x40
  171         b.lt    by_dwords_start
  172 
  173         /* Block copy */
  174         lsr     x15, x2, #0x06
  175 by_blocks:
  176         ldp     x3, x4, [x0], #0x10
  177         ldp     x5, x6, [x0], #0x10
  178         ldp     x7, x8, [x0], #0x10
  179         ldp     x9, x10, [x0], #0x10
  180         stp     x3, x4, [x1], #0x10
  181         stp     x5, x6, [x1], #0x10
  182         stp     x7, x8, [x1], #0x10
  183         stp     x9, x10, [x1], #0x10
  184 
  185         subs    x15, x15, #0x01
  186         b.ne    by_blocks
  187 
  188         and     x2, x2, #0x3f
  189 
  190 by_dwords_start:
  191         lsr     x15, x2, #0x04
  192         cbz     x15, lead_out
  193 by_dwords:
  194         ldp     x3, x4, [x0], #0x10
  195         stp     x3, x4, [x1], #0x10
  196         subs    x15, x15, #0x01
  197         b.ne    by_dwords
  198 
  199         /* Less than 16 bytes to copy */
  200 lead_out:
  201         tbz     x2, #0x03, last_word
  202         ldr     x3, [x0], #0x08
  203         str     x3, [x1], #0x08
  204 
  205 last_word:
  206         tbz     x2, #0x02, last_hword
  207         ldr     w3, [x0], #0x04
  208         str     w3, [x1], #0x04
  209 
  210 last_hword:
  211         tbz     x2, #0x01, last_byte
  212         ldrh    w3, [x0], #0x02
  213         strh    w3, [x1], #0x02
  214 
  215 last_byte:
  216         tbz     x2, #0x00, ending
  217         ldrb    w3, [x0]
  218         strb    w3, [x1]
  219 
  220 ending:
  221         EXIT_USER_ACCESS_CHECK(w6, x7)
  222         SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */
  223 
  224         mov     x0, xzr         /* return 0 */
  225         ret
  226         .size   copycommon, . - copycommon

Cache object: ceb1e85142acb0bb47c33fcc009b4ed7

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.

FreeBSD/Linux Kernel Cross Reference sys/arm64/arm64/copyinout.S

FreeBSD/Linux Kernel Cross Reference
sys/arm64/arm64/copyinout.S