swtch.S

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2014 Andrew Turner
    3  * Copyright (c) 2014 The FreeBSD Foundation
    4  * All rights reserved.
    5  *
    6  * This software was developed by Andrew Turner under sponsorship from
    7  * the FreeBSD Foundation.
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  */
   31 
   32 #include "assym.s"
   33 #include "opt_kstack_pages.h"
   34 #include "opt_sched.h"
   35 
   36 #include <machine/asm.h>
   37 
   38 __FBSDID("$FreeBSD: releng/11.2/sys/arm64/arm64/swtch.S 317147 2017-04-19 15:59:16Z andrew $");
   39 
   40 .macro clear_step_flag pcbflags, tmp
   41         tbz     \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
   42         mrs     \tmp, mdscr_el1
   43         bic     \tmp, \tmp, #1
   44         msr     mdscr_el1, \tmp
   45         isb
   46 999:
   47 .endm
   48 
   49 .macro set_step_flag pcbflags, tmp
   50         tbz     \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
   51         mrs     \tmp, mdscr_el1
   52         orr     \tmp, \tmp, #1
   53         msr     mdscr_el1, \tmp
   54         isb
   55 999:
   56 .endm
   57 
   58 /*
   59  * void cpu_throw(struct thread *old, struct thread *new)
   60  */
   61 ENTRY(cpu_throw)
   62         /* Of old == NULL skip disabling stepping */
   63         cbz     x0, 1f
   64 
   65         /* If we were single stepping, disable it */
   66         ldr     x4, [x0, #TD_PCB]
   67         ldr     w5, [x4, #PCB_FLAGS]
   68         clear_step_flag w5, x6
   69 1:
   70 
   71 #ifdef VFP
   72         /* Backup the new thread pointer around a call to C code */
   73         mov     x19, x1
   74         bl      vfp_discard
   75         mov     x1, x19
   76 #endif
   77 
   78         /* Store the new curthread */
   79         str     x1, [x18, #PC_CURTHREAD]
   80         /* And the new pcb */
   81         ldr     x4, [x1, #TD_PCB]
   82         str     x4, [x18, #PC_CURPCB]
   83 
   84         /*
   85          * TODO: We may need to flush the cache here.
   86          */
   87 
   88         /* Switch to the new pmap */
   89         ldr     x5, [x4, #PCB_L0ADDR]
   90         msr     ttbr0_el1, x5
   91         isb
   92 
   93         /* Invalidate the TLB */
   94         dsb     sy
   95         tlbi    vmalle1is
   96         dsb     sy
   97         isb
   98 
   99         /* If we are single stepping, enable it */
  100         ldr     w5, [x4, #PCB_FLAGS]
  101         set_step_flag w5, x6
  102 
  103         /* Restore the registers */
  104         ldp     x5, x6, [x4, #PCB_SP]
  105         mov     sp, x5
  106         msr     tpidr_el0, x6
  107         ldp     x8, x9, [x4, #PCB_REGS + 8 * 8]
  108         ldp     x10, x11, [x4, #PCB_REGS + 10 * 8]
  109         ldp     x12, x13, [x4, #PCB_REGS + 12 * 8]
  110         ldp     x14, x15, [x4, #PCB_REGS + 14 * 8]
  111         ldp     x16, x17, [x4, #PCB_REGS + 16 * 8]
  112         ldr          x19, [x4, #PCB_REGS + 19 * 8]
  113         ldp     x20, x21, [x4, #PCB_REGS + 20 * 8]
  114         ldp     x22, x23, [x4, #PCB_REGS + 22 * 8]
  115         ldp     x24, x25, [x4, #PCB_REGS + 24 * 8]
  116         ldp     x26, x27, [x4, #PCB_REGS + 26 * 8]
  117         ldp     x28, x29, [x4, #PCB_REGS + 28 * 8]
  118         ldr     x30, [x4, #PCB_REGS + 30 * 8]
  119 
  120         ret
  121 END(cpu_throw)
  122 
  123 /*
  124  * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
  125  *
  126  * x0 = old
  127  * x1 = new
  128  * x2 = mtx
  129  * x3 to x7, x16 and x17 are caller saved
  130  */
  131 ENTRY(cpu_switch)
  132         /*
  133          * Save the old context.
  134          */
  135         ldr     x4, [x0, #TD_PCB]
  136 
  137         /* Store the callee-saved registers */
  138         stp     x8, x9, [x4, #PCB_REGS + 8 * 8]
  139         stp     x10, x11, [x4, #PCB_REGS + 10 * 8]
  140         stp     x12, x13, [x4, #PCB_REGS + 12 * 8]
  141         stp     x14, x15, [x4, #PCB_REGS + 14 * 8]
  142         stp     x16, x17, [x4, #PCB_REGS + 16 * 8]
  143         stp     x18, x19, [x4, #PCB_REGS + 18 * 8]
  144         stp     x20, x21, [x4, #PCB_REGS + 20 * 8]
  145         stp     x22, x23, [x4, #PCB_REGS + 22 * 8]
  146         stp     x24, x25, [x4, #PCB_REGS + 24 * 8]
  147         stp     x26, x27, [x4, #PCB_REGS + 26 * 8]
  148         stp     x28, x29, [x4, #PCB_REGS + 28 * 8]
  149         str     x30, [x4, #PCB_REGS + 30 * 8]
  150         /* And the old stack pointer */
  151         mov     x5, sp
  152         mrs     x6, tpidr_el0
  153         stp     x5, x6, [x4, #PCB_SP]
  154 
  155         /* If we were single stepping, disable it */
  156         ldr     w5, [x4, #PCB_FLAGS]
  157         clear_step_flag w5, x6
  158 
  159 #ifdef VFP
  160         mov     x19, x0
  161         mov     x20, x1
  162         mov     x21, x2
  163         /* Load the pcb address */
  164         mov     x1, x4
  165         bl      vfp_save_state
  166         mov     x2, x21
  167         mov     x1, x20
  168         mov     x0, x19
  169 #endif
  170 
  171         /* Store the new curthread */
  172         str     x1, [x18, #PC_CURTHREAD]
  173 
  174         /*
  175          * Restore the saved context and set it as curpcb.
  176          */
  177         ldr     x4, [x1, #TD_PCB]
  178         str     x4, [x18, #PC_CURPCB]
  179 
  180         /*
  181          * TODO: We may need to flush the cache here if switching
  182          * to a user process.
  183          */
  184 
  185         /* Switch to the new pmap */
  186         ldr     x5, [x4, #PCB_L0ADDR]
  187         msr     ttbr0_el1, x5
  188         isb
  189 
  190         /* Invalidate the TLB */
  191         dsb     sy
  192         tlbi    vmalle1is
  193         dsb     sy
  194         isb
  195 
  196         /*
  197          * Release the old thread. This doesn't need to be a store-release
  198          * as the above dsb instruction will provide release semantics.
  199          */
  200         str     x2, [x0, #TD_LOCK]
  201 #if defined(SCHED_ULE) && defined(SMP)
  202         /* Spin if TD_LOCK points to a blocked_lock */
  203         ldr     x2, =_C_LABEL(blocked_lock)
  204 1:
  205         ldar    x3, [x1, #TD_LOCK]
  206         cmp     x3, x2
  207         b.eq    1b
  208 #endif
  209 
  210         /* If we are single stepping, enable it */
  211         ldr     w5, [x4, #PCB_FLAGS]
  212         set_step_flag w5, x6
  213 
  214         /* Restore the registers */
  215         ldp     x5, x6, [x4, #PCB_SP]
  216         mov     sp, x5
  217         msr     tpidr_el0, x6
  218         ldp     x8, x9, [x4, #PCB_REGS + 8 * 8]
  219         ldp     x10, x11, [x4, #PCB_REGS + 10 * 8]
  220         ldp     x12, x13, [x4, #PCB_REGS + 12 * 8]
  221         ldp     x14, x15, [x4, #PCB_REGS + 14 * 8]
  222         ldp     x16, x17, [x4, #PCB_REGS + 16 * 8]
  223         ldr          x19, [x4, #PCB_REGS + 19 * 8]
  224         ldp     x20, x21, [x4, #PCB_REGS + 20 * 8]
  225         ldp     x22, x23, [x4, #PCB_REGS + 22 * 8]
  226         ldp     x24, x25, [x4, #PCB_REGS + 24 * 8]
  227         ldp     x26, x27, [x4, #PCB_REGS + 26 * 8]
  228         ldp     x28, x29, [x4, #PCB_REGS + 28 * 8]
  229         ldr     x30, [x4, #PCB_REGS + 30 * 8]
  230 
  231         str     xzr, [x4, #PCB_REGS + 18 * 8]
  232         ret
  233 .Lcpu_switch_panic_str:
  234         .asciz "cpu_switch: %p\0"
  235 END(cpu_switch)
  236 
  237 ENTRY(fork_trampoline)
  238         mov     x0, x8
  239         mov     x1, x9
  240         mov     x2, sp
  241         mov     fp, #0  /* Stack traceback stops here. */
  242         bl      _C_LABEL(fork_exit)
  243 
  244         /* Restore the registers other than x0 and x1 */
  245         ldp     x2, x3, [sp, #TF_X + 2 * 8]
  246         ldp     x4, x5, [sp, #TF_X + 4 * 8]
  247         ldp     x6, x7, [sp, #TF_X + 6 * 8]
  248         ldp     x8, x9, [sp, #TF_X + 8 * 8]
  249         ldp     x10, x11, [sp, #TF_X + 10 * 8]
  250         ldp     x12, x13, [sp, #TF_X + 12 * 8]
  251         ldp     x14, x15, [sp, #TF_X + 14 * 8]
  252         ldp     x16, x17, [sp, #TF_X + 16 * 8]
  253         ldr          x19, [sp, #TF_X + 19 * 8]
  254         ldp     x20, x21, [sp, #TF_X + 20 * 8]
  255         ldp     x22, x23, [sp, #TF_X + 22 * 8]
  256         ldp     x24, x25, [sp, #TF_X + 24 * 8]
  257         ldp     x26, x27, [sp, #TF_X + 26 * 8]
  258         ldp     x28, x29, [sp, #TF_X + 28 * 8]
  259 
  260         /*
  261          * Disable interrupts to avoid
  262          * overwriting spsr_el1 and sp_el0 by an IRQ exception.
  263          */
  264         msr     daifset, #2
  265 
  266         /* Restore sp and lr */
  267         ldp     x0, x1, [sp]
  268         msr     sp_el0, x0
  269         mov     lr, x1
  270 
  271         /* Restore elr and spsr */
  272         ldp     x0, x1, [sp, #16]
  273         msr     elr_el1, x0
  274         msr     spsr_el1, x1
  275 
  276         /* Finally x0 and x1 */
  277         ldp     x0, x1, [sp, #TF_X + 0 * 8]
  278         ldr     x18, [sp, #TF_X + 18 * 8]
  279 
  280         /*
  281          * No need for interrupts reenabling since PSR
  282          * will be set to the desired value anyway.
  283          */
  284         eret
  285         
  286 END(fork_trampoline)
  287 
  288 ENTRY(savectx)
  289         /* Store the callee-saved registers */
  290         stp     x8,  x9,  [x0, #PCB_REGS + 8 * 8]
  291         stp     x10, x11, [x0, #PCB_REGS + 10 * 8]
  292         stp     x12, x13, [x0, #PCB_REGS + 12 * 8]
  293         stp     x14, x15, [x0, #PCB_REGS + 14 * 8]
  294         stp     x16, x17, [x0, #PCB_REGS + 16 * 8]
  295         stp     x18, x19, [x0, #PCB_REGS + 18 * 8]
  296         stp     x20, x21, [x0, #PCB_REGS + 20 * 8]
  297         stp     x22, x23, [x0, #PCB_REGS + 22 * 8]
  298         stp     x24, x25, [x0, #PCB_REGS + 24 * 8]
  299         stp     x26, x27, [x0, #PCB_REGS + 26 * 8]
  300         stp     x28, x29, [x0, #PCB_REGS + 28 * 8]
  301         str     x30, [x0, #PCB_REGS + 30 * 8]
  302         /* And the old stack pointer */
  303         mov     x5, sp
  304         mrs     x6, tpidr_el0
  305         stp     x5, x6, [x0, #PCB_SP]
  306 
  307         /* Store the VFP registers */
  308 #ifdef VFP
  309         mov     x28, lr
  310         mov     x1, x0                  /* move pcb to the correct register */
  311         mov     x0, xzr                 /* td = NULL */
  312         bl      vfp_save_state
  313         mov     lr, x28
  314 #endif
  315 
  316         ret
  317 END(savectx)
  318
Cache object: 6139318613a9d17bb1803ccc312c469d
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/arm64/arm64/swtch.S

FreeBSD/Linux Kernel Cross Reference
sys/arm64/arm64/swtch.S
