The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_sharedpage.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2010, 2012 Konstantin Belousov <kib@FreeBSD.org>
    3  * Copyright (c) 2015 The FreeBSD Foundation
    4  * All rights reserved.
    5  *
    6  * Portions of this software were developed by Konstantin Belousov
    7  * under sponsorship from the FreeBSD Foundation.
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  */
   30 
   31 #include <sys/cdefs.h>
   32 __FBSDID("$FreeBSD: releng/11.2/sys/kern/kern_sharedpage.c 291171 2015-11-23 07:09:35Z kib $");
   33 
   34 #include "opt_compat.h"
   35 #include "opt_vm.h"
   36 
   37 #include <sys/param.h>
   38 #include <sys/systm.h>
   39 #include <sys/kernel.h>
   40 #include <sys/lock.h>
   41 #include <sys/malloc.h>
   42 #include <sys/rwlock.h>
   43 #include <sys/sysent.h>
   44 #include <sys/sysctl.h>
   45 #include <sys/vdso.h>
   46 
   47 #include <vm/vm.h>
   48 #include <vm/vm_param.h>
   49 #include <vm/pmap.h>
   50 #include <vm/vm_extern.h>
   51 #include <vm/vm_kern.h>
   52 #include <vm/vm_map.h>
   53 #include <vm/vm_object.h>
   54 #include <vm/vm_page.h>
   55 #include <vm/vm_pager.h>
   56 
   57 static struct sx shared_page_alloc_sx;
   58 static vm_object_t shared_page_obj;
   59 static int shared_page_free;
   60 char *shared_page_mapping;
   61 
   62 void
   63 shared_page_write(int base, int size, const void *data)
   64 {
   65 
   66         bcopy(data, shared_page_mapping + base, size);
   67 }
   68 
   69 static int
   70 shared_page_alloc_locked(int size, int align)
   71 {
   72         int res;
   73 
   74         res = roundup(shared_page_free, align);
   75         if (res + size >= IDX_TO_OFF(shared_page_obj->size))
   76                 res = -1;
   77         else
   78                 shared_page_free = res + size;
   79         return (res);
   80 }
   81 
   82 int
   83 shared_page_alloc(int size, int align)
   84 {
   85         int res;
   86 
   87         sx_xlock(&shared_page_alloc_sx);
   88         res = shared_page_alloc_locked(size, align);
   89         sx_xunlock(&shared_page_alloc_sx);
   90         return (res);
   91 }
   92 
   93 int
   94 shared_page_fill(int size, int align, const void *data)
   95 {
   96         int res;
   97 
   98         sx_xlock(&shared_page_alloc_sx);
   99         res = shared_page_alloc_locked(size, align);
  100         if (res != -1)
  101                 shared_page_write(res, size, data);
  102         sx_xunlock(&shared_page_alloc_sx);
  103         return (res);
  104 }
  105 
  106 static void
  107 shared_page_init(void *dummy __unused)
  108 {
  109         vm_page_t m;
  110         vm_offset_t addr;
  111 
  112         sx_init(&shared_page_alloc_sx, "shpsx");
  113         shared_page_obj = vm_pager_allocate(OBJT_PHYS, 0, PAGE_SIZE,
  114             VM_PROT_DEFAULT, 0, NULL);
  115         VM_OBJECT_WLOCK(shared_page_obj);
  116         m = vm_page_grab(shared_page_obj, 0, VM_ALLOC_NOBUSY | VM_ALLOC_ZERO);
  117         m->valid = VM_PAGE_BITS_ALL;
  118         VM_OBJECT_WUNLOCK(shared_page_obj);
  119         addr = kva_alloc(PAGE_SIZE);
  120         pmap_qenter(addr, &m, 1);
  121         shared_page_mapping = (char *)addr;
  122 }
  123 
  124 SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init,
  125     NULL);
  126 
  127 /*
  128  * Push the timehands update to the shared page.
  129  *
  130  * The lockless update scheme is similar to the one used to update the
  131  * in-kernel timehands, see sys/kern/kern_tc.c:tc_windup() (which
  132  * calls us after the timehands are updated).
  133  */
  134 static void
  135 timehands_update(struct vdso_sv_tk *svtk)
  136 {
  137         struct vdso_timehands th;
  138         struct vdso_timekeep *tk;
  139         uint32_t enabled, idx;
  140 
  141         enabled = tc_fill_vdso_timehands(&th);
  142         th.th_gen = 0;
  143         idx = svtk->sv_timekeep_curr;
  144         if (++idx >= VDSO_TH_NUM)
  145                 idx = 0;
  146         svtk->sv_timekeep_curr = idx;
  147         if (++svtk->sv_timekeep_gen == 0)
  148                 svtk->sv_timekeep_gen = 1;
  149 
  150         tk = (struct vdso_timekeep *)(shared_page_mapping +
  151             svtk->sv_timekeep_off);
  152         tk->tk_th[idx].th_gen = 0;
  153         atomic_thread_fence_rel();
  154         if (enabled)
  155                 tk->tk_th[idx] = th;
  156         atomic_store_rel_32(&tk->tk_th[idx].th_gen, svtk->sv_timekeep_gen);
  157         atomic_store_rel_32(&tk->tk_current, idx);
  158 
  159         /*
  160          * The ordering of the assignment to tk_enabled relative to
  161          * the update of the vdso_timehands is not important.
  162          */
  163         tk->tk_enabled = enabled;
  164 }
  165 
  166 #ifdef COMPAT_FREEBSD32
  167 static void
  168 timehands_update32(struct vdso_sv_tk *svtk)
  169 {
  170         struct vdso_timehands32 th;
  171         struct vdso_timekeep32 *tk;
  172         uint32_t enabled, idx;
  173 
  174         enabled = tc_fill_vdso_timehands32(&th);
  175         th.th_gen = 0;
  176         idx = svtk->sv_timekeep_curr;
  177         if (++idx >= VDSO_TH_NUM)
  178                 idx = 0;
  179         svtk->sv_timekeep_curr = idx;
  180         if (++svtk->sv_timekeep_gen == 0)
  181                 svtk->sv_timekeep_gen = 1;
  182 
  183         tk = (struct vdso_timekeep32 *)(shared_page_mapping +
  184             svtk->sv_timekeep_off);
  185         tk->tk_th[idx].th_gen = 0;
  186         atomic_thread_fence_rel();
  187         if (enabled)
  188                 tk->tk_th[idx] = th;
  189         atomic_store_rel_32(&tk->tk_th[idx].th_gen, svtk->sv_timekeep_gen);
  190         atomic_store_rel_32(&tk->tk_current, idx);
  191         tk->tk_enabled = enabled;
  192 }
  193 #endif
  194 
  195 /*
  196  * This is hackish, but easiest way to avoid creating list structures
  197  * that needs to be iterated over from the hardclock interrupt
  198  * context.
  199  */
  200 static struct vdso_sv_tk *host_svtk;
  201 #ifdef COMPAT_FREEBSD32
  202 static struct vdso_sv_tk *compat32_svtk;
  203 #endif
  204 
  205 void
  206 timekeep_push_vdso(void)
  207 {
  208 
  209         if (host_svtk != NULL)
  210                 timehands_update(host_svtk);
  211 #ifdef COMPAT_FREEBSD32
  212         if (compat32_svtk != NULL)
  213                 timehands_update32(compat32_svtk);
  214 #endif
  215 }
  216 
  217 struct vdso_sv_tk *
  218 alloc_sv_tk(void)
  219 {
  220         struct vdso_sv_tk *svtk;
  221         int tk_base;
  222         uint32_t tk_ver;
  223 
  224         tk_ver = VDSO_TK_VER_CURR;
  225         svtk = malloc(sizeof(struct vdso_sv_tk), M_TEMP, M_WAITOK | M_ZERO);
  226         tk_base = shared_page_alloc(sizeof(struct vdso_timekeep) +
  227             sizeof(struct vdso_timehands) * VDSO_TH_NUM, 16);
  228         KASSERT(tk_base != -1, ("tk_base -1 for native"));
  229         shared_page_write(tk_base + offsetof(struct vdso_timekeep, tk_ver),
  230             sizeof(uint32_t), &tk_ver);
  231         svtk->sv_timekeep_off = tk_base;
  232         timekeep_push_vdso();
  233         return (svtk);
  234 }
  235 
  236 #ifdef COMPAT_FREEBSD32
  237 struct vdso_sv_tk *
  238 alloc_sv_tk_compat32(void)
  239 {
  240         struct vdso_sv_tk *svtk;
  241         int tk_base;
  242         uint32_t tk_ver;
  243 
  244         svtk = malloc(sizeof(struct vdso_sv_tk), M_TEMP, M_WAITOK | M_ZERO);
  245         tk_ver = VDSO_TK_VER_CURR;
  246         tk_base = shared_page_alloc(sizeof(struct vdso_timekeep32) +
  247             sizeof(struct vdso_timehands32) * VDSO_TH_NUM, 16);
  248         KASSERT(tk_base != -1, ("tk_base -1 for 32bit"));
  249         shared_page_write(tk_base + offsetof(struct vdso_timekeep32,
  250             tk_ver), sizeof(uint32_t), &tk_ver);
  251         svtk->sv_timekeep_off = tk_base;
  252         timekeep_push_vdso();
  253         return (svtk);
  254 }
  255 #endif
  256 
  257 void
  258 exec_sysvec_init(void *param)
  259 {
  260         struct sysentvec *sv;
  261 
  262         sv = (struct sysentvec *)param;
  263         if ((sv->sv_flags & SV_SHP) == 0)
  264                 return;
  265         sv->sv_shared_page_obj = shared_page_obj;
  266         sv->sv_sigcode_base = sv->sv_shared_page_base +
  267             shared_page_fill(*(sv->sv_szsigcode), 16, sv->sv_sigcode);
  268         if ((sv->sv_flags & SV_ABI_MASK) != SV_ABI_FREEBSD)
  269                 return;
  270         if ((sv->sv_flags & SV_TIMEKEEP) != 0) {
  271 #ifdef COMPAT_FREEBSD32
  272                 if ((sv->sv_flags & SV_ILP32) != 0) {
  273                         KASSERT(compat32_svtk == NULL,
  274                             ("Compat32 already registered"));
  275                         compat32_svtk = alloc_sv_tk_compat32();
  276                         sv->sv_timekeep_base = sv->sv_shared_page_base +
  277                             compat32_svtk->sv_timekeep_off;
  278                 } else {
  279 #endif
  280                         KASSERT(host_svtk == NULL, ("Host already registered"));
  281                         host_svtk = alloc_sv_tk();
  282                         sv->sv_timekeep_base = sv->sv_shared_page_base +
  283                             host_svtk->sv_timekeep_off;
  284 #ifdef COMPAT_FREEBSD32
  285                 }
  286 #endif
  287         }
  288 }

Cache object: 05e0777ee1f53e6bcfd887f8fcb89cc5


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.