The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_kmem.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: subr_kmem.c,v 1.87 2022/05/30 23:36:26 mrg Exp $       */
    2 
    3 /*
    4  * Copyright (c) 2009-2020 The NetBSD Foundation, Inc.
    5  * All rights reserved.
    6  *
    7  * This code is derived from software contributed to The NetBSD Foundation
    8  * by Andrew Doran and Maxime Villard.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  *
   19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
   20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
   21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
   23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   29  * POSSIBILITY OF SUCH DAMAGE.
   30  */
   31 
   32 /*
   33  * Copyright (c)2006 YAMAMOTO Takashi,
   34  * All rights reserved.
   35  *
   36  * Redistribution and use in source and binary forms, with or without
   37  * modification, are permitted provided that the following conditions
   38  * are met:
   39  * 1. Redistributions of source code must retain the above copyright
   40  *    notice, this list of conditions and the following disclaimer.
   41  * 2. Redistributions in binary form must reproduce the above copyright
   42  *    notice, this list of conditions and the following disclaimer in the
   43  *    documentation and/or other materials provided with the distribution.
   44  *
   45  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   46  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   47  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   48  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   49  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   50  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   51  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   52  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   53  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   54  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   55  * SUCH DAMAGE.
   56  */
   57 
   58 /*
   59  * Allocator of kernel wired memory. This allocator has some debug features
   60  * enabled with "option DIAGNOSTIC" and "option DEBUG".
   61  */
   62 
   63 /*
   64  * KMEM_SIZE: detect alloc/free size mismatch bugs.
   65  *      Append to each allocation a fixed-sized footer and record the exact
   66  *      user-requested allocation size in it.  When freeing, compare it with
   67  *      kmem_free's "size" argument.
   68  *
   69  * This option is enabled on DIAGNOSTIC.
   70  *
   71  *  |CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK| |
   72  *  +-----+-----+-----+-----+-----+-----+-----+-----+-----+-+
   73  *  |     |     |     |     |     |     |     |     |/////|U|
   74  *  |     |     |     |     |     |     |     |     |/HSZ/|U|
   75  *  |     |     |     |     |     |     |     |     |/////|U|
   76  *  +-----+-----+-----+-----+-----+-----+-----+-----+-----+-+
   77  *  | Buffer usable by the caller (requested size)  |Size |Unused
   78  */
   79 
   80 #include <sys/cdefs.h>
   81 __KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.87 2022/05/30 23:36:26 mrg Exp $");
   82 
   83 #ifdef _KERNEL_OPT
   84 #include "opt_kmem.h"
   85 #endif
   86 
   87 #include <sys/param.h>
   88 #include <sys/callback.h>
   89 #include <sys/kmem.h>
   90 #include <sys/pool.h>
   91 #include <sys/debug.h>
   92 #include <sys/lockdebug.h>
   93 #include <sys/cpu.h>
   94 #include <sys/asan.h>
   95 #include <sys/msan.h>
   96 #include <sys/sdt.h>
   97 
   98 #include <uvm/uvm_extern.h>
   99 #include <uvm/uvm_map.h>
  100 
  101 #include <lib/libkern/libkern.h>
  102 
  103 struct kmem_cache_info {
  104         size_t          kc_size;
  105         const char *    kc_name;
  106 #ifdef KDTRACE_HOOKS
  107         const id_t      *kc_alloc_probe_id;
  108         const id_t      *kc_free_probe_id;
  109 #endif
  110 };
  111 
  112 #define KMEM_CACHE_SIZES(F)                                                   \
  113         F(8, kmem-00008, kmem__00008)                                         \
  114         F(16, kmem-00016, kmem__00016)                                        \
  115         F(24, kmem-00024, kmem__00024)                                        \
  116         F(32, kmem-00032, kmem__00032)                                        \
  117         F(40, kmem-00040, kmem__00040)                                        \
  118         F(48, kmem-00048, kmem__00048)                                        \
  119         F(56, kmem-00056, kmem__00056)                                        \
  120         F(64, kmem-00064, kmem__00064)                                        \
  121         F(80, kmem-00080, kmem__00080)                                        \
  122         F(96, kmem-00096, kmem__00096)                                        \
  123         F(112, kmem-00112, kmem__00112)                                       \
  124         F(128, kmem-00128, kmem__00128)                                       \
  125         F(160, kmem-00160, kmem__00160)                                       \
  126         F(192, kmem-00192, kmem__00192)                                       \
  127         F(224, kmem-00224, kmem__00224)                                       \
  128         F(256, kmem-00256, kmem__00256)                                       \
  129         F(320, kmem-00320, kmem__00320)                                       \
  130         F(384, kmem-00384, kmem__00384)                                       \
  131         F(448, kmem-00448, kmem__00448)                                       \
  132         F(512, kmem-00512, kmem__00512)                                       \
  133         F(768, kmem-00768, kmem__00768)                                       \
  134         F(1024, kmem-01024, kmem__01024)                                      \
  135         /* end of KMEM_CACHE_SIZES */
  136 
  137 #define KMEM_CACHE_BIG_SIZES(F)                                               \
  138         F(2048, kmem-02048, kmem__02048)                                      \
  139         F(4096, kmem-04096, kmem__04096)                                      \
  140         F(8192, kmem-08192, kmem__08192)                                      \
  141         F(16384, kmem-16384, kmem__16384)                                     \
  142         /* end of KMEM_CACHE_BIG_SIZES */
  143 
  144 /* sdt:kmem:alloc:kmem-* probes */
  145 #define F(SZ, NAME, PROBENAME)                                                \
  146         SDT_PROBE_DEFINE4(sdt, kmem, alloc, PROBENAME,                        \
  147             "void *"/*ptr*/,                                                  \
  148             "size_t"/*requested_size*/,                                       \
  149             "size_t"/*allocated_size*/,                                       \
  150             "km_flag_t"/*kmflags*/);
  151 KMEM_CACHE_SIZES(F);
  152 KMEM_CACHE_BIG_SIZES(F);
  153 #undef  F
  154 
  155 /* sdt:kmem:free:kmem-* probes */
  156 #define F(SZ, NAME, PROBENAME)                                                \
  157         SDT_PROBE_DEFINE3(sdt, kmem, free, PROBENAME,                         \
  158             "void *"/*ptr*/,                                                  \
  159             "size_t"/*requested_size*/,                                       \
  160             "size_t"/*allocated_size*/);
  161 KMEM_CACHE_SIZES(F);
  162 KMEM_CACHE_BIG_SIZES(F);
  163 #undef  F
  164 
  165 /* sdt:kmem:alloc:large, sdt:kmem:free:large probes */
  166 SDT_PROBE_DEFINE4(sdt, kmem, alloc, large,
  167     "void *"/*ptr*/,
  168     "size_t"/*requested_size*/,
  169     "size_t"/*allocated_size*/,
  170     "km_flag_t"/*kmflags*/);
  171 SDT_PROBE_DEFINE3(sdt, kmem, free, large,
  172     "void *"/*ptr*/,
  173     "size_t"/*requested_size*/,
  174     "size_t"/*allocated_size*/);
  175 
  176 #ifdef KDTRACE_HOOKS
  177 #define F(SZ, NAME, PROBENAME)                                                \
  178         { SZ, #NAME,                                                          \
  179           &sdt_sdt_kmem_alloc_##PROBENAME->id,                                \
  180           &sdt_sdt_kmem_free_##PROBENAME->id },
  181 #else
  182 #define F(SZ, NAME, PROBENAME)  { SZ, #NAME },
  183 #endif
  184 
  185 static const struct kmem_cache_info kmem_cache_sizes[] = {
  186         KMEM_CACHE_SIZES(F)
  187         { 0 }
  188 };
  189 
  190 static const struct kmem_cache_info kmem_cache_big_sizes[] = {
  191         KMEM_CACHE_BIG_SIZES(F)
  192         { 0 }
  193 };
  194 
  195 #undef  F
  196 
  197 /*
  198  * KMEM_ALIGN is the smallest guaranteed alignment and also the
  199  * smallest allocateable quantum.
  200  * Every cache size >= CACHE_LINE_SIZE gets CACHE_LINE_SIZE alignment.
  201  */
  202 #define KMEM_ALIGN              8
  203 #define KMEM_SHIFT              3
  204 #define KMEM_MAXSIZE            1024
  205 #define KMEM_CACHE_COUNT        (KMEM_MAXSIZE >> KMEM_SHIFT)
  206 
  207 static pool_cache_t kmem_cache[KMEM_CACHE_COUNT] __cacheline_aligned;
  208 static size_t kmem_cache_maxidx __read_mostly;
  209 
  210 #define KMEM_BIG_ALIGN          2048
  211 #define KMEM_BIG_SHIFT          11
  212 #define KMEM_BIG_MAXSIZE        16384
  213 #define KMEM_CACHE_BIG_COUNT    (KMEM_BIG_MAXSIZE >> KMEM_BIG_SHIFT)
  214 
  215 static pool_cache_t kmem_cache_big[KMEM_CACHE_BIG_COUNT] __cacheline_aligned;
  216 static size_t kmem_cache_big_maxidx __read_mostly;
  217 
  218 #if defined(DIAGNOSTIC) && defined(_HARDKERNEL)
  219 #define KMEM_SIZE
  220 #endif
  221 
  222 #if defined(DEBUG) && defined(_HARDKERNEL)
  223 static void *kmem_freecheck;
  224 #endif
  225 
  226 #if defined(KMEM_SIZE)
  227 #define SIZE_SIZE       sizeof(size_t)
  228 static void kmem_size_set(void *, size_t);
  229 static void kmem_size_check(void *, size_t);
  230 #else
  231 #define SIZE_SIZE       0
  232 #define kmem_size_set(p, sz)    /* nothing */
  233 #define kmem_size_check(p, sz)  /* nothing */
  234 #endif
  235 
  236 #ifndef KDTRACE_HOOKS
  237 
  238 static const id_t **const kmem_cache_alloc_probe_id = NULL;
  239 static const id_t **const kmem_cache_big_alloc_probe_id = NULL;
  240 static const id_t **const kmem_cache_free_probe_id = NULL;
  241 static const id_t **const kmem_cache_big_free_probe_id = NULL;
  242 
  243 #define KMEM_CACHE_PROBE(ARRAY, INDEX, PTR, REQSIZE, ALLOCSIZE, FLAGS)        \
  244         __nothing
  245 
  246 #else
  247 
  248 static const id_t *kmem_cache_alloc_probe_id[KMEM_CACHE_COUNT];
  249 static const id_t *kmem_cache_big_alloc_probe_id[KMEM_CACHE_COUNT];
  250 static const id_t *kmem_cache_free_probe_id[KMEM_CACHE_COUNT];
  251 static const id_t *kmem_cache_big_free_probe_id[KMEM_CACHE_COUNT];
  252 
  253 #define KMEM_CACHE_PROBE(ARRAY, INDEX, PTR, REQSIZE, ALLOCSIZE, FLAGS) do     \
  254 {                                                                             \
  255         id_t id;                                                              \
  256                                                                               \
  257         KDASSERT((INDEX) < __arraycount(ARRAY));                              \
  258         if (__predict_false((id = *(ARRAY)[INDEX]) != 0)) {                   \
  259                 (*sdt_probe_func)(id,                                         \
  260                     (uintptr_t)(PTR),                                         \
  261                     (uintptr_t)(REQSIZE),                                     \
  262                     (uintptr_t)(ALLOCSIZE),                                   \
  263                     (uintptr_t)(FLAGS),                                       \
  264                     (uintptr_t)0);                                            \
  265         }                                                                     \
  266 } while (0)
  267 
  268 #endif  /* KDTRACE_HOOKS */
  269 
  270 #define KMEM_CACHE_ALLOC_PROBE(I, P, RS, AS, F)                               \
  271         KMEM_CACHE_PROBE(kmem_cache_alloc_probe_id, I, P, RS, AS, F)
  272 #define KMEM_CACHE_BIG_ALLOC_PROBE(I, P, RS, AS, F)                           \
  273         KMEM_CACHE_PROBE(kmem_cache_big_alloc_probe_id, I, P, RS, AS, F)
  274 #define KMEM_CACHE_FREE_PROBE(I, P, RS, AS)                                   \
  275         KMEM_CACHE_PROBE(kmem_cache_free_probe_id, I, P, RS, AS, 0)
  276 #define KMEM_CACHE_BIG_FREE_PROBE(I, P, RS, AS)                               \
  277         KMEM_CACHE_PROBE(kmem_cache_big_free_probe_id, I, P, RS, AS, 0)
  278 
  279 CTASSERT(KM_SLEEP == PR_WAITOK);
  280 CTASSERT(KM_NOSLEEP == PR_NOWAIT);
  281 
  282 /*
  283  * kmem_intr_alloc: allocate wired memory.
  284  */
  285 void *
  286 kmem_intr_alloc(size_t requested_size, km_flag_t kmflags)
  287 {
  288 #ifdef KASAN
  289         const size_t origsize = requested_size;
  290 #endif
  291         size_t allocsz, index;
  292         size_t size;
  293         pool_cache_t pc;
  294         uint8_t *p;
  295 
  296         KASSERT(requested_size > 0);
  297 
  298         KASSERT((kmflags & KM_SLEEP) || (kmflags & KM_NOSLEEP));
  299         KASSERT(!(kmflags & KM_SLEEP) || !(kmflags & KM_NOSLEEP));
  300 
  301         kasan_add_redzone(&requested_size);
  302         size = kmem_roundup_size(requested_size);
  303         allocsz = size + SIZE_SIZE;
  304 
  305         if ((index = ((allocsz - 1) >> KMEM_SHIFT))
  306             < kmem_cache_maxidx) {
  307                 pc = kmem_cache[index];
  308                 p = pool_cache_get(pc, kmflags);
  309                 KMEM_CACHE_ALLOC_PROBE(index,
  310                     p, requested_size, allocsz, kmflags);
  311         } else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT))
  312             < kmem_cache_big_maxidx) {
  313                 pc = kmem_cache_big[index];
  314                 p = pool_cache_get(pc, kmflags);
  315                 KMEM_CACHE_BIG_ALLOC_PROBE(index,
  316                     p, requested_size, allocsz, kmflags);
  317         } else {
  318                 int ret = uvm_km_kmem_alloc(kmem_va_arena,
  319                     (vsize_t)round_page(size),
  320                     ((kmflags & KM_SLEEP) ? VM_SLEEP : VM_NOSLEEP)
  321                      | VM_INSTANTFIT, (vmem_addr_t *)&p);
  322                 SDT_PROBE4(sdt, kmem, alloc, large,
  323                     ret ? NULL : p, requested_size, round_page(size), kmflags);
  324                 if (ret) {
  325                         return NULL;
  326                 }
  327                 FREECHECK_OUT(&kmem_freecheck, p);
  328                 return p;
  329         }
  330 
  331         if (__predict_true(p != NULL)) {
  332                 FREECHECK_OUT(&kmem_freecheck, p);
  333                 kmem_size_set(p, requested_size);
  334                 kasan_mark(p, origsize, size, KASAN_KMEM_REDZONE);
  335                 return p;
  336         }
  337         return p;
  338 }
  339 
  340 /*
  341  * kmem_intr_zalloc: allocate zeroed wired memory.
  342  */
  343 void *
  344 kmem_intr_zalloc(size_t size, km_flag_t kmflags)
  345 {
  346         void *p;
  347 
  348         p = kmem_intr_alloc(size, kmflags);
  349         if (p != NULL) {
  350                 memset(p, 0, size);
  351         }
  352         return p;
  353 }
  354 
  355 /*
  356  * kmem_intr_free: free wired memory allocated by kmem_alloc.
  357  */
  358 void
  359 kmem_intr_free(void *p, size_t requested_size)
  360 {
  361         size_t allocsz, index;
  362         size_t size;
  363         pool_cache_t pc;
  364 
  365         KASSERT(p != NULL);
  366         KASSERTMSG(requested_size > 0, "kmem_intr_free(%p, 0)", p);
  367 
  368         kasan_add_redzone(&requested_size);
  369         size = kmem_roundup_size(requested_size);
  370         allocsz = size + SIZE_SIZE;
  371 
  372         if ((index = ((allocsz - 1) >> KMEM_SHIFT))
  373             < kmem_cache_maxidx) {
  374                 KMEM_CACHE_FREE_PROBE(index, p, requested_size, allocsz);
  375                 pc = kmem_cache[index];
  376         } else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT))
  377             < kmem_cache_big_maxidx) {
  378                 KMEM_CACHE_BIG_FREE_PROBE(index, p, requested_size, allocsz);
  379                 pc = kmem_cache_big[index];
  380         } else {
  381                 FREECHECK_IN(&kmem_freecheck, p);
  382                 SDT_PROBE3(sdt, kmem, free, large,
  383                     p, requested_size, round_page(size));
  384                 uvm_km_kmem_free(kmem_va_arena, (vaddr_t)p,
  385                     round_page(size));
  386                 return;
  387         }
  388 
  389         kasan_mark(p, size, size, 0);
  390 
  391         kmem_size_check(p, requested_size);
  392         FREECHECK_IN(&kmem_freecheck, p);
  393         LOCKDEBUG_MEM_CHECK(p, size);
  394 
  395         pool_cache_put(pc, p);
  396 }
  397 
  398 /* -------------------------------- Kmem API -------------------------------- */
  399 
  400 /*
  401  * kmem_alloc: allocate wired memory.
  402  * => must not be called from interrupt context.
  403  */
  404 void *
  405 kmem_alloc(size_t size, km_flag_t kmflags)
  406 {
  407         void *v;
  408 
  409         KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()),
  410             "kmem(9) should not be used from the interrupt context");
  411         v = kmem_intr_alloc(size, kmflags);
  412         if (__predict_true(v != NULL)) {
  413                 kmsan_mark(v, size, KMSAN_STATE_UNINIT);
  414                 kmsan_orig(v, size, KMSAN_TYPE_KMEM, __RET_ADDR);
  415         }
  416         KASSERT(v || (kmflags & KM_NOSLEEP) != 0);
  417         return v;
  418 }
  419 
  420 /*
  421  * kmem_zalloc: allocate zeroed wired memory.
  422  * => must not be called from interrupt context.
  423  */
  424 void *
  425 kmem_zalloc(size_t size, km_flag_t kmflags)
  426 {
  427         void *v;
  428 
  429         KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()),
  430             "kmem(9) should not be used from the interrupt context");
  431         v = kmem_intr_zalloc(size, kmflags);
  432         KASSERT(v || (kmflags & KM_NOSLEEP) != 0);
  433         return v;
  434 }
  435 
  436 /*
  437  * kmem_free: free wired memory allocated by kmem_alloc.
  438  * => must not be called from interrupt context.
  439  */
  440 void
  441 kmem_free(void *p, size_t size)
  442 {
  443         KASSERT(!cpu_intr_p());
  444         KASSERT(!cpu_softintr_p());
  445         kmem_intr_free(p, size);
  446         kmsan_mark(p, size, KMSAN_STATE_INITED);
  447 }
  448 
  449 static size_t
  450 kmem_create_caches(const struct kmem_cache_info *array,
  451     const id_t *alloc_probe_table[], const id_t *free_probe_table[],
  452     pool_cache_t alloc_table[], size_t maxsize, int shift, int ipl)
  453 {
  454         size_t maxidx = 0;
  455         size_t table_unit = (1 << shift);
  456         size_t size = table_unit;
  457         int i;
  458 
  459         for (i = 0; array[i].kc_size != 0 ; i++) {
  460                 const char *name = array[i].kc_name;
  461                 size_t cache_size = array[i].kc_size;
  462                 struct pool_allocator *pa;
  463                 int flags = 0;
  464                 pool_cache_t pc;
  465                 size_t align;
  466 
  467                 /* check if we reached the requested size */
  468                 if (cache_size > maxsize || cache_size > PAGE_SIZE) {
  469                         break;
  470                 }
  471 
  472                 /*
  473                  * Exclude caches with size not a factor or multiple of the
  474                  * coherency unit.
  475                  */
  476                 if (cache_size < COHERENCY_UNIT) {
  477                         if (COHERENCY_UNIT % cache_size > 0) {
  478                                 continue;
  479                         }
  480                         flags |= PR_NOTOUCH;
  481                         align = KMEM_ALIGN;
  482                 } else if ((cache_size & (PAGE_SIZE - 1)) == 0) {
  483                         align = PAGE_SIZE;
  484                 } else {
  485                         if ((cache_size % COHERENCY_UNIT) > 0) {
  486                                 continue;
  487                         }
  488                         align = COHERENCY_UNIT;
  489                 }
  490 
  491                 if ((cache_size >> shift) > maxidx) {
  492                         maxidx = cache_size >> shift;
  493                 }
  494 
  495                 pa = &pool_allocator_kmem;
  496                 pc = pool_cache_init(cache_size, align, 0, flags,
  497                     name, pa, ipl, NULL, NULL, NULL);
  498 
  499                 while (size <= cache_size) {
  500                         alloc_table[(size - 1) >> shift] = pc;
  501 #ifdef KDTRACE_HOOKS
  502                         if (alloc_probe_table) {
  503                                 alloc_probe_table[(size - 1) >> shift] =
  504                                     array[i].kc_alloc_probe_id;
  505                         }
  506                         if (free_probe_table) {
  507                                 free_probe_table[(size - 1) >> shift] =
  508                                     array[i].kc_free_probe_id;
  509                         }
  510 #endif
  511                         size += table_unit;
  512                 }
  513         }
  514         return maxidx;
  515 }
  516 
  517 void
  518 kmem_init(void)
  519 {
  520         kmem_cache_maxidx = kmem_create_caches(kmem_cache_sizes,
  521             kmem_cache_alloc_probe_id, kmem_cache_free_probe_id,
  522             kmem_cache, KMEM_MAXSIZE, KMEM_SHIFT, IPL_VM);
  523         kmem_cache_big_maxidx = kmem_create_caches(kmem_cache_big_sizes,
  524             kmem_cache_big_alloc_probe_id, kmem_cache_big_free_probe_id,
  525             kmem_cache_big, PAGE_SIZE, KMEM_BIG_SHIFT, IPL_VM);
  526 }
  527 
  528 size_t
  529 kmem_roundup_size(size_t size)
  530 {
  531         return (size + (KMEM_ALIGN - 1)) & ~(KMEM_ALIGN - 1);
  532 }
  533 
  534 /*
  535  * Used to dynamically allocate string with kmem accordingly to format.
  536  */
  537 char *
  538 kmem_asprintf(const char *fmt, ...)
  539 {
  540         int size __diagused, len;
  541         va_list va;
  542         char *str;
  543 
  544         va_start(va, fmt);
  545         len = vsnprintf(NULL, 0, fmt, va);
  546         va_end(va);
  547 
  548         str = kmem_alloc(len + 1, KM_SLEEP);
  549 
  550         va_start(va, fmt);
  551         size = vsnprintf(str, len + 1, fmt, va);
  552         va_end(va);
  553 
  554         KASSERT(size == len);
  555 
  556         return str;
  557 }
  558 
  559 char *
  560 kmem_strdupsize(const char *str, size_t *lenp, km_flag_t flags)
  561 {
  562         size_t len = strlen(str) + 1;
  563         char *ptr = kmem_alloc(len, flags);
  564         if (ptr == NULL)
  565                 return NULL;
  566 
  567         if (lenp)
  568                 *lenp = len;
  569         memcpy(ptr, str, len);
  570         return ptr;
  571 }
  572 
  573 char *
  574 kmem_strndup(const char *str, size_t maxlen, km_flag_t flags)
  575 {
  576         KASSERT(str != NULL);
  577         KASSERT(maxlen != 0);
  578 
  579         size_t len = strnlen(str, maxlen);
  580         char *ptr = kmem_alloc(len + 1, flags);
  581         if (ptr == NULL)
  582                 return NULL;
  583 
  584         memcpy(ptr, str, len);
  585         ptr[len] = '\0';
  586 
  587         return ptr;
  588 }
  589 
  590 void
  591 kmem_strfree(char *str)
  592 {
  593         if (str == NULL)
  594                 return;
  595 
  596         kmem_free(str, strlen(str) + 1);
  597 }
  598 
  599 /*
  600  * Utility routine to maybe-allocate a temporary buffer if the size
  601  * is larger than we're willing to put on the stack.
  602  */
  603 void *
  604 kmem_tmpbuf_alloc(size_t size, void *stackbuf, size_t stackbufsize,
  605     km_flag_t flags)
  606 {
  607         if (size <= stackbufsize) {
  608                 return stackbuf;
  609         }
  610 
  611         return kmem_alloc(size, flags);
  612 }
  613 
  614 void
  615 kmem_tmpbuf_free(void *buf, size_t size, void *stackbuf)
  616 {
  617         if (buf != stackbuf) {
  618                 kmem_free(buf, size);
  619         }
  620 }
  621 
  622 /* --------------------------- DEBUG / DIAGNOSTIC --------------------------- */
  623 
  624 #if defined(KMEM_SIZE)
  625 static void
  626 kmem_size_set(void *p, size_t sz)
  627 {
  628         memcpy((char *)p + sz, &sz, sizeof(size_t));
  629 }
  630 
  631 static void
  632 kmem_size_check(void *p, size_t sz)
  633 {
  634         size_t hsz;
  635 
  636         memcpy(&hsz, (char *)p + sz, sizeof(size_t));
  637 
  638         if (hsz != sz) {
  639                 panic("kmem_free(%p, %zu) != allocated size %zu; overwrote?",
  640                     p, sz, hsz);
  641         }
  642 
  643         memset((char *)p + sz, 0xff, sizeof(size_t));
  644 }
  645 #endif /* defined(KMEM_SIZE) */

Cache object: 6397cf22bdbb46981814ac6beb9428e7


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.