refcount.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*
    2  * CDDL HEADER START
    3  *
    4  * The contents of this file are subject to the terms of the
    5  * Common Development and Distribution License (the "License").
    6  * You may not use this file except in compliance with the License.
    7  *
    8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
    9  * or https://opensource.org/licenses/CDDL-1.0.
   10  * See the License for the specific language governing permissions
   11  * and limitations under the License.
   12  *
   13  * When distributing Covered Code, include this CDDL HEADER in each
   14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
   15  * If applicable, add the following below this CDDL HEADER, with the
   16  * fields enclosed by brackets "[]" replaced with your own identifying
   17  * information: Portions Copyright [yyyy] [name of copyright owner]
   18  *
   19  * CDDL HEADER END
   20  */
   21 /*
   22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
   23  * Copyright (c) 2012, 2021 by Delphix. All rights reserved.
   24  */
   25 
   26 #include <sys/zfs_context.h>
   27 #include <sys/zfs_refcount.h>
   28 
   29 #ifdef  ZFS_DEBUG
   30 /*
   31  * Reference count tracking is disabled by default.  It's memory requirements
   32  * are reasonable, however as implemented it consumes a significant amount of
   33  * cpu time.  Until its performance is improved it should be manually enabled.
   34  */
   35 int reference_tracking_enable = B_FALSE;
   36 static uint_t reference_history = 3; /* tunable */
   37 
   38 static kmem_cache_t *reference_cache;
   39 static kmem_cache_t *reference_history_cache;
   40 
   41 void
   42 zfs_refcount_init(void)
   43 {
   44         reference_cache = kmem_cache_create("reference_cache",
   45             sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
   46 
   47         reference_history_cache = kmem_cache_create("reference_history_cache",
   48             sizeof (uint64_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
   49 }
   50 
   51 void
   52 zfs_refcount_fini(void)
   53 {
   54         kmem_cache_destroy(reference_cache);
   55         kmem_cache_destroy(reference_history_cache);
   56 }
   57 
   58 void
   59 zfs_refcount_create(zfs_refcount_t *rc)
   60 {
   61         mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL);
   62         list_create(&rc->rc_list, sizeof (reference_t),
   63             offsetof(reference_t, ref_link));
   64         list_create(&rc->rc_removed, sizeof (reference_t),
   65             offsetof(reference_t, ref_link));
   66         rc->rc_count = 0;
   67         rc->rc_removed_count = 0;
   68         rc->rc_tracked = reference_tracking_enable;
   69 }
   70 
   71 void
   72 zfs_refcount_create_tracked(zfs_refcount_t *rc)
   73 {
   74         zfs_refcount_create(rc);
   75         rc->rc_tracked = B_TRUE;
   76 }
   77 
   78 void
   79 zfs_refcount_create_untracked(zfs_refcount_t *rc)
   80 {
   81         zfs_refcount_create(rc);
   82         rc->rc_tracked = B_FALSE;
   83 }
   84 
   85 void
   86 zfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number)
   87 {
   88         reference_t *ref;
   89 
   90         ASSERT3U(rc->rc_count, ==, number);
   91         while ((ref = list_head(&rc->rc_list))) {
   92                 list_remove(&rc->rc_list, ref);
   93                 kmem_cache_free(reference_cache, ref);
   94         }
   95         list_destroy(&rc->rc_list);
   96 
   97         while ((ref = list_head(&rc->rc_removed))) {
   98                 list_remove(&rc->rc_removed, ref);
   99                 kmem_cache_free(reference_history_cache, ref->ref_removed);
  100                 kmem_cache_free(reference_cache, ref);
  101         }
  102         list_destroy(&rc->rc_removed);
  103         mutex_destroy(&rc->rc_mtx);
  104 }
  105 
  106 void
  107 zfs_refcount_destroy(zfs_refcount_t *rc)
  108 {
  109         zfs_refcount_destroy_many(rc, 0);
  110 }
  111 
  112 int
  113 zfs_refcount_is_zero(zfs_refcount_t *rc)
  114 {
  115         return (zfs_refcount_count(rc) == 0);
  116 }
  117 
  118 int64_t
  119 zfs_refcount_count(zfs_refcount_t *rc)
  120 {
  121         return (atomic_load_64(&rc->rc_count));
  122 }
  123 
  124 int64_t
  125 zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder)
  126 {
  127         reference_t *ref = NULL;
  128         int64_t count;
  129 
  130         if (!rc->rc_tracked) {
  131                 count = atomic_add_64_nv(&(rc)->rc_count, number);
  132                 ASSERT3U(count, >=, number);
  133                 return (count);
  134         }
  135 
  136         ref = kmem_cache_alloc(reference_cache, KM_SLEEP);
  137         ref->ref_holder = holder;
  138         ref->ref_number = number;
  139         mutex_enter(&rc->rc_mtx);
  140         list_insert_head(&rc->rc_list, ref);
  141         rc->rc_count += number;
  142         count = rc->rc_count;
  143         mutex_exit(&rc->rc_mtx);
  144 
  145         return (count);
  146 }
  147 
  148 int64_t
  149 zfs_refcount_add(zfs_refcount_t *rc, const void *holder)
  150 {
  151         return (zfs_refcount_add_many(rc, 1, holder));
  152 }
  153 
  154 int64_t
  155 zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number,
  156     const void *holder)
  157 {
  158         reference_t *ref;
  159         int64_t count;
  160 
  161         if (!rc->rc_tracked) {
  162                 count = atomic_add_64_nv(&(rc)->rc_count, -number);
  163                 ASSERT3S(count, >=, 0);
  164                 return (count);
  165         }
  166 
  167         mutex_enter(&rc->rc_mtx);
  168         ASSERT3U(rc->rc_count, >=, number);
  169         for (ref = list_head(&rc->rc_list); ref;
  170             ref = list_next(&rc->rc_list, ref)) {
  171                 if (ref->ref_holder == holder && ref->ref_number == number) {
  172                         list_remove(&rc->rc_list, ref);
  173                         if (reference_history > 0) {
  174                                 ref->ref_removed =
  175                                     kmem_cache_alloc(reference_history_cache,
  176                                     KM_SLEEP);
  177                                 list_insert_head(&rc->rc_removed, ref);
  178                                 rc->rc_removed_count++;
  179                                 if (rc->rc_removed_count > reference_history) {
  180                                         ref = list_tail(&rc->rc_removed);
  181                                         list_remove(&rc->rc_removed, ref);
  182                                         kmem_cache_free(reference_history_cache,
  183                                             ref->ref_removed);
  184                                         kmem_cache_free(reference_cache, ref);
  185                                         rc->rc_removed_count--;
  186                                 }
  187                         } else {
  188                                 kmem_cache_free(reference_cache, ref);
  189                         }
  190                         rc->rc_count -= number;
  191                         count = rc->rc_count;
  192                         mutex_exit(&rc->rc_mtx);
  193                         return (count);
  194                 }
  195         }
  196         panic("No such hold %p on refcount %llx", holder,
  197             (u_longlong_t)(uintptr_t)rc);
  198         return (-1);
  199 }
  200 
  201 int64_t
  202 zfs_refcount_remove(zfs_refcount_t *rc, const void *holder)
  203 {
  204         return (zfs_refcount_remove_many(rc, 1, holder));
  205 }
  206 
  207 void
  208 zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src)
  209 {
  210         int64_t count, removed_count;
  211         list_t list, removed;
  212 
  213         list_create(&list, sizeof (reference_t),
  214             offsetof(reference_t, ref_link));
  215         list_create(&removed, sizeof (reference_t),
  216             offsetof(reference_t, ref_link));
  217 
  218         mutex_enter(&src->rc_mtx);
  219         count = src->rc_count;
  220         removed_count = src->rc_removed_count;
  221         src->rc_count = 0;
  222         src->rc_removed_count = 0;
  223         list_move_tail(&list, &src->rc_list);
  224         list_move_tail(&removed, &src->rc_removed);
  225         mutex_exit(&src->rc_mtx);
  226 
  227         mutex_enter(&dst->rc_mtx);
  228         dst->rc_count += count;
  229         dst->rc_removed_count += removed_count;
  230         list_move_tail(&dst->rc_list, &list);
  231         list_move_tail(&dst->rc_removed, &removed);
  232         mutex_exit(&dst->rc_mtx);
  233 
  234         list_destroy(&list);
  235         list_destroy(&removed);
  236 }
  237 
  238 void
  239 zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number,
  240     const void *current_holder, const void *new_holder)
  241 {
  242         reference_t *ref;
  243         boolean_t found = B_FALSE;
  244 
  245         if (!rc->rc_tracked)
  246                 return;
  247 
  248         mutex_enter(&rc->rc_mtx);
  249         for (ref = list_head(&rc->rc_list); ref;
  250             ref = list_next(&rc->rc_list, ref)) {
  251                 if (ref->ref_holder == current_holder &&
  252                     ref->ref_number == number) {
  253                         ref->ref_holder = new_holder;
  254                         found = B_TRUE;
  255                         break;
  256                 }
  257         }
  258         ASSERT(found);
  259         mutex_exit(&rc->rc_mtx);
  260 }
  261 
  262 void
  263 zfs_refcount_transfer_ownership(zfs_refcount_t *rc, const void *current_holder,
  264     const void *new_holder)
  265 {
  266         return (zfs_refcount_transfer_ownership_many(rc, 1, current_holder,
  267             new_holder));
  268 }
  269 
  270 /*
  271  * If tracking is enabled, return true if a reference exists that matches
  272  * the "holder" tag. If tracking is disabled, then return true if a reference
  273  * might be held.
  274  */
  275 boolean_t
  276 zfs_refcount_held(zfs_refcount_t *rc, const void *holder)
  277 {
  278         reference_t *ref;
  279 
  280         if (!rc->rc_tracked)
  281                 return (zfs_refcount_count(rc) > 0);
  282 
  283         mutex_enter(&rc->rc_mtx);
  284         for (ref = list_head(&rc->rc_list); ref;
  285             ref = list_next(&rc->rc_list, ref)) {
  286                 if (ref->ref_holder == holder) {
  287                         mutex_exit(&rc->rc_mtx);
  288                         return (B_TRUE);
  289                 }
  290         }
  291         mutex_exit(&rc->rc_mtx);
  292         return (B_FALSE);
  293 }
  294 
  295 /*
  296  * If tracking is enabled, return true if a reference does not exist that
  297  * matches the "holder" tag. If tracking is disabled, always return true
  298  * since the reference might not be held.
  299  */
  300 boolean_t
  301 zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder)
  302 {
  303         reference_t *ref;
  304 
  305         if (!rc->rc_tracked)
  306                 return (B_TRUE);
  307 
  308         mutex_enter(&rc->rc_mtx);
  309         for (ref = list_head(&rc->rc_list); ref;
  310             ref = list_next(&rc->rc_list, ref)) {
  311                 if (ref->ref_holder == holder) {
  312                         mutex_exit(&rc->rc_mtx);
  313                         return (B_FALSE);
  314                 }
  315         }
  316         mutex_exit(&rc->rc_mtx);
  317         return (B_TRUE);
  318 }
  319 
  320 EXPORT_SYMBOL(zfs_refcount_create);
  321 EXPORT_SYMBOL(zfs_refcount_destroy);
  322 EXPORT_SYMBOL(zfs_refcount_is_zero);
  323 EXPORT_SYMBOL(zfs_refcount_count);
  324 EXPORT_SYMBOL(zfs_refcount_add);
  325 EXPORT_SYMBOL(zfs_refcount_remove);
  326 EXPORT_SYMBOL(zfs_refcount_held);
  327 
  328 /* BEGIN CSTYLED */
  329 ZFS_MODULE_PARAM(zfs, , reference_tracking_enable, INT, ZMOD_RW,
  330         "Track reference holders to refcount_t objects");
  331 
  332 ZFS_MODULE_PARAM(zfs, , reference_history, UINT, ZMOD_RW,
  333         "Maximum reference holders being tracked");
  334 /* END CSTYLED */
  335 #endif  /* ZFS_DEBUG */
Cache object: ebd193912a93e4219ee1cf53d62667bc
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/contrib/openzfs/module/zfs/refcount.c

FreeBSD/Linux Kernel Cross Reference
sys/contrib/openzfs/module/zfs/refcount.c
