The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_mpipe.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * (MPSAFE)
    3  *
    4  * Copyright (c) 2003,2004 The DragonFly Project.  All rights reserved.
    5  * 
    6  * This code is derived from software contributed to The DragonFly Project
    7  * by Matthew Dillon <dillon@backplane.com>
    8  * 
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in
   17  *    the documentation and/or other materials provided with the
   18  *    distribution.
   19  * 3. Neither the name of The DragonFly Project nor the names of its
   20  *    contributors may be used to endorse or promote products derived
   21  *    from this software without specific, prior written permission.
   22  * 
   23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   24  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
   25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
   26  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
   27  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
   28  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
   29  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
   30  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
   31  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   32  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
   33  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   34  * SUCH DAMAGE.
   35  */
   36 
   37 #include <sys/param.h>
   38 #include <sys/systm.h>
   39 #include <sys/kernel.h>
   40 #include <sys/slaballoc.h>
   41 #include <sys/mbuf.h>
   42 #include <sys/vmmeter.h>
   43 #include <sys/lock.h>
   44 #include <sys/thread.h>
   45 #include <sys/globaldata.h>
   46 #include <sys/mpipe.h>
   47 #include <sys/kthread.h>
   48 
   49 #include <sys/thread2.h>
   50 
   51 struct mpipe_callback {
   52         STAILQ_ENTRY(mpipe_callback) entry;
   53         void (*func)(void *arg1, void *arg2);
   54         void *arg1;
   55         void *arg2;
   56 };
   57 
   58 static MALLOC_DEFINE(M_MPIPEARY, "MPipe Array", "Auxiliary MPIPE structure");
   59 
   60 static void mpipe_thread(void *arg);
   61 
   62 /*
   63  * Initialize a malloc pipeline for the specified malloc type and allocation
   64  * size.  Create an array to cache up to nom_count buffers and preallocate
   65  * them.
   66  */
   67 void
   68 mpipe_init(malloc_pipe_t mpipe, malloc_type_t type, int bytes,
   69         int nnom, int nmax, 
   70         int mpflags, 
   71         void (*construct)(void *, void *),
   72         void (*deconstruct)(void *, void *),
   73         void *priv)
   74 {
   75     int n;
   76 
   77     if (nnom < 1)
   78         nnom = 1;
   79     if (nmax < 0)
   80         nmax = 0x7FFF0000;      /* some very large number */
   81     if (nmax < nnom)
   82         nmax = nnom;
   83     bzero(mpipe, sizeof(struct malloc_pipe));
   84     mpipe->type = type;
   85     mpipe->bytes = bytes;
   86     mpipe->mpflags = mpflags;
   87     mpipe->construct = construct;
   88     mpipe->deconstruct = deconstruct;
   89     mpipe->priv = priv;
   90     if ((mpflags & MPF_NOZERO) == 0)
   91         mpipe->mflags |= M_ZERO;
   92     if (mpflags & MPF_INT)
   93         mpipe->mflags |= M_USE_RESERVE | M_USE_INTERRUPT_RESERVE;
   94     mpipe->ary_count = nnom;
   95     mpipe->max_count = nmax;
   96     mpipe->array = kmalloc(nnom * sizeof(mpipe->array[0]), M_MPIPEARY, 
   97                             M_WAITOK | M_ZERO);
   98 
   99     while (mpipe->free_count < nnom) {
  100         n = mpipe->free_count;
  101         mpipe->array[n] = kmalloc(bytes, mpipe->type, M_WAITOK | mpipe->mflags);
  102         if (construct)
  103             construct(mpipe->array[n], priv);
  104         ++mpipe->free_count;
  105         ++mpipe->total_count;
  106     }
  107     STAILQ_INIT(&mpipe->queue);
  108 
  109     lwkt_token_init(&mpipe->token, "mpipe token");
  110 
  111     /*
  112      * Create a support thread for the mpipe queue
  113      */
  114     if (mpflags & MPF_CALLBACK) {
  115             kthread_create(mpipe_thread, mpipe, &mpipe->thread,
  116                            "mpipe_%s", type->ks_shortdesc);
  117     }
  118 }
  119 
  120 /*
  121  * Destroy a previously initialized mpipe.  This routine can also safely be
  122  * called on an uninitialized mpipe structure if it was zero'd or mpipe_done()
  123  * was previously called on it.
  124  */
  125 void
  126 mpipe_done(malloc_pipe_t mpipe)
  127 {
  128     void *buf;
  129     int n;
  130 
  131     KKASSERT(mpipe->free_count == mpipe->total_count);  /* no outstanding mem */
  132 
  133     /*
  134      * Clean up the kthread
  135      */
  136     lwkt_gettoken(&mpipe->token);
  137     mpipe->mpflags |= MPF_EXITING;
  138     while (mpipe->thread) {
  139         wakeup(&mpipe->queue);
  140         tsleep(mpipe, 0, "mpipex", 1);
  141     }
  142 
  143     /*
  144      * Clean up the mpipe buffers
  145      */
  146     for (n = mpipe->free_count - 1; n >= 0; --n) {
  147         buf = mpipe->array[n];
  148         mpipe->array[n] = NULL;
  149         KKASSERT(buf != NULL);
  150         if (mpipe->deconstruct)
  151             mpipe->deconstruct(buf, mpipe->priv);
  152         kfree(buf, mpipe->type);
  153     }
  154     mpipe->free_count = 0;
  155     mpipe->total_count = 0;
  156     if (mpipe->array) {
  157         kfree(mpipe->array, M_MPIPEARY);
  158         mpipe->array = NULL;
  159     }
  160     lwkt_reltoken(&mpipe->token);
  161     lwkt_token_uninit(&mpipe->token);
  162 }
  163 
  164 /*
  165  * mpipe support thread for request failures when mpipe_alloc_callback()
  166  * is called.
  167  */
  168 static void
  169 mpipe_thread(void *arg)
  170 {
  171     malloc_pipe_t mpipe = arg;
  172     struct mpipe_callback *mcb;
  173 
  174     lwkt_gettoken(&mpipe->token);
  175     while ((mpipe->mpflags & MPF_EXITING) == 0) {
  176         while (mpipe->free_count &&
  177                (mcb = STAILQ_FIRST(&mpipe->queue)) != NULL) {
  178                 STAILQ_REMOVE(&mpipe->queue, mcb, mpipe_callback, entry);
  179                 mcb->func(mcb->arg1, mcb->arg2);
  180                 kfree(mcb, M_MPIPEARY);
  181         }
  182         mpipe->mpflags |= MPF_QUEUEWAIT;
  183         tsleep(&mpipe->queue, 0, "wait", 0);
  184     }
  185     mpipe->thread = NULL;
  186     wakeup(mpipe);
  187     lwkt_reltoken(&mpipe->token);
  188 }
  189 
  190 
  191 /*
  192  * Allocate an entry (inline suppot routine).  The allocation is guarenteed
  193  * to return non-NULL up to the nominal count after which it may return NULL.
  194  * Note that the implementation is defined to be allowed to block for short
  195  * periods of time.
  196  *
  197  * Use mpipe_alloc_callback() for non-blocking operation with a callback
  198  * Use mpipe_alloc_nowait() for non-blocking operation without a callback
  199  * Use mpipe_alloc_waitok() for blocking operation & guarenteed non-NULL
  200  */
  201 static __inline
  202 void *
  203 _mpipe_alloc_locked(malloc_pipe_t mpipe, int mfailed)
  204 {
  205     void *buf;
  206     int n;
  207 
  208     if ((n = mpipe->free_count) != 0) {
  209         /*
  210          * Use a free entry if it exists.
  211          */
  212         --n;
  213         buf = mpipe->array[n];
  214         mpipe->array[n] = NULL; /* sanity check, not absolutely needed */
  215         mpipe->free_count = n;
  216     } else if (mpipe->total_count >= mpipe->max_count || mfailed) {
  217         /*
  218          * Return NULL if we have hit our limit
  219          */
  220         buf = NULL;
  221     } else {
  222         /*
  223          * Otherwise try to malloc() non-blocking.
  224          */
  225         buf = kmalloc(mpipe->bytes, mpipe->type, M_NOWAIT | mpipe->mflags);
  226         if (buf) {
  227             ++mpipe->total_count; 
  228             if (mpipe->construct)
  229                 mpipe->construct(buf, mpipe->priv);
  230         }
  231     }
  232     return(buf);
  233 }
  234 
  235 /*
  236  * Nominal non-blocking mpipe allocation
  237  */
  238 void *
  239 mpipe_alloc_nowait(malloc_pipe_t mpipe)
  240 {
  241     void *buf;
  242 
  243     lwkt_gettoken(&mpipe->token);
  244     buf = _mpipe_alloc_locked(mpipe, 0);
  245     lwkt_reltoken(&mpipe->token);
  246 
  247     return(buf);
  248 }
  249 
  250 /*
  251  * non-blocking mpipe allocation with callback for retry.
  252  *
  253  * If NULL is returned func(arg) is queued and will be called back when
  254  * space is likely (but not necessarily) available.
  255  *
  256  * If non-NULL is returned func(arg) is ignored.
  257  */
  258 void *
  259 mpipe_alloc_callback(malloc_pipe_t mpipe, void (*func)(void *arg1, void *arg2),
  260                      void *arg1, void *arg2)
  261 {
  262     struct mpipe_callback *mcb;
  263     void *buf;
  264 
  265     lwkt_gettoken(&mpipe->token);
  266     buf = _mpipe_alloc_locked(mpipe, 0);
  267     if (buf == NULL) {
  268         mcb = kmalloc(sizeof(*mcb), M_MPIPEARY, M_INTWAIT);
  269         buf = _mpipe_alloc_locked(mpipe, 0);
  270         if (buf == NULL) {
  271             mcb->func = func;
  272             mcb->arg1 = arg1;
  273             mcb->arg2 = arg2;
  274             STAILQ_INSERT_TAIL(&mpipe->queue, mcb, entry);
  275         } else {
  276             kfree(mcb, M_MPIPEARY);
  277         }
  278     }
  279     lwkt_reltoken(&mpipe->token);
  280 
  281     return(buf);
  282 }
  283 
  284 /*
  285  * This function can be called to nominally wait until resources are
  286  * available and mpipe_alloc_nowait() is likely to return non-NULL.
  287  *
  288  * NOTE: mpipe_alloc_nowait() can still return NULL.
  289  */
  290 void
  291 mpipe_wait(malloc_pipe_t mpipe)
  292 {
  293     if (mpipe->free_count == 0) {
  294         lwkt_gettoken(&mpipe->token);
  295         while ((mpipe->mpflags & MPF_EXITING) == 0) {
  296             if (mpipe->free_count)
  297                     break;
  298             mpipe->mpflags |= MPF_QUEUEWAIT;
  299             tsleep(&mpipe->queue, 0, "wait", 0);
  300         }
  301         lwkt_reltoken(&mpipe->token);
  302     }
  303 }
  304 
  305 /*
  306  * Allocate an entry, block until the allocation succeeds.  This may cause
  307  * us to block waiting for a prior allocation to be freed.
  308  */
  309 void *
  310 mpipe_alloc_waitok(malloc_pipe_t mpipe)
  311 {
  312     void *buf;
  313     int mfailed;
  314 
  315     lwkt_gettoken(&mpipe->token);
  316     mfailed = 0;
  317     while ((buf = _mpipe_alloc_locked(mpipe, mfailed)) == NULL) {
  318         /*
  319          * Block if we have hit our limit
  320          */
  321         mpipe->pending = 1;
  322         tsleep(mpipe, 0, "mpipe1", 0);
  323         mfailed = 1;
  324     }
  325     lwkt_reltoken(&mpipe->token);
  326 
  327     return(buf);
  328 }
  329 
  330 /*
  331  * Free an entry, unblock any waiters.  Allow NULL.
  332  */
  333 void
  334 mpipe_free(malloc_pipe_t mpipe, void *buf)
  335 {
  336     int n;
  337 
  338     if (buf == NULL)
  339         return;
  340 
  341     lwkt_gettoken(&mpipe->token);
  342     if ((n = mpipe->free_count) < mpipe->ary_count) {
  343         /*
  344          * Free slot available in free array (LIFO)
  345          */
  346         mpipe->array[n] = buf;
  347         ++mpipe->free_count;
  348         if ((mpipe->mpflags & (MPF_CACHEDATA|MPF_NOZERO)) == 0) 
  349             bzero(buf, mpipe->bytes);
  350         if (mpipe->mpflags & MPF_QUEUEWAIT) {
  351                 mpipe->mpflags &= ~MPF_QUEUEWAIT;
  352                 lwkt_reltoken(&mpipe->token);
  353                 wakeup(&mpipe->queue);
  354         } else {
  355                 lwkt_reltoken(&mpipe->token);
  356         }
  357         /*
  358          * Wakeup anyone blocked in mpipe_alloc_*().
  359          */
  360         if (mpipe->pending) {
  361             mpipe->pending = 0;
  362             wakeup(mpipe);
  363         }
  364     } else {
  365         /*
  366          * All the free slots are full, free the buffer directly.
  367          */
  368         --mpipe->total_count;
  369         KKASSERT(mpipe->total_count >= mpipe->free_count);
  370         if (mpipe->deconstruct)
  371             mpipe->deconstruct(buf, mpipe->priv);
  372         lwkt_reltoken(&mpipe->token);
  373         kfree(buf, mpipe->type);
  374     }
  375 }
  376 

Cache object: eed69ad7d5998f7c9ed2889d942c846d


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.