FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_kcont.c

     /* $NetBSD: kern_kcont.c,v 1.13 2005/12/24 19:12:23 perry Exp $ */
     
     /*
      * Copyright 2003 Jonathan Stone.
      * All rights reserved.
      * Copyright (c) 2004 The NetBSD Foundation, Inc.
      * All rights reserved.
      *
      * This code is derived from software contributed to The NetBSD Foundation
     * by Jonathan Stone.
     *
     * Redistribution and use in source and binary forms, with or without
     * modification, are permitted provided that the following conditions
     * are met:
     * 1. Redistributions of source code must retain the above copyright
     *    notice, this list of conditions and the following disclaimer.
     * 2. Redistributions in binary form must reproduce the above copyright
     *    notice, this list of conditions and the following disclaimer in the
     *    documentation and/or other materials provided with the distribution.
     * 3. Neither the name of The NetBSD Foundation nor the names of its
     *    contributors may be used to endorse or promote products derived
     *    from this software without specific prior written permission.
     *
     * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     * POSSIBILITY OF SUCH DAMAGE.
     */
    
    /*
     */
    #include <sys/cdefs.h>
    __KERNEL_RCSID(0, "$NetBSD: kern_kcont.c,v 1.13 2005/12/24 19:12:23 perry Exp $ ");
    
    #include <sys/types.h>
    #include <sys/param.h>
    #include <sys/queue.h>
    #include <sys/errno.h>
    #include <sys/malloc.h>
    #include <sys/kernel.h>
    #include <sys/pool.h>
    #include <sys/kthread.h>
    #include <sys/proc.h>
    #include <sys/systm.h>
    #include <lib/libkern/libkern.h>
    
    #include <machine/intr.h>       /* IPL_*, and schedsoftnet() */
                                    /* XXX: schedsofnet() should die. */
    
    #include <sys/kcont.h>
    
    #ifdef __HAVE_GENERIC_SOFT_INTERRUPTS
    /*
     * Software-interrupt priority continuation queues.
     */
    static kcq_t kcq_softnet;
    static kcq_t kcq_softclock;
    static kcq_t kcq_softserial;
    
    static void *kc_si_softnet;
    static void *kc_si_softclock;
    static void *kc_si_softserial;
    #endif /* __HAVE_GENERIC_SOFT_INTERRUPTS */
    
    /*
     * Pool allocator structure.
     */
    POOL_INIT(kc_pool, sizeof(struct kc), 0, 0, 0, "kcpl", NULL);
    
    /*
     * Process-context continuation queue.
     */
    static kcq_t kcq_process_ctxt;
    
    /*
     * Insert/Remove a fully-formed struct kc * into the kc_queue *
     * of some kernel object (e.g., a struct buf *).
     * For fine-grained SMP, both enqueueing and dequeueing will
     * need a locking mechanism.
     */
    static inline void
    kcont_enqueue_atomic(kcq_t *kcq, struct kc *kc)
    {
            int s;
    
            s = splvm();
            SIMPLEQ_INSERT_TAIL(kcq, kc, kc_next);
            splx(s);
    }
    
    static inline struct kc *
    kcont_dequeue_atomic(kcq_t *kcq)
   {
           struct kc *kc;
           int s;
   
           s = splvm();
           kc = SIMPLEQ_FIRST(kcq);
           if (kc != NULL) {
                   SIMPLEQ_REMOVE_HEAD(kcq, kc_next);
                   SIMPLEQ_NEXT(kc, kc_next) = NULL;
           }
           splx(s);
           return kc;
   }
   
   /*
    * Construct a continuation object from pre-allocated memory.
    * Used by functions that are about call an asynchronous operation,
    * to build a continuation to be called once the operation completes.
    */
   static inline struct kc *
   kc_set(struct kc *kc, void (*func)(void *, void *, int),
       void *env_arg, int ipl)
   {
   
           kc->kc_fn = func;
           kc->kc_env_arg = env_arg;
           kc->kc_ipl = ipl;
           kc->kc_flags = 0;
   #ifdef DEBUG
           kc->kc_obj = NULL;
           kc->kc_status = -1;
           SIMPLEQ_NEXT(kc, kc_next) = NULL;
   #endif
           return kc;
   }
   
   /*
    * Request a continuation.  Caller provides space for the struct kc *.
    */
   struct kc *
   kcont(struct kc *kc, void (*func)(void *, void *, int),
       void *env_arg, int continue_ipl)
   {
   
           /* Just save the arguments in the kcont *. */
           return kc_set(kc, func, env_arg, continue_ipl);
   }
   
   /*
    * Request a malloc'ed/auto-freed continuation. The kcont framework
    * mallocs the struct kc, and initializes it with the caller-supplied args.
    * Once the asynchronous operation completes and the continuation function
    * has been called, the kcont framework will free the struct kc *
    * immediately after the continuation function returns.
    */
   struct kc *
   kcont_malloc(int malloc_flags,
       void (*func)(void *, void *, int),
       void *env_arg, int continue_ipl)
   {
           struct kc *kc;
           int pool_flags;
   
           pool_flags = (malloc_flags & M_NOWAIT) ? 0 : PR_WAITOK;
           pool_flags |= (malloc_flags & M_CANFAIL) ? PR_LIMITFAIL : 0;
   
           kc = pool_get(&kc_pool, pool_flags);
           if (kc == NULL)
                   return kc;
           return kc_set(kc, func, env_arg, continue_ipl);
   }
   
   /*
    * Dispatch a dequeued continuation which requested deferral
    * into the appropriate lower-priority queue.
    * Public API entry to defer execution of a pre-built kcont.
    */
   void
   kcont_defer(struct kc *kc, void *obj, int status)
   {
           /*
            * IPL at which to synchronize access to object is
            * above the continuer's requested callback IPL,
            * (e.g., continuer wants IPL_SOFTNET but the object
            * holding this continuation incurred the wakeup()able
            * event whilst at IPL_BIO).
            * Defer this kc to a lower-priority kc queue,
            * to be serviced slightly later at a lower IPL.
            */
   
           /*
            * If we already deferred this kcont, don't clobber
            * the previously-saved kc_object and kc_status.
            * (The obj/status arguments passed in by ck_run() should
            * be the same as kc_object/kc_status, but don't rely on that.)
            */
           if ((kc->kc_flags & KC_DEFERRED) == 0) {
                   kc->kc_flags |= KC_DEFERRED;
                   kc->kc_obj = obj;
                   kc->kc_status = status;
           }
   
           switch (kc->kc_ipl) {
   #ifdef __HAVE_GENERIC_SOFT_INTERRUPTS
           case KC_IPL_DEFER_SOFTCLOCK:
                   kcont_enqueue_atomic(&kcq_softclock, kc);
                   softintr_schedule(kc_si_softclock);
                   break;
           case KC_IPL_DEFER_SOFTNET:
                   kcont_enqueue_atomic(&kcq_softnet, kc);
                   softintr_schedule(kc_si_softnet);
                   break;
           case KC_IPL_DEFER_SOFTSERIAL:
                   kcont_enqueue_atomic(&kcq_softserial, kc);
                   softintr_schedule(kc_si_softserial);
                   break;
   
   #else /* !__HAVE_GENERIC_SOFT_INTERRUPTS */
           /* What to do? For now, punt to process context */
           case KC_IPL_DEFER_SOFTCLOCK:
           case KC_IPL_DEFER_SOFTSERIAL:
           case KC_IPL_DEFER_SOFTNET:
                   /*FALLTHROUGH*/
   #endif /* __HAVE_GENERIC_SOFT_INTERRUPTS */
   
           case KC_IPL_DEFER_PROCESS:
                   kcont_enqueue_atomic(&kcq_process_ctxt, kc);
                   wakeup(&kcq_process_ctxt);
                   break;
           default:
                   KASSERT(0);
           }
   }
   
   void
   kcont_defer_malloc(int mallocflags,
       void (*func)(void *, void *, int),
       void *obj, void *env_arg, int status, int ipl)
   {
           struct kc *kc;
   
           kc = kcont_malloc(mallocflags, func, env_arg, ipl);
           if (kc != NULL)
                   kcont_defer(kc, obj, status);
   }
   
   /*
    * Enqueue a pre-existing kcont onto a struct kcq completion queue
    * of some pre-existing kernel object.
    */
   void
   kcont_enqueue(kcq_t *kcq, struct kc *kc)
   {
   
           kcont_enqueue_atomic(kcq, kc);
   }
   
   
   /*
    * Run through a list of continuations, calling (or handing off)
    * continuation functions.
    * If the caller-provided IPL is the same as the requested IPL,
    * deliver the callback.
    * If the caller-provided IPL is higher than the requested
    * callback IPL, re-enqueue the continuation to a lower-priority queue.
    */
   void
   kcont_run(kcq_t *kcq, void *obj, int status, int curipl)
   {
           struct kc *kc;
   
           while ((kc = kcont_dequeue_atomic(kcq)) != NULL) {
   
                   /* If execution of kc was already deferred, restore context. */
                   if (kc->kc_flags & KC_DEFERRED) {
                           KASSERT(obj == NULL);
                           obj = kc->kc_obj;
                           status = kc->kc_status;
                   }
   
                   /* Check whether to execute now or to defer. */
                   if (kc->kc_ipl == KC_IPL_IMMED || curipl <= kc->kc_ipl) {
                           int saved_flags = kc->kc_flags; /* XXX see below */
   
                           /* Satisfy our raison d'e^tre */
                           (*kc->kc_fn)(obj, kc->kc_env_arg, status);
   
                           /*
                            * We must not touch (*kc) after calling
                            * (*kc->kc_fn), unless we were specifically
                            * asked to free it.  The memory for (*kc) may
                            * be a sub-field of some other object (for example,
                            * of kc->kc_env_arg) and (*kc_fn)() may already
                            * have freed it by the time we get here.  So save
                            * kc->kc_flags (above) and use that saved copy
                            * to test for auto-free.
                            */
                           if (saved_flags & KC_AUTOFREE)
                                   pool_put(&kc_pool, kc);
                   } else {
                           kcont_defer(kc, obj, status);
                   }
           }
   }
   
   #ifdef __HAVE_GENERIC_SOFT_INTERRUPTS
   /*
    * Trampolines for processing software-interrupt kcont queues.
    */
   static void
   kcont_run_softclock(void *arg)
   {
   
           kcont_run((struct kcqueue *)arg, NULL, 0, KC_IPL_DEFER_SOFTCLOCK);
   }
   
   static void
   kcont_run_softnet(void *arg)
   {
   
           kcont_run((struct kcqueue *)arg, NULL, 0, KC_IPL_DEFER_SOFTNET);
   }
   
   static void
   kcont_run_softserial(void *arg)
   {
   
           kcont_run((struct kcqueue *)arg, NULL, 0, KC_IPL_DEFER_SOFTSERIAL);
   }
   #endif /* __HAVE_GENERIC_SOFT_INTERRUPTS */
   
   /*
    * Main entrypoint for kcont worker kthreads to execute
    * a continuation which requested deferral to process context.
    */
   static void
   kcont_worker(void *arg)
   {
           int status;
   
           (void)arg;      /* kill GCC warning */
   
           while (1) {
                   status = ltsleep(&kcq_process_ctxt, PCATCH, "kcont", hz, NULL);
                   if (status != 0 && status != EWOULDBLOCK)
                           break;
                   kcont_run(&kcq_process_ctxt, NULL, 0, KC_IPL_DEFER_PROCESS);
           }
           kthread_exit(0);
   }
   
   static void
   kcont_create_worker(void *arg)
   {
           if (kthread_create1(kcont_worker, NULL, NULL, "kcont"))
                   panic("fork kcont");
   }
   
   /*
    * Initialize kcont subsystem.
    */
   void
   kcont_init(void)
   {
   
   #ifdef __HAVE_GENERIC_SOFT_INTERRUPTS
           /*
            * Initialize kc_queue and callout for soft-int deferred
            * continuations. (If not available, deferrals fall back
            * to deferring all the way to process context).
            */
           SIMPLEQ_INIT(&kcq_softclock);
           kc_si_softclock = softintr_establish(IPL_SOFTCLOCK,
               kcont_run_softclock, &kcq_softnet);
   
           SIMPLEQ_INIT(&kcq_softnet);
           kc_si_softnet = softintr_establish(IPL_SOFTNET,
               kcont_run_softnet, &kcq_softnet);
   
           SIMPLEQ_INIT(&kcq_softserial);
           kc_si_softserial = softintr_establish(IPL_SOFTSERIAL,
               kcont_run_softserial, &kcq_softserial);
   #endif  /* __HAVE_GENERIC_SOFT_INTERRUPTS */
   
           /*
            * Create kc_queue for process-context continuations, and
            * a worker kthread to process the queue. (Fine-grained SMP
            * locking should have at least one worker kthread per CPU).
            */
           SIMPLEQ_INIT(&kcq_process_ctxt);
           kthread_create(kcont_create_worker, NULL);
   }

Cache object: e42caa240bfbb67169c991f7c313f984