The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_event.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
    3  * Copyright 2004 John-Mark Gurney <jmg@FreeBSD.org>
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   25  * SUCH DAMAGE.
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __FBSDID("$FreeBSD: releng/7.4/sys/kern/kern_event.c 206606 2010-04-14 15:33:30Z jhb $");
   30 
   31 #include "opt_ktrace.h"
   32 
   33 #include <sys/param.h>
   34 #include <sys/systm.h>
   35 #include <sys/kernel.h>
   36 #include <sys/lock.h>
   37 #include <sys/mutex.h>
   38 #include <sys/proc.h>
   39 #include <sys/malloc.h>
   40 #include <sys/unistd.h>
   41 #include <sys/file.h>
   42 #include <sys/filedesc.h>
   43 #include <sys/filio.h>
   44 #include <sys/fcntl.h>
   45 #include <sys/kthread.h>
   46 #include <sys/selinfo.h>
   47 #include <sys/queue.h>
   48 #include <sys/event.h>
   49 #include <sys/eventvar.h>
   50 #include <sys/poll.h>
   51 #include <sys/protosw.h>
   52 #include <sys/sigio.h>
   53 #include <sys/signalvar.h>
   54 #include <sys/socket.h>
   55 #include <sys/socketvar.h>
   56 #include <sys/stat.h>
   57 #include <sys/sysctl.h>
   58 #include <sys/sysproto.h>
   59 #include <sys/syscallsubr.h>
   60 #include <sys/taskqueue.h>
   61 #include <sys/uio.h>
   62 #ifdef KTRACE
   63 #include <sys/ktrace.h>
   64 #endif
   65 
   66 #include <vm/uma.h>
   67 
   68 static MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system");
   69 
   70 /*
   71  * This lock is used if multiple kq locks are required.  This possibly
   72  * should be made into a per proc lock.
   73  */
   74 static struct mtx       kq_global;
   75 MTX_SYSINIT(kq_global, &kq_global, "kqueue order", MTX_DEF);
   76 #define KQ_GLOBAL_LOCK(lck, haslck)     do {    \
   77         if (!haslck)                            \
   78                 mtx_lock(lck);                  \
   79         haslck = 1;                             \
   80 } while (0)
   81 #define KQ_GLOBAL_UNLOCK(lck, haslck)   do {    \
   82         if (haslck)                             \
   83                 mtx_unlock(lck);                        \
   84         haslck = 0;                             \
   85 } while (0)
   86 
   87 TASKQUEUE_DEFINE_THREAD(kqueue);
   88 
   89 static int      kevent_copyout(void *arg, struct kevent *kevp, int count);
   90 static int      kevent_copyin(void *arg, struct kevent *kevp, int count);
   91 static int      kqueue_register(struct kqueue *kq, struct kevent *kev,
   92                     struct thread *td, int waitok);
   93 static int      kqueue_acquire(struct file *fp, struct kqueue **kqp);
   94 static void     kqueue_release(struct kqueue *kq, int locked);
   95 static int      kqueue_expand(struct kqueue *kq, struct filterops *fops,
   96                     uintptr_t ident, int waitok);
   97 static void     kqueue_task(void *arg, int pending);
   98 static int      kqueue_scan(struct kqueue *kq, int maxevents,
   99                     struct kevent_copyops *k_ops,
  100                     const struct timespec *timeout,
  101                     struct kevent *keva, struct thread *td);
  102 static void     kqueue_wakeup(struct kqueue *kq);
  103 static struct filterops *kqueue_fo_find(int filt);
  104 static void     kqueue_fo_release(int filt);
  105 
  106 static fo_rdwr_t        kqueue_read;
  107 static fo_rdwr_t        kqueue_write;
  108 static fo_ioctl_t       kqueue_ioctl;
  109 static fo_poll_t        kqueue_poll;
  110 static fo_kqfilter_t    kqueue_kqfilter;
  111 static fo_stat_t        kqueue_stat;
  112 static fo_close_t       kqueue_close;
  113 
  114 static struct fileops kqueueops = {
  115         .fo_read = kqueue_read,
  116         .fo_write = kqueue_write,
  117         .fo_ioctl = kqueue_ioctl,
  118         .fo_poll = kqueue_poll,
  119         .fo_kqfilter = kqueue_kqfilter,
  120         .fo_stat = kqueue_stat,
  121         .fo_close = kqueue_close,
  122 };
  123 
  124 static int      knote_attach(struct knote *kn, struct kqueue *kq);
  125 static void     knote_drop(struct knote *kn, struct thread *td);
  126 static void     knote_enqueue(struct knote *kn);
  127 static void     knote_dequeue(struct knote *kn);
  128 static void     knote_init(void);
  129 static struct   knote *knote_alloc(int waitok);
  130 static void     knote_free(struct knote *kn);
  131 
  132 static void     filt_kqdetach(struct knote *kn);
  133 static int      filt_kqueue(struct knote *kn, long hint);
  134 static int      filt_procattach(struct knote *kn);
  135 static void     filt_procdetach(struct knote *kn);
  136 static int      filt_proc(struct knote *kn, long hint);
  137 static int      filt_fileattach(struct knote *kn);
  138 static void     filt_timerexpire(void *knx);
  139 static int      filt_timerattach(struct knote *kn);
  140 static void     filt_timerdetach(struct knote *kn);
  141 static int      filt_timer(struct knote *kn, long hint);
  142 
  143 static struct filterops file_filtops =
  144         { 1, filt_fileattach, NULL, NULL };
  145 static struct filterops kqread_filtops =
  146         { 1, NULL, filt_kqdetach, filt_kqueue };
  147 /* XXX - move to kern_proc.c?  */
  148 static struct filterops proc_filtops =
  149         { 0, filt_procattach, filt_procdetach, filt_proc };
  150 static struct filterops timer_filtops =
  151         { 0, filt_timerattach, filt_timerdetach, filt_timer };
  152 
  153 static uma_zone_t       knote_zone;
  154 static int              kq_ncallouts = 0;
  155 static int              kq_calloutmax = (4 * 1024);
  156 SYSCTL_INT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW,
  157     &kq_calloutmax, 0, "Maximum number of callouts allocated for kqueue");
  158 
  159 /* XXX - ensure not KN_INFLUX?? */
  160 #define KNOTE_ACTIVATE(kn, islock) do {                                 \
  161         if ((islock))                                                   \
  162                 mtx_assert(&(kn)->kn_kq->kq_lock, MA_OWNED);            \
  163         else                                                            \
  164                 KQ_LOCK((kn)->kn_kq);                                   \
  165         (kn)->kn_status |= KN_ACTIVE;                                   \
  166         if (((kn)->kn_status & (KN_QUEUED | KN_DISABLED)) == 0)         \
  167                 knote_enqueue((kn));                                    \
  168         if (!(islock))                                                  \
  169                 KQ_UNLOCK((kn)->kn_kq);                                 \
  170 } while(0)
  171 #define KQ_LOCK(kq) do {                                                \
  172         mtx_lock(&(kq)->kq_lock);                                       \
  173 } while (0)
  174 #define KQ_FLUX_WAKEUP(kq) do {                                         \
  175         if (((kq)->kq_state & KQ_FLUXWAIT) == KQ_FLUXWAIT) {            \
  176                 (kq)->kq_state &= ~KQ_FLUXWAIT;                         \
  177                 wakeup((kq));                                           \
  178         }                                                               \
  179 } while (0)
  180 #define KQ_UNLOCK_FLUX(kq) do {                                         \
  181         KQ_FLUX_WAKEUP(kq);                                             \
  182         mtx_unlock(&(kq)->kq_lock);                                     \
  183 } while (0)
  184 #define KQ_UNLOCK(kq) do {                                              \
  185         mtx_unlock(&(kq)->kq_lock);                                     \
  186 } while (0)
  187 #define KQ_OWNED(kq) do {                                               \
  188         mtx_assert(&(kq)->kq_lock, MA_OWNED);                           \
  189 } while (0)
  190 #define KQ_NOTOWNED(kq) do {                                            \
  191         mtx_assert(&(kq)->kq_lock, MA_NOTOWNED);                        \
  192 } while (0)
  193 #define KN_LIST_LOCK(kn) do {                                           \
  194         if (kn->kn_knlist != NULL)                                      \
  195                 kn->kn_knlist->kl_lock(kn->kn_knlist->kl_lockarg);      \
  196 } while (0)
  197 #define KN_LIST_UNLOCK(kn) do {                                         \
  198         if (kn->kn_knlist != NULL)                                      \
  199                 kn->kn_knlist->kl_unlock(kn->kn_knlist->kl_lockarg);    \
  200 } while (0)
  201 #define KNL_ASSERT_LOCK(knl, islocked) do {                             \
  202         if (islocked)                                                   \
  203                 KNL_ASSERT_LOCKED(knl);                         \
  204         else                                                            \
  205                 KNL_ASSERT_UNLOCKED(knl);                               \
  206 } while (0)
  207 #ifdef INVARIANTS
  208 #define KNL_ASSERT_LOCKED(knl) do {                                     \
  209         if (knl->kl_locked != NULL &&                                   \
  210             !knl->kl_locked((knl)->kl_lockarg))                         \
  211                         panic("knlist not locked, but should be");      \
  212 } while (0)
  213 #define KNL_ASSERT_UNLOCKED(knl) do {                           \
  214         if (knl->kl_locked != NULL &&                                   \
  215             knl->kl_locked((knl)->kl_lockarg))                          \
  216                 panic("knlist locked, but should not be");              \
  217 } while (0)
  218 #else /* !INVARIANTS */
  219 #define KNL_ASSERT_LOCKED(knl) do {} while(0)
  220 #define KNL_ASSERT_UNLOCKED(knl) do {} while (0)
  221 #endif /* INVARIANTS */
  222 
  223 #define KN_HASHSIZE             64              /* XXX should be tunable */
  224 #define KN_HASH(val, mask)      (((val) ^ (val >> 8)) & (mask))
  225 
  226 static int
  227 filt_nullattach(struct knote *kn)
  228 {
  229 
  230         return (ENXIO);
  231 };
  232 
  233 struct filterops null_filtops =
  234         { 0, filt_nullattach, NULL, NULL };
  235 
  236 /* XXX - make SYSINIT to add these, and move into respective modules. */
  237 extern struct filterops sig_filtops;
  238 extern struct filterops fs_filtops;
  239 
  240 /*
  241  * Table for for all system-defined filters.
  242  */
  243 static struct mtx       filterops_lock;
  244 MTX_SYSINIT(kqueue_filterops, &filterops_lock, "protect sysfilt_ops",
  245         MTX_DEF);
  246 static struct {
  247         struct filterops *for_fop;
  248         int for_refcnt;
  249 } sysfilt_ops[EVFILT_SYSCOUNT] = {
  250         { &file_filtops },                      /* EVFILT_READ */
  251         { &file_filtops },                      /* EVFILT_WRITE */
  252         { &null_filtops },                      /* EVFILT_AIO */
  253         { &file_filtops },                      /* EVFILT_VNODE */
  254         { &proc_filtops },                      /* EVFILT_PROC */
  255         { &sig_filtops },                       /* EVFILT_SIGNAL */
  256         { &timer_filtops },                     /* EVFILT_TIMER */
  257         { &file_filtops },                      /* EVFILT_NETDEV */
  258         { &fs_filtops },                        /* EVFILT_FS */
  259         { &null_filtops },                      /* EVFILT_LIO */
  260 };
  261 
  262 /*
  263  * Simple redirection for all cdevsw style objects to call their fo_kqfilter
  264  * method.
  265  */
  266 static int
  267 filt_fileattach(struct knote *kn)
  268 {
  269 
  270         return (fo_kqfilter(kn->kn_fp, kn));
  271 }
  272 
  273 /*ARGSUSED*/
  274 static int
  275 kqueue_kqfilter(struct file *fp, struct knote *kn)
  276 {
  277         struct kqueue *kq = kn->kn_fp->f_data;
  278 
  279         if (kn->kn_filter != EVFILT_READ)
  280                 return (EINVAL);
  281 
  282         kn->kn_status |= KN_KQUEUE;
  283         kn->kn_fop = &kqread_filtops;
  284         knlist_add(&kq->kq_sel.si_note, kn, 0);
  285 
  286         return (0);
  287 }
  288 
  289 static void
  290 filt_kqdetach(struct knote *kn)
  291 {
  292         struct kqueue *kq = kn->kn_fp->f_data;
  293 
  294         knlist_remove(&kq->kq_sel.si_note, kn, 0);
  295 }
  296 
  297 /*ARGSUSED*/
  298 static int
  299 filt_kqueue(struct knote *kn, long hint)
  300 {
  301         struct kqueue *kq = kn->kn_fp->f_data;
  302 
  303         kn->kn_data = kq->kq_count;
  304         return (kn->kn_data > 0);
  305 }
  306 
  307 /* XXX - move to kern_proc.c?  */
  308 static int
  309 filt_procattach(struct knote *kn)
  310 {
  311         struct proc *p;
  312         int immediate;
  313         int error;
  314 
  315         immediate = 0;
  316         p = pfind(kn->kn_id);
  317         if (p == NULL && (kn->kn_sfflags & NOTE_EXIT)) {
  318                 p = zpfind(kn->kn_id);
  319                 immediate = 1;
  320         } else if (p != NULL && (p->p_flag & P_WEXIT)) {
  321                 immediate = 1;
  322         }
  323 
  324         if (p == NULL)
  325                 return (ESRCH);
  326         if ((error = p_cansee(curthread, p))) {
  327                 PROC_UNLOCK(p);
  328                 return (error);
  329         }
  330 
  331         kn->kn_ptr.p_proc = p;
  332         kn->kn_flags |= EV_CLEAR;               /* automatically set */
  333 
  334         /*
  335          * internal flag indicating registration done by kernel
  336          */
  337         if (kn->kn_flags & EV_FLAG1) {
  338                 kn->kn_data = kn->kn_sdata;             /* ppid */
  339                 kn->kn_fflags = NOTE_CHILD;
  340                 kn->kn_flags &= ~EV_FLAG1;
  341         }
  342 
  343         if (immediate == 0)
  344                 knlist_add(&p->p_klist, kn, 1);
  345 
  346         /*
  347          * Immediately activate any exit notes if the target process is a
  348          * zombie.  This is necessary to handle the case where the target
  349          * process, e.g. a child, dies before the kevent is registered.
  350          */
  351         if (immediate && filt_proc(kn, NOTE_EXIT))
  352                 KNOTE_ACTIVATE(kn, 0);
  353 
  354         PROC_UNLOCK(p);
  355 
  356         return (0);
  357 }
  358 
  359 /*
  360  * The knote may be attached to a different process, which may exit,
  361  * leaving nothing for the knote to be attached to.  So when the process
  362  * exits, the knote is marked as DETACHED and also flagged as ONESHOT so
  363  * it will be deleted when read out.  However, as part of the knote deletion,
  364  * this routine is called, so a check is needed to avoid actually performing
  365  * a detach, because the original process does not exist any more.
  366  */
  367 /* XXX - move to kern_proc.c?  */
  368 static void
  369 filt_procdetach(struct knote *kn)
  370 {
  371         struct proc *p;
  372 
  373         p = kn->kn_ptr.p_proc;
  374         knlist_remove(&p->p_klist, kn, 0);
  375         kn->kn_ptr.p_proc = NULL;
  376 }
  377 
  378 /* XXX - move to kern_proc.c?  */
  379 static int
  380 filt_proc(struct knote *kn, long hint)
  381 {
  382         struct proc *p = kn->kn_ptr.p_proc;
  383         u_int event;
  384 
  385         /*
  386          * mask off extra data
  387          */
  388         event = (u_int)hint & NOTE_PCTRLMASK;
  389 
  390         /*
  391          * if the user is interested in this event, record it.
  392          */
  393         if (kn->kn_sfflags & event)
  394                 kn->kn_fflags |= event;
  395 
  396         /*
  397          * process is gone, so flag the event as finished.
  398          */
  399         if (event == NOTE_EXIT) {
  400                 if (!(kn->kn_status & KN_DETACHED))
  401                         knlist_remove_inevent(&p->p_klist, kn);
  402                 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
  403                 kn->kn_data = p->p_xstat;
  404                 kn->kn_ptr.p_proc = NULL;
  405                 return (1);
  406         }
  407 
  408         return (kn->kn_fflags != 0);
  409 }
  410 
  411 /*
  412  * Called when the process forked. It mostly does the same as the
  413  * knote(), activating all knotes registered to be activated when the
  414  * process forked. Additionally, for each knote attached to the
  415  * parent, check whether user wants to track the new process. If so
  416  * attach a new knote to it, and immediately report an event with the
  417  * child's pid.
  418  */
  419 void
  420 knote_fork(struct knlist *list, int pid)
  421 {
  422         struct kqueue *kq;
  423         struct knote *kn;
  424         struct kevent kev;
  425         int error;
  426 
  427         if (list == NULL)
  428                 return;
  429         list->kl_lock(list->kl_lockarg);
  430 
  431         SLIST_FOREACH(kn, &list->kl_list, kn_selnext) {
  432                 if ((kn->kn_status & KN_INFLUX) == KN_INFLUX)
  433                         continue;
  434                 kq = kn->kn_kq;
  435                 KQ_LOCK(kq);
  436                 if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
  437                         KQ_UNLOCK(kq);
  438                         continue;
  439                 }
  440 
  441                 /*
  442                  * The same as knote(), activate the event.
  443                  */
  444                 if ((kn->kn_sfflags & NOTE_TRACK) == 0) {
  445                         kn->kn_status |= KN_HASKQLOCK;
  446                         if (kn->kn_fop->f_event(kn, NOTE_FORK | pid))
  447                                 KNOTE_ACTIVATE(kn, 1);
  448                         kn->kn_status &= ~KN_HASKQLOCK;
  449                         KQ_UNLOCK(kq);
  450                         continue;
  451                 }
  452 
  453                 /*
  454                  * The NOTE_TRACK case. In addition to the activation
  455                  * of the event, we need to register new event to
  456                  * track the child. Drop the locks in preparation for
  457                  * the call to kqueue_register().
  458                  */
  459                 kn->kn_status |= KN_INFLUX;
  460                 KQ_UNLOCK(kq);
  461                 list->kl_unlock(list->kl_lockarg);
  462 
  463                 /*
  464                  * Activate existing knote and register a knote with
  465                  * new process.
  466                  */
  467                 kev.ident = pid;
  468                 kev.filter = kn->kn_filter;
  469                 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1;
  470                 kev.fflags = kn->kn_sfflags;
  471                 kev.data = kn->kn_id;           /* parent */
  472                 kev.udata = kn->kn_kevent.udata;/* preserve udata */
  473                 error = kqueue_register(kq, &kev, NULL, 0);
  474                 if (kn->kn_fop->f_event(kn, NOTE_FORK | pid))
  475                         KNOTE_ACTIVATE(kn, 0);
  476                 if (error)
  477                         kn->kn_fflags |= NOTE_TRACKERR;
  478                 KQ_LOCK(kq);
  479                 kn->kn_status &= ~KN_INFLUX;
  480                 KQ_UNLOCK_FLUX(kq);
  481                 list->kl_lock(list->kl_lockarg);
  482         }
  483         list->kl_unlock(list->kl_lockarg);
  484 }
  485 
  486 static int
  487 timertoticks(intptr_t data)
  488 {
  489         struct timeval tv;
  490         int tticks;
  491 
  492         tv.tv_sec = data / 1000;
  493         tv.tv_usec = (data % 1000) * 1000;
  494         tticks = tvtohz(&tv);
  495 
  496         return tticks;
  497 }
  498 
  499 /* XXX - move to kern_timeout.c? */
  500 static void
  501 filt_timerexpire(void *knx)
  502 {
  503         struct knote *kn = knx;
  504         struct callout *calloutp;
  505 
  506         kn->kn_data++;
  507         KNOTE_ACTIVATE(kn, 0);  /* XXX - handle locking */
  508 
  509         if ((kn->kn_flags & EV_ONESHOT) != EV_ONESHOT) {
  510                 calloutp = (struct callout *)kn->kn_hook;
  511                 callout_reset(calloutp, timertoticks(kn->kn_sdata),
  512                     filt_timerexpire, kn);
  513         }
  514 }
  515 
  516 /*
  517  * data contains amount of time to sleep, in milliseconds
  518  */
  519 /* XXX - move to kern_timeout.c? */
  520 static int
  521 filt_timerattach(struct knote *kn)
  522 {
  523         struct callout *calloutp;
  524 
  525         atomic_add_int(&kq_ncallouts, 1);
  526 
  527         if (kq_ncallouts >= kq_calloutmax) {
  528                 atomic_add_int(&kq_ncallouts, -1);
  529                 return (ENOMEM);
  530         }
  531 
  532         kn->kn_flags |= EV_CLEAR;               /* automatically set */
  533         kn->kn_status &= ~KN_DETACHED;          /* knlist_add usually sets it */
  534         MALLOC(calloutp, struct callout *, sizeof(*calloutp),
  535             M_KQUEUE, M_WAITOK);
  536         callout_init(calloutp, CALLOUT_MPSAFE);
  537         kn->kn_hook = calloutp;
  538         callout_reset(calloutp, timertoticks(kn->kn_sdata), filt_timerexpire,
  539             kn);
  540 
  541         return (0);
  542 }
  543 
  544 /* XXX - move to kern_timeout.c? */
  545 static void
  546 filt_timerdetach(struct knote *kn)
  547 {
  548         struct callout *calloutp;
  549 
  550         calloutp = (struct callout *)kn->kn_hook;
  551         callout_drain(calloutp);
  552         FREE(calloutp, M_KQUEUE);
  553         atomic_add_int(&kq_ncallouts, -1);
  554         kn->kn_status |= KN_DETACHED;   /* knlist_remove usually clears it */
  555 }
  556 
  557 /* XXX - move to kern_timeout.c? */
  558 static int
  559 filt_timer(struct knote *kn, long hint)
  560 {
  561 
  562         return (kn->kn_data != 0);
  563 }
  564 
  565 int
  566 kqueue(struct thread *td, struct kqueue_args *uap)
  567 {
  568         struct filedesc *fdp;
  569         struct kqueue *kq;
  570         struct file *fp;
  571         int fd, error;
  572 
  573         fdp = td->td_proc->p_fd;
  574         error = falloc(td, &fp, &fd);
  575         if (error)
  576                 goto done2;
  577 
  578         /* An extra reference on `nfp' has been held for us by falloc(). */
  579         kq = malloc(sizeof *kq, M_KQUEUE, M_WAITOK | M_ZERO);
  580         mtx_init(&kq->kq_lock, "kqueue", NULL, MTX_DEF|MTX_DUPOK);
  581         TAILQ_INIT(&kq->kq_head);
  582         kq->kq_fdp = fdp;
  583         knlist_init_mtx(&kq->kq_sel.si_note, &kq->kq_lock);
  584         TASK_INIT(&kq->kq_task, 0, kqueue_task, kq);
  585 
  586         FILEDESC_XLOCK(fdp);
  587         SLIST_INSERT_HEAD(&fdp->fd_kqlist, kq, kq_list);
  588         FILEDESC_XUNLOCK(fdp);
  589 
  590         FILE_LOCK(fp);
  591         fp->f_flag = FREAD | FWRITE;
  592         fp->f_type = DTYPE_KQUEUE;
  593         fp->f_data = kq;
  594         fp->f_ops = &kqueueops;
  595         FILE_UNLOCK(fp);
  596         fdrop(fp, td);
  597 
  598         td->td_retval[0] = fd;
  599 done2:
  600         return (error);
  601 }
  602 
  603 #ifndef _SYS_SYSPROTO_H_
  604 struct kevent_args {
  605         int     fd;
  606         const struct kevent *changelist;
  607         int     nchanges;
  608         struct  kevent *eventlist;
  609         int     nevents;
  610         const struct timespec *timeout;
  611 };
  612 #endif
  613 int
  614 kevent(struct thread *td, struct kevent_args *uap)
  615 {
  616         struct timespec ts, *tsp;
  617         struct kevent_copyops k_ops = { uap,
  618                                         kevent_copyout,
  619                                         kevent_copyin};
  620         int error;
  621 #ifdef KTRACE
  622         struct uio ktruio;
  623         struct iovec ktriov;
  624         struct uio *ktruioin = NULL;
  625         struct uio *ktruioout = NULL;
  626 #endif
  627 
  628         if (uap->timeout != NULL) {
  629                 error = copyin(uap->timeout, &ts, sizeof(ts));
  630                 if (error)
  631                         return (error);
  632                 tsp = &ts;
  633         } else
  634                 tsp = NULL;
  635 
  636 #ifdef KTRACE
  637         if (KTRPOINT(td, KTR_GENIO)) {
  638                 ktriov.iov_base = uap->changelist;
  639                 ktriov.iov_len = uap->nchanges * sizeof(struct kevent);
  640                 ktruio = (struct uio){ .uio_iov = &ktriov, .uio_iovcnt = 1,
  641                     .uio_segflg = UIO_USERSPACE, .uio_rw = UIO_READ,
  642                     .uio_td = td };
  643                 ktruioin = cloneuio(&ktruio);
  644                 ktriov.iov_base = uap->eventlist;
  645                 ktriov.iov_len = uap->nevents * sizeof(struct kevent);
  646                 ktruioout = cloneuio(&ktruio);
  647         }
  648 #endif
  649 
  650         error = kern_kevent(td, uap->fd, uap->nchanges, uap->nevents,
  651             &k_ops, tsp);
  652 
  653 #ifdef KTRACE
  654         if (ktruioin != NULL) {
  655                 ktruioin->uio_resid = uap->nchanges * sizeof(struct kevent);
  656                 ktrgenio(uap->fd, UIO_WRITE, ktruioin, 0);
  657                 ktruioout->uio_resid = td->td_retval[0] * sizeof(struct kevent);
  658                 ktrgenio(uap->fd, UIO_READ, ktruioout, error);
  659         }
  660 #endif
  661 
  662         return (error);
  663 }
  664 
  665 /*
  666  * Copy 'count' items into the destination list pointed to by uap->eventlist.
  667  */
  668 static int
  669 kevent_copyout(void *arg, struct kevent *kevp, int count)
  670 {
  671         struct kevent_args *uap;
  672         int error;
  673 
  674         KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
  675         uap = (struct kevent_args *)arg;
  676 
  677         error = copyout(kevp, uap->eventlist, count * sizeof *kevp);
  678         if (error == 0)
  679                 uap->eventlist += count;
  680         return (error);
  681 }
  682 
  683 /*
  684  * Copy 'count' items from the list pointed to by uap->changelist.
  685  */
  686 static int
  687 kevent_copyin(void *arg, struct kevent *kevp, int count)
  688 {
  689         struct kevent_args *uap;
  690         int error;
  691 
  692         KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
  693         uap = (struct kevent_args *)arg;
  694 
  695         error = copyin(uap->changelist, kevp, count * sizeof *kevp);
  696         if (error == 0)
  697                 uap->changelist += count;
  698         return (error);
  699 }
  700 
  701 int
  702 kern_kevent(struct thread *td, int fd, int nchanges, int nevents,
  703     struct kevent_copyops *k_ops, const struct timespec *timeout)
  704 {
  705         struct kevent keva[KQ_NEVENTS];
  706         struct kevent *kevp, *changes;
  707         struct kqueue *kq;
  708         struct file *fp;
  709         int i, n, nerrors, error;
  710 
  711         if ((error = fget(td, fd, &fp)) != 0)
  712                 return (error);
  713         if ((error = kqueue_acquire(fp, &kq)) != 0)
  714                 goto done_norel;
  715 
  716         nerrors = 0;
  717 
  718         while (nchanges > 0) {
  719                 n = nchanges > KQ_NEVENTS ? KQ_NEVENTS : nchanges;
  720                 error = k_ops->k_copyin(k_ops->arg, keva, n);
  721                 if (error)
  722                         goto done;
  723                 changes = keva;
  724                 for (i = 0; i < n; i++) {
  725                         kevp = &changes[i];
  726                         if (!kevp->filter)
  727                                 continue;
  728                         kevp->flags &= ~EV_SYSFLAGS;
  729                         error = kqueue_register(kq, kevp, td, 1);
  730                         if (error) {
  731                                 if (nevents != 0) {
  732                                         kevp->flags = EV_ERROR;
  733                                         kevp->data = error;
  734                                         (void) k_ops->k_copyout(k_ops->arg,
  735                                             kevp, 1);
  736                                         nevents--;
  737                                         nerrors++;
  738                                 } else {
  739                                         goto done;
  740                                 }
  741                         }
  742                 }
  743                 nchanges -= n;
  744         }
  745         if (nerrors) {
  746                 td->td_retval[0] = nerrors;
  747                 error = 0;
  748                 goto done;
  749         }
  750 
  751         error = kqueue_scan(kq, nevents, k_ops, timeout, keva, td);
  752 done:
  753         kqueue_release(kq, 0);
  754 done_norel:
  755         fdrop(fp, td);
  756         return (error);
  757 }
  758 
  759 int
  760 kqueue_add_filteropts(int filt, struct filterops *filtops)
  761 {
  762         int error;
  763 
  764         error = 0;
  765         if (filt > 0 || filt + EVFILT_SYSCOUNT < 0) {
  766                 printf(
  767 "trying to add a filterop that is out of range: %d is beyond %d\n",
  768                     ~filt, EVFILT_SYSCOUNT);
  769                 return EINVAL;
  770         }
  771         mtx_lock(&filterops_lock);
  772         if (sysfilt_ops[~filt].for_fop != &null_filtops &&
  773             sysfilt_ops[~filt].for_fop != NULL)
  774                 error = EEXIST;
  775         else {
  776                 sysfilt_ops[~filt].for_fop = filtops;
  777                 sysfilt_ops[~filt].for_refcnt = 0;
  778         }
  779         mtx_unlock(&filterops_lock);
  780 
  781         return (error);
  782 }
  783 
  784 int
  785 kqueue_del_filteropts(int filt)
  786 {
  787         int error;
  788 
  789         error = 0;
  790         if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
  791                 return EINVAL;
  792 
  793         mtx_lock(&filterops_lock);
  794         if (sysfilt_ops[~filt].for_fop == &null_filtops ||
  795             sysfilt_ops[~filt].for_fop == NULL)
  796                 error = EINVAL;
  797         else if (sysfilt_ops[~filt].for_refcnt != 0)
  798                 error = EBUSY;
  799         else {
  800                 sysfilt_ops[~filt].for_fop = &null_filtops;
  801                 sysfilt_ops[~filt].for_refcnt = 0;
  802         }
  803         mtx_unlock(&filterops_lock);
  804 
  805         return error;
  806 }
  807 
  808 static struct filterops *
  809 kqueue_fo_find(int filt)
  810 {
  811 
  812         if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
  813                 return NULL;
  814 
  815         mtx_lock(&filterops_lock);
  816         sysfilt_ops[~filt].for_refcnt++;
  817         if (sysfilt_ops[~filt].for_fop == NULL)
  818                 sysfilt_ops[~filt].for_fop = &null_filtops;
  819         mtx_unlock(&filterops_lock);
  820 
  821         return sysfilt_ops[~filt].for_fop;
  822 }
  823 
  824 static void
  825 kqueue_fo_release(int filt)
  826 {
  827 
  828         if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
  829                 return;
  830 
  831         mtx_lock(&filterops_lock);
  832         KASSERT(sysfilt_ops[~filt].for_refcnt > 0,
  833             ("filter object refcount not valid on release"));
  834         sysfilt_ops[~filt].for_refcnt--;
  835         mtx_unlock(&filterops_lock);
  836 }
  837 
  838 /*
  839  * A ref to kq (obtained via kqueue_acquire) must be held.  waitok will
  840  * influence if memory allocation should wait.  Make sure it is 0 if you
  841  * hold any mutexes.
  842  */
  843 static int
  844 kqueue_register(struct kqueue *kq, struct kevent *kev, struct thread *td, int waitok)
  845 {
  846         struct filterops *fops;
  847         struct file *fp;
  848         struct knote *kn, *tkn;
  849         int error, filt, event;
  850         int haskqglobal;
  851 
  852         fp = NULL;
  853         kn = NULL;
  854         error = 0;
  855         haskqglobal = 0;
  856 
  857         filt = kev->filter;
  858         fops = kqueue_fo_find(filt);
  859         if (fops == NULL)
  860                 return EINVAL;
  861 
  862         tkn = knote_alloc(waitok);              /* prevent waiting with locks */
  863 
  864 findkn:
  865         if (fops->f_isfd) {
  866                 KASSERT(td != NULL, ("td is NULL"));
  867                 error = fget(td, kev->ident, &fp);
  868                 if (error)
  869                         goto done;
  870 
  871                 if ((kev->flags & EV_ADD) == EV_ADD && kqueue_expand(kq, fops,
  872                     kev->ident, 0) != 0) {
  873                         /* try again */
  874                         fdrop(fp, td);
  875                         fp = NULL;
  876                         error = kqueue_expand(kq, fops, kev->ident, waitok);
  877                         if (error)
  878                                 goto done;
  879                         goto findkn;
  880                 }
  881 
  882                 if (fp->f_type == DTYPE_KQUEUE) {
  883                         /*
  884                          * if we add some inteligence about what we are doing,
  885                          * we should be able to support events on ourselves.
  886                          * We need to know when we are doing this to prevent
  887                          * getting both the knlist lock and the kq lock since
  888                          * they are the same thing.
  889                          */
  890                         if (fp->f_data == kq) {
  891                                 error = EINVAL;
  892                                 goto done;
  893                         }
  894 
  895                         KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
  896                 }
  897 
  898                 KQ_LOCK(kq);
  899                 if (kev->ident < kq->kq_knlistsize) {
  900                         SLIST_FOREACH(kn, &kq->kq_knlist[kev->ident], kn_link)
  901                                 if (kev->filter == kn->kn_filter)
  902                                         break;
  903                 }
  904         } else {
  905                 if ((kev->flags & EV_ADD) == EV_ADD)
  906                         kqueue_expand(kq, fops, kev->ident, waitok);
  907 
  908                 KQ_LOCK(kq);
  909                 if (kq->kq_knhashmask != 0) {
  910                         struct klist *list;
  911 
  912                         list = &kq->kq_knhash[
  913                             KN_HASH((u_long)kev->ident, kq->kq_knhashmask)];
  914                         SLIST_FOREACH(kn, list, kn_link)
  915                                 if (kev->ident == kn->kn_id &&
  916                                     kev->filter == kn->kn_filter)
  917                                         break;
  918                 }
  919         }
  920 
  921         /* knote is in the process of changing, wait for it to stablize. */
  922         if (kn != NULL && (kn->kn_status & KN_INFLUX) == KN_INFLUX) {
  923                 if (fp != NULL) {
  924                         fdrop(fp, td);
  925                         fp = NULL;
  926                 }
  927                 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
  928                 kq->kq_state |= KQ_FLUXWAIT;
  929                 msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqflxwt", 0);
  930                 goto findkn;
  931         }
  932 
  933         if (kn == NULL && ((kev->flags & EV_ADD) == 0)) {
  934                 KQ_UNLOCK(kq);
  935                 error = ENOENT;
  936                 goto done;
  937         }
  938 
  939         /*
  940          * kn now contains the matching knote, or NULL if no match
  941          */
  942         if (kev->flags & EV_ADD) {
  943                 if (kn == NULL) {
  944                         kn = tkn;
  945                         tkn = NULL;
  946                         if (kn == NULL) {
  947                                 KQ_UNLOCK(kq);
  948                                 error = ENOMEM;
  949                                 goto done;
  950                         }
  951                         kn->kn_fp = fp;
  952                         kn->kn_kq = kq;
  953                         kn->kn_fop = fops;
  954                         /*
  955                          * apply reference counts to knote structure, and
  956                          * do not release it at the end of this routine.
  957                          */
  958                         fops = NULL;
  959                         fp = NULL;
  960 
  961                         kn->kn_sfflags = kev->fflags;
  962                         kn->kn_sdata = kev->data;
  963                         kev->fflags = 0;
  964                         kev->data = 0;
  965                         kn->kn_kevent = *kev;
  966                         kn->kn_kevent.flags &= ~(EV_ADD | EV_DELETE |
  967                             EV_ENABLE | EV_DISABLE);
  968                         kn->kn_status = KN_INFLUX|KN_DETACHED;
  969 
  970                         error = knote_attach(kn, kq);
  971                         KQ_UNLOCK(kq);
  972                         if (error != 0) {
  973                                 tkn = kn;
  974                                 goto done;
  975                         }
  976 
  977                         if ((error = kn->kn_fop->f_attach(kn)) != 0) {
  978                                 knote_drop(kn, td);
  979                                 goto done;
  980                         }
  981                         KN_LIST_LOCK(kn);
  982                 } else {
  983                         /*
  984                          * The user may change some filter values after the
  985                          * initial EV_ADD, but doing so will not reset any
  986                          * filter which has already been triggered.
  987                          */
  988                         kn->kn_status |= KN_INFLUX;
  989                         KQ_UNLOCK(kq);
  990                         KN_LIST_LOCK(kn);
  991                         kn->kn_sfflags = kev->fflags;
  992                         kn->kn_sdata = kev->data;
  993                         kn->kn_kevent.udata = kev->udata;
  994                 }
  995 
  996                 /*
  997                  * We can get here with kn->kn_knlist == NULL.
  998                  * This can happen when the initial attach event decides that
  999                  * the event is "completed" already.  i.e. filt_procattach
 1000                  * is called on a zombie process.  It will call filt_proc
 1001                  * which will remove it from the list, and NULL kn_knlist.
 1002                  */
 1003                 event = kn->kn_fop->f_event(kn, 0);
 1004                 KQ_LOCK(kq);
 1005                 if (event)
 1006                         KNOTE_ACTIVATE(kn, 1);
 1007                 kn->kn_status &= ~KN_INFLUX;
 1008                 KN_LIST_UNLOCK(kn);
 1009         } else if (kev->flags & EV_DELETE) {
 1010                 kn->kn_status |= KN_INFLUX;
 1011                 KQ_UNLOCK(kq);
 1012                 if (!(kn->kn_status & KN_DETACHED))
 1013                         kn->kn_fop->f_detach(kn);
 1014                 knote_drop(kn, td);
 1015                 goto done;
 1016         }
 1017 
 1018         if ((kev->flags & EV_DISABLE) &&
 1019             ((kn->kn_status & KN_DISABLED) == 0)) {
 1020                 kn->kn_status |= KN_DISABLED;
 1021         }
 1022 
 1023         if ((kev->flags & EV_ENABLE) && (kn->kn_status & KN_DISABLED)) {
 1024                 kn->kn_status &= ~KN_DISABLED;
 1025                 if ((kn->kn_status & KN_ACTIVE) &&
 1026                     ((kn->kn_status & KN_QUEUED) == 0))
 1027                         knote_enqueue(kn);
 1028         }
 1029         KQ_UNLOCK_FLUX(kq);
 1030 
 1031 done:
 1032         KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
 1033         if (fp != NULL)
 1034                 fdrop(fp, td);
 1035         if (tkn != NULL)
 1036                 knote_free(tkn);
 1037         if (fops != NULL)
 1038                 kqueue_fo_release(filt);
 1039         return (error);
 1040 }
 1041 
 1042 static int
 1043 kqueue_acquire(struct file *fp, struct kqueue **kqp)
 1044 {
 1045         int error;
 1046         struct kqueue *kq;
 1047 
 1048         error = 0;
 1049 
 1050         FILE_LOCK(fp);
 1051         do {
 1052                 kq = fp->f_data;
 1053                 if (fp->f_type != DTYPE_KQUEUE || kq == NULL) {
 1054                         error = EBADF;
 1055                         break;
 1056                 }
 1057                 *kqp = kq;
 1058                 KQ_LOCK(kq);
 1059                 if ((kq->kq_state & KQ_CLOSING) == KQ_CLOSING) {
 1060                         KQ_UNLOCK(kq);
 1061                         error = EBADF;
 1062                         break;
 1063                 }
 1064                 kq->kq_refcnt++;
 1065                 KQ_UNLOCK(kq);
 1066         } while (0);
 1067         FILE_UNLOCK(fp);
 1068 
 1069         return error;
 1070 }
 1071 
 1072 static void
 1073 kqueue_release(struct kqueue *kq, int locked)
 1074 {
 1075         if (locked)
 1076                 KQ_OWNED(kq);
 1077         else
 1078                 KQ_LOCK(kq);
 1079         kq->kq_refcnt--;
 1080         if (kq->kq_refcnt == 1)
 1081                 wakeup(&kq->kq_refcnt);
 1082         if (!locked)
 1083                 KQ_UNLOCK(kq);
 1084 }
 1085 
 1086 static void
 1087 kqueue_schedtask(struct kqueue *kq)
 1088 {
 1089 
 1090         KQ_OWNED(kq);
 1091         KASSERT(((kq->kq_state & KQ_TASKDRAIN) != KQ_TASKDRAIN),
 1092             ("scheduling kqueue task while draining"));
 1093 
 1094         if ((kq->kq_state & KQ_TASKSCHED) != KQ_TASKSCHED) {
 1095                 taskqueue_enqueue(taskqueue_kqueue, &kq->kq_task);
 1096                 kq->kq_state |= KQ_TASKSCHED;
 1097         }
 1098 }
 1099 
 1100 /*
 1101  * Expand the kq to make sure we have storage for fops/ident pair.
 1102  *
 1103  * Return 0 on success (or no work necessary), return errno on failure.
 1104  *
 1105  * Not calling hashinit w/ waitok (proper malloc flag) should be safe.
 1106  * If kqueue_register is called from a non-fd context, there usually/should
 1107  * be no locks held.
 1108  */
 1109 static int
 1110 kqueue_expand(struct kqueue *kq, struct filterops *fops, uintptr_t ident,
 1111         int waitok)
 1112 {
 1113         struct klist *list, *tmp_knhash, *to_free;
 1114         u_long tmp_knhashmask;
 1115         int size;
 1116         int fd;
 1117         int mflag = waitok ? M_WAITOK : M_NOWAIT;
 1118 
 1119         KQ_NOTOWNED(kq);
 1120 
 1121         to_free = NULL;
 1122         if (fops->f_isfd) {
 1123                 fd = ident;
 1124                 if (kq->kq_knlistsize <= fd) {
 1125                         size = kq->kq_knlistsize;
 1126                         while (size <= fd)
 1127                                 size += KQEXTENT;
 1128                         MALLOC(list, struct klist *,
 1129                             size * sizeof list, M_KQUEUE, mflag);
 1130                         if (list == NULL)
 1131                                 return ENOMEM;
 1132                         KQ_LOCK(kq);
 1133                         if (kq->kq_knlistsize > fd) {
 1134                                 to_free = list;
 1135                                 list = NULL;
 1136                         } else {
 1137                                 if (kq->kq_knlist != NULL) {
 1138                                         bcopy(kq->kq_knlist, list,
 1139                                             kq->kq_knlistsize * sizeof list);
 1140                                         to_free = kq->kq_knlist;
 1141                                         kq->kq_knlist = NULL;
 1142                                 }
 1143                                 bzero((caddr_t)list +
 1144                                     kq->kq_knlistsize * sizeof list,
 1145                                     (size - kq->kq_knlistsize) * sizeof list);
 1146                                 kq->kq_knlistsize = size;
 1147                                 kq->kq_knlist = list;
 1148                         }
 1149                         KQ_UNLOCK(kq);
 1150                 }
 1151         } else {
 1152                 if (kq->kq_knhashmask == 0) {
 1153                         tmp_knhash = hashinit(KN_HASHSIZE, M_KQUEUE,
 1154                             &tmp_knhashmask);
 1155                         if (tmp_knhash == NULL)
 1156                                 return ENOMEM;
 1157                         KQ_LOCK(kq);
 1158                         if (kq->kq_knhashmask == 0) {
 1159                                 kq->kq_knhash = tmp_knhash;
 1160                                 kq->kq_knhashmask = tmp_knhashmask;
 1161                         } else {
 1162                                 to_free = tmp_knhash;
 1163                         }
 1164                         KQ_UNLOCK(kq);
 1165                 }
 1166         }
 1167         free(to_free, M_KQUEUE);
 1168 
 1169         KQ_NOTOWNED(kq);
 1170         return 0;
 1171 }
 1172 
 1173 static void
 1174 kqueue_task(void *arg, int pending)
 1175 {
 1176         struct kqueue *kq;
 1177         int haskqglobal;
 1178 
 1179         haskqglobal = 0;
 1180         kq = arg;
 1181 
 1182         KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
 1183         KQ_LOCK(kq);
 1184 
 1185         KNOTE_LOCKED(&kq->kq_sel.si_note, 0);
 1186 
 1187         kq->kq_state &= ~KQ_TASKSCHED;
 1188         if ((kq->kq_state & KQ_TASKDRAIN) == KQ_TASKDRAIN) {
 1189                 wakeup(&kq->kq_state);
 1190         }
 1191         KQ_UNLOCK(kq);
 1192         KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
 1193 }
 1194 
 1195 /*
 1196  * Scan, update kn_data (if not ONESHOT), and copyout triggered events.
 1197  * We treat KN_MARKER knotes as if they are INFLUX.
 1198  */
 1199 static int
 1200 kqueue_scan(struct kqueue *kq, int maxevents, struct kevent_copyops *k_ops,
 1201     const struct timespec *tsp, struct kevent *keva, struct thread *td)
 1202 {
 1203         struct kevent *kevp;
 1204         struct timeval atv, rtv, ttv;
 1205         struct knote *kn, *marker;
 1206         int count, timeout, nkev, error, influx;
 1207         int haskqglobal;
 1208 
 1209         count = maxevents;
 1210         nkev = 0;
 1211         error = 0;
 1212         haskqglobal = 0;
 1213 
 1214         if (maxevents == 0)
 1215                 goto done_nl;
 1216 
 1217         if (tsp != NULL) {
 1218                 TIMESPEC_TO_TIMEVAL(&atv, tsp);
 1219                 if (itimerfix(&atv)) {
 1220                         error = EINVAL;
 1221                         goto done_nl;
 1222                 }
 1223                 if (tsp->tv_sec == 0 && tsp->tv_nsec == 0)
 1224                         timeout = -1;
 1225                 else
 1226                         timeout = atv.tv_sec > 24 * 60 * 60 ?
 1227                             24 * 60 * 60 * hz : tvtohz(&atv);
 1228                 getmicrouptime(&rtv);
 1229                 timevaladd(&atv, &rtv);
 1230         } else {
 1231                 atv.tv_sec = 0;
 1232                 atv.tv_usec = 0;
 1233                 timeout = 0;
 1234         }
 1235         marker = knote_alloc(1);
 1236         if (marker == NULL) {
 1237                 error = ENOMEM;
 1238                 goto done_nl;
 1239         }
 1240         marker->kn_status = KN_MARKER;
 1241         KQ_LOCK(kq);
 1242         goto start;
 1243 
 1244 retry:
 1245         if (atv.tv_sec || atv.tv_usec) {
 1246                 getmicrouptime(&rtv);
 1247                 if (timevalcmp(&rtv, &atv, >=))
 1248                         goto done;
 1249                 ttv = atv;
 1250                 timevalsub(&ttv, &rtv);
 1251                 timeout = ttv.tv_sec > 24 * 60 * 60 ?
 1252                         24 * 60 * 60 * hz : tvtohz(&ttv);
 1253         }
 1254 
 1255 start:
 1256         kevp = keva;
 1257         if (kq->kq_count == 0) {
 1258                 if (timeout < 0) {
 1259                         error = EWOULDBLOCK;
 1260                 } else {
 1261                         kq->kq_state |= KQ_SLEEP;
 1262                         error = msleep(kq, &kq->kq_lock, PSOCK | PCATCH,
 1263                             "kqread", timeout);
 1264                 }
 1265                 if (error == 0)
 1266                         goto retry;
 1267                 /* don't restart after signals... */
 1268                 if (error == ERESTART)
 1269                         error = EINTR;
 1270                 else if (error == EWOULDBLOCK)
 1271                         error = 0;
 1272                 goto done;
 1273         }
 1274 
 1275         TAILQ_INSERT_TAIL(&kq->kq_head, marker, kn_tqe);
 1276         influx = 0;
 1277         while (count) {
 1278                 KQ_OWNED(kq);
 1279                 kn = TAILQ_FIRST(&kq->kq_head);
 1280 
 1281                 if ((kn->kn_status == KN_MARKER && kn != marker) ||
 1282                     (kn->kn_status & KN_INFLUX) == KN_INFLUX) {
 1283                         if (influx) {
 1284                                 influx = 0;
 1285                                 KQ_FLUX_WAKEUP(kq);
 1286                         }
 1287                         kq->kq_state |= KQ_FLUXWAIT;
 1288                         error = msleep(kq, &kq->kq_lock, PSOCK,
 1289                             "kqflxwt", 0);
 1290                         continue;
 1291                 }
 1292 
 1293                 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
 1294                 if ((kn->kn_status & KN_DISABLED) == KN_DISABLED) {
 1295                         kn->kn_status &= ~KN_QUEUED;
 1296                         kq->kq_count--;
 1297                         continue;
 1298                 }
 1299                 if (kn == marker) {
 1300                         KQ_FLUX_WAKEUP(kq);
 1301                         if (count == maxevents)
 1302                                 goto retry;
 1303                         goto done;
 1304                 }
 1305                 KASSERT((kn->kn_status & KN_INFLUX) == 0,
 1306                     ("KN_INFLUX set when not suppose to be"));
 1307 
 1308                 if ((kn->kn_flags & EV_ONESHOT) == EV_ONESHOT) {
 1309                         kn->kn_status &= ~KN_QUEUED;
 1310                         kn->kn_status |= KN_INFLUX;
 1311                         kq->kq_count--;
 1312                         KQ_UNLOCK(kq);
 1313                         /*
 1314                          * We don't need to lock the list since we've marked
 1315                          * it _INFLUX.
 1316                          */
 1317                         *kevp = kn->kn_kevent;
 1318                         if (!(kn->kn_status & KN_DETACHED))
 1319                                 kn->kn_fop->f_detach(kn);
 1320                         knote_drop(kn, td);
 1321                         KQ_LOCK(kq);
 1322                         kn = NULL;
 1323                 } else {
 1324                         kn->kn_status |= KN_INFLUX;
 1325                         KQ_UNLOCK(kq);
 1326                         if ((kn->kn_status & KN_KQUEUE) == KN_KQUEUE)
 1327                                 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
 1328                         KN_LIST_LOCK(kn);
 1329                         if (kn->kn_fop->f_event(kn, 0) == 0) {
 1330                                 KQ_LOCK(kq);
 1331                                 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
 1332                                 kn->kn_status &=
 1333                                     ~(KN_QUEUED | KN_ACTIVE | KN_INFLUX);
 1334                                 kq->kq_count--;
 1335                                 KN_LIST_UNLOCK(kn);
 1336                                 influx = 1;
 1337                                 continue;
 1338                         }
 1339                         *kevp = kn->kn_kevent;
 1340                         KQ_LOCK(kq);
 1341                         KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
 1342                         if (kn->kn_flags & EV_CLEAR) {
 1343                                 kn->kn_data = 0;
 1344                                 kn->kn_fflags = 0;
 1345                                 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE);
 1346                                 kq->kq_count--;
 1347                         } else
 1348                                 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
 1349                         
 1350                         kn->kn_status &= ~(KN_INFLUX);
 1351                         KN_LIST_UNLOCK(kn);
 1352                         influx = 1;
 1353                 }
 1354 
 1355                 /* we are returning a copy to the user */
 1356                 kevp++;
 1357                 nkev++;
 1358                 count--;
 1359 
 1360                 if (nkev == KQ_NEVENTS) {
 1361                         influx = 0;
 1362                         KQ_UNLOCK_FLUX(kq);
 1363                         error = k_ops->k_copyout(k_ops->arg, keva, nkev);
 1364                         nkev = 0;
 1365                         kevp = keva;
 1366                         KQ_LOCK(kq);
 1367                         if (error)
 1368                                 break;
 1369                 }
 1370         }
 1371         TAILQ_REMOVE(&kq->kq_head, marker, kn_tqe);
 1372 done:
 1373         KQ_OWNED(kq);
 1374         KQ_UNLOCK_FLUX(kq);
 1375         knote_free(marker);
 1376 done_nl:
 1377         KQ_NOTOWNED(kq);
 1378         if (nkev != 0)
 1379                 error = k_ops->k_copyout(k_ops->arg, keva, nkev);
 1380         td->td_retval[0] = maxevents - count;
 1381         return (error);
 1382 }
 1383 
 1384 /*
 1385  * XXX
 1386  * This could be expanded to call kqueue_scan, if desired.
 1387  */
 1388 /*ARGSUSED*/
 1389 static int
 1390 kqueue_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
 1391         int flags, struct thread *td)
 1392 {
 1393         return (ENXIO);
 1394 }
 1395 
 1396 /*ARGSUSED*/
 1397 static int
 1398 kqueue_write(struct file *fp, struct uio *uio, struct ucred *active_cred,
 1399          int flags, struct thread *td)
 1400 {
 1401         return (ENXIO);
 1402 }
 1403 
 1404 /*ARGSUSED*/
 1405 static int
 1406 kqueue_ioctl(struct file *fp, u_long cmd, void *data,
 1407         struct ucred *active_cred, struct thread *td)
 1408 {
 1409         /*
 1410          * Enabling sigio causes two major problems:
 1411          * 1) infinite recursion:
 1412          * Synopsys: kevent is being used to track signals and have FIOASYNC
 1413          * set.  On receipt of a signal this will cause a kqueue to recurse
 1414          * into itself over and over.  Sending the sigio causes the kqueue
 1415          * to become ready, which in turn posts sigio again, forever.
 1416          * Solution: this can be solved by setting a flag in the kqueue that
 1417          * we have a SIGIO in progress.
 1418          * 2) locking problems:
 1419          * Synopsys: Kqueue is a leaf subsystem, but adding signalling puts
 1420          * us above the proc and pgrp locks.
 1421          * Solution: Post a signal using an async mechanism, being sure to
 1422          * record a generation count in the delivery so that we do not deliver
 1423          * a signal to the wrong process.
 1424          *
 1425          * Note, these two mechanisms are somewhat mutually exclusive!
 1426          */
 1427 #if 0
 1428         struct kqueue *kq;
 1429 
 1430         kq = fp->f_data;
 1431         switch (cmd) {
 1432         case FIOASYNC:
 1433                 if (*(int *)data) {
 1434                         kq->kq_state |= KQ_ASYNC;
 1435                 } else {
 1436                         kq->kq_state &= ~KQ_ASYNC;
 1437                 }
 1438                 return (0);
 1439 
 1440         case FIOSETOWN:
 1441                 return (fsetown(*(int *)data, &kq->kq_sigio));
 1442 
 1443         case FIOGETOWN:
 1444                 *(int *)data = fgetown(&kq->kq_sigio);
 1445                 return (0);
 1446         }
 1447 #endif
 1448 
 1449         return (ENOTTY);
 1450 }
 1451 
 1452 /*ARGSUSED*/
 1453 static int
 1454 kqueue_poll(struct file *fp, int events, struct ucred *active_cred,
 1455         struct thread *td)
 1456 {
 1457         struct kqueue *kq;
 1458         int revents = 0;
 1459         int error;
 1460 
 1461         if ((error = kqueue_acquire(fp, &kq)))
 1462                 return POLLERR;
 1463 
 1464         KQ_LOCK(kq);
 1465         if (events & (POLLIN | POLLRDNORM)) {
 1466                 if (kq->kq_count) {
 1467                         revents |= events & (POLLIN | POLLRDNORM);
 1468                 } else {
 1469                         selrecord(td, &kq->kq_sel);
 1470                         kq->kq_state |= KQ_SEL;
 1471                 }
 1472         }
 1473         kqueue_release(kq, 1);
 1474         KQ_UNLOCK(kq);
 1475         return (revents);
 1476 }
 1477 
 1478 /*ARGSUSED*/
 1479 static int
 1480 kqueue_stat(struct file *fp, struct stat *st, struct ucred *active_cred,
 1481         struct thread *td)
 1482 {
 1483 
 1484         bzero((void *)st, sizeof *st);
 1485         /*
 1486          * We no longer return kq_count because the unlocked value is useless.
 1487          * If you spent all this time getting the count, why not spend your
 1488          * syscall better by calling kevent?
 1489          *
 1490          * XXX - This is needed for libc_r.
 1491          */
 1492         st->st_mode = S_IFIFO;
 1493         return (0);
 1494 }
 1495 
 1496 /*ARGSUSED*/
 1497 static int
 1498 kqueue_close(struct file *fp, struct thread *td)
 1499 {
 1500         struct kqueue *kq = fp->f_data;
 1501         struct filedesc *fdp;
 1502         struct knote *kn;
 1503         int i;
 1504         int error;
 1505 
 1506         if ((error = kqueue_acquire(fp, &kq)))
 1507                 return error;
 1508 
 1509         KQ_LOCK(kq);
 1510 
 1511         KASSERT((kq->kq_state & KQ_CLOSING) != KQ_CLOSING,
 1512             ("kqueue already closing"));
 1513         kq->kq_state |= KQ_CLOSING;
 1514         if (kq->kq_refcnt > 1)
 1515                 msleep(&kq->kq_refcnt, &kq->kq_lock, PSOCK, "kqclose", 0);
 1516 
 1517         KASSERT(kq->kq_refcnt == 1, ("other refs are out there!"));
 1518         fdp = kq->kq_fdp;
 1519 
 1520         KASSERT(knlist_empty(&kq->kq_sel.si_note),
 1521             ("kqueue's knlist not empty"));
 1522 
 1523         for (i = 0; i < kq->kq_knlistsize; i++) {
 1524                 while ((kn = SLIST_FIRST(&kq->kq_knlist[i])) != NULL) {
 1525                         if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
 1526                                 kq->kq_state |= KQ_FLUXWAIT;
 1527                                 msleep(kq, &kq->kq_lock, PSOCK, "kqclo1", 0);
 1528                                 continue;
 1529                         }
 1530                         kn->kn_status |= KN_INFLUX;
 1531                         KQ_UNLOCK(kq);
 1532                         if (!(kn->kn_status & KN_DETACHED))
 1533                                 kn->kn_fop->f_detach(kn);
 1534                         knote_drop(kn, td);
 1535                         KQ_LOCK(kq);
 1536                 }
 1537         }
 1538         if (kq->kq_knhashmask != 0) {
 1539                 for (i = 0; i <= kq->kq_knhashmask; i++) {
 1540                         while ((kn = SLIST_FIRST(&kq->kq_knhash[i])) != NULL) {
 1541                                 if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
 1542                                         kq->kq_state |= KQ_FLUXWAIT;
 1543                                         msleep(kq, &kq->kq_lock, PSOCK,
 1544                                                "kqclo2", 0);
 1545                                         continue;
 1546                                 }
 1547                                 kn->kn_status |= KN_INFLUX;
 1548                                 KQ_UNLOCK(kq);
 1549                                 if (!(kn->kn_status & KN_DETACHED))
 1550                                         kn->kn_fop->f_detach(kn);
 1551                                 knote_drop(kn, td);
 1552                                 KQ_LOCK(kq);
 1553                         }
 1554                 }
 1555         }
 1556 
 1557         if ((kq->kq_state & KQ_TASKSCHED) == KQ_TASKSCHED) {
 1558                 kq->kq_state |= KQ_TASKDRAIN;
 1559                 msleep(&kq->kq_state, &kq->kq_lock, PSOCK, "kqtqdr", 0);
 1560         }
 1561 
 1562         if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
 1563                 kq->kq_state &= ~KQ_SEL;
 1564                 selwakeuppri(&kq->kq_sel, PSOCK);
 1565         }
 1566 
 1567         KQ_UNLOCK(kq);
 1568 
 1569         FILEDESC_XLOCK(fdp);
 1570         SLIST_REMOVE(&fdp->fd_kqlist, kq, kqueue, kq_list);
 1571         FILEDESC_XUNLOCK(fdp);
 1572 
 1573         knlist_destroy(&kq->kq_sel.si_note);
 1574         mtx_destroy(&kq->kq_lock);
 1575         kq->kq_fdp = NULL;
 1576 
 1577         if (kq->kq_knhash != NULL)
 1578                 free(kq->kq_knhash, M_KQUEUE);
 1579         if (kq->kq_knlist != NULL)
 1580                 free(kq->kq_knlist, M_KQUEUE);
 1581 
 1582         funsetown(&kq->kq_sigio);
 1583         free(kq, M_KQUEUE);
 1584         fp->f_data = NULL;
 1585 
 1586         return (0);
 1587 }
 1588 
 1589 static void
 1590 kqueue_wakeup(struct kqueue *kq)
 1591 {
 1592         KQ_OWNED(kq);
 1593 
 1594         if ((kq->kq_state & KQ_SLEEP) == KQ_SLEEP) {
 1595                 kq->kq_state &= ~KQ_SLEEP;
 1596                 wakeup(kq);
 1597         }
 1598         if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
 1599                 kq->kq_state &= ~KQ_SEL;
 1600                 selwakeuppri(&kq->kq_sel, PSOCK);
 1601         }
 1602         if (!knlist_empty(&kq->kq_sel.si_note))
 1603                 kqueue_schedtask(kq);
 1604         if ((kq->kq_state & KQ_ASYNC) == KQ_ASYNC) {
 1605                 pgsigio(&kq->kq_sigio, SIGIO, 0);
 1606         }
 1607 }
 1608 
 1609 /*
 1610  * Walk down a list of knotes, activating them if their event has triggered.
 1611  *
 1612  * There is a possibility to optimize in the case of one kq watching another.
 1613  * Instead of scheduling a task to wake it up, you could pass enough state
 1614  * down the chain to make up the parent kqueue.  Make this code functional
 1615  * first.
 1616  */
 1617 void
 1618 knote(struct knlist *list, long hint, int lockflags)
 1619 {
 1620         struct kqueue *kq;
 1621         struct knote *kn;
 1622         int error;
 1623 
 1624         if (list == NULL)
 1625                 return;
 1626 
 1627         KNL_ASSERT_LOCK(list, lockflags & KNF_LISTLOCKED);
 1628 
 1629         if ((lockflags & KNF_LISTLOCKED) == 0)
 1630                 list->kl_lock(list->kl_lockarg); 
 1631 
 1632         /*
 1633          * If we unlock the list lock (and set KN_INFLUX), we can eliminate
 1634          * the kqueue scheduling, but this will introduce four
 1635          * lock/unlock's for each knote to test.  If we do, continue to use
 1636          * SLIST_FOREACH, SLIST_FOREACH_SAFE is not safe in our case, it is
 1637          * only safe if you want to remove the current item, which we are
 1638          * not doing.
 1639          */
 1640         SLIST_FOREACH(kn, &list->kl_list, kn_selnext) {
 1641                 kq = kn->kn_kq;
 1642                 if ((kn->kn_status & KN_INFLUX) != KN_INFLUX) {
 1643                         KQ_LOCK(kq);
 1644                         if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
 1645                                 KQ_UNLOCK(kq);
 1646                         } else if ((lockflags & KNF_NOKQLOCK) != 0) {
 1647                                 kn->kn_status |= KN_INFLUX;
 1648                                 KQ_UNLOCK(kq);
 1649                                 error = kn->kn_fop->f_event(kn, hint);
 1650                                 KQ_LOCK(kq);
 1651                                 kn->kn_status &= ~KN_INFLUX;
 1652                                 if (error)
 1653                                         KNOTE_ACTIVATE(kn, 1);
 1654                                 KQ_UNLOCK_FLUX(kq);
 1655                         } else {
 1656                                 kn->kn_status |= KN_HASKQLOCK;
 1657                                 if (kn->kn_fop->f_event(kn, hint))
 1658                                         KNOTE_ACTIVATE(kn, 1);
 1659                                 kn->kn_status &= ~KN_HASKQLOCK;
 1660                                 KQ_UNLOCK(kq);
 1661                         }
 1662                 }
 1663                 kq = NULL;
 1664         }
 1665         if ((lockflags & KNF_LISTLOCKED) == 0)
 1666                 list->kl_unlock(list->kl_lockarg); 
 1667 }
 1668 
 1669 /*
 1670  * add a knote to a knlist
 1671  */
 1672 void
 1673 knlist_add(struct knlist *knl, struct knote *kn, int islocked)
 1674 {
 1675         KNL_ASSERT_LOCK(knl, islocked);
 1676         KQ_NOTOWNED(kn->kn_kq);
 1677         KASSERT((kn->kn_status & (KN_INFLUX|KN_DETACHED)) ==
 1678             (KN_INFLUX|KN_DETACHED), ("knote not KN_INFLUX and KN_DETACHED"));
 1679         if (!islocked)
 1680                 knl->kl_lock(knl->kl_lockarg);
 1681         SLIST_INSERT_HEAD(&knl->kl_list, kn, kn_selnext);
 1682         if (!islocked)
 1683                 knl->kl_unlock(knl->kl_lockarg);
 1684         KQ_LOCK(kn->kn_kq);
 1685         kn->kn_knlist = knl;
 1686         kn->kn_status &= ~KN_DETACHED;
 1687         KQ_UNLOCK(kn->kn_kq);
 1688 }
 1689 
 1690 static void
 1691 knlist_remove_kq(struct knlist *knl, struct knote *kn, int knlislocked, int kqislocked)
 1692 {
 1693         KASSERT(!(!!kqislocked && !knlislocked), ("kq locked w/o knl locked"));
 1694         KNL_ASSERT_LOCK(knl, knlislocked);
 1695         mtx_assert(&kn->kn_kq->kq_lock, kqislocked ? MA_OWNED : MA_NOTOWNED);
 1696         if (!kqislocked)
 1697                 KASSERT((kn->kn_status & (KN_INFLUX|KN_DETACHED)) == KN_INFLUX,
 1698     ("knlist_remove called w/o knote being KN_INFLUX or already removed"));
 1699         if (!knlislocked)
 1700                 knl->kl_lock(knl->kl_lockarg);
 1701         SLIST_REMOVE(&knl->kl_list, kn, knote, kn_selnext);
 1702         kn->kn_knlist = NULL;
 1703         if (!knlislocked)
 1704                 knl->kl_unlock(knl->kl_lockarg);
 1705         if (!kqislocked)
 1706                 KQ_LOCK(kn->kn_kq);
 1707         kn->kn_status |= KN_DETACHED;
 1708         if (!kqislocked)
 1709                 KQ_UNLOCK(kn->kn_kq);
 1710 }
 1711 
 1712 /*
 1713  * remove all knotes from a specified klist
 1714  */
 1715 void
 1716 knlist_remove(struct knlist *knl, struct knote *kn, int islocked)
 1717 {
 1718 
 1719         knlist_remove_kq(knl, kn, islocked, 0);
 1720 }
 1721 
 1722 /*
 1723  * remove knote from a specified klist while in f_event handler.
 1724  */
 1725 void
 1726 knlist_remove_inevent(struct knlist *knl, struct knote *kn)
 1727 {
 1728 
 1729         knlist_remove_kq(knl, kn, 1,
 1730             (kn->kn_status & KN_HASKQLOCK) == KN_HASKQLOCK);
 1731 }
 1732 
 1733 int
 1734 knlist_empty(struct knlist *knl)
 1735 {
 1736         KNL_ASSERT_LOCKED(knl);
 1737         return SLIST_EMPTY(&knl->kl_list);
 1738 }
 1739 
 1740 static struct mtx       knlist_lock;
 1741 MTX_SYSINIT(knlist_lock, &knlist_lock, "knlist lock for lockless objects",
 1742         MTX_DEF);
 1743 static void knlist_mtx_lock(void *arg);
 1744 static void knlist_mtx_unlock(void *arg);
 1745 static int knlist_mtx_locked(void *arg);
 1746 
 1747 static void
 1748 knlist_mtx_lock(void *arg)
 1749 {
 1750         mtx_lock((struct mtx *)arg);
 1751 }
 1752 
 1753 static void
 1754 knlist_mtx_unlock(void *arg)
 1755 {
 1756         mtx_unlock((struct mtx *)arg);
 1757 }
 1758 
 1759 static int
 1760 knlist_mtx_locked(void *arg)
 1761 {
 1762         return (mtx_owned((struct mtx *)arg));
 1763 }
 1764 
 1765 void
 1766 knlist_init(struct knlist *knl, void *lock, void (*kl_lock)(void *),
 1767     void (*kl_unlock)(void *), int (*kl_locked)(void *))
 1768 {
 1769 
 1770         if (lock == NULL)
 1771                 knl->kl_lockarg = &knlist_lock;
 1772         else
 1773                 knl->kl_lockarg = lock;
 1774 
 1775         if (kl_lock == NULL)
 1776                 knl->kl_lock = knlist_mtx_lock;
 1777         else
 1778                 knl->kl_lock = kl_lock;
 1779         if (kl_unlock == NULL)
 1780                 knl->kl_unlock = knlist_mtx_unlock;
 1781         else
 1782                 knl->kl_unlock = kl_unlock;
 1783         if (kl_locked == NULL && kl_lock == NULL && kl_unlock == NULL)
 1784                 knl->kl_locked = knlist_mtx_locked;
 1785         else
 1786                 knl->kl_locked = kl_locked;
 1787 
 1788         SLIST_INIT(&knl->kl_list);
 1789 }
 1790 
 1791 void
 1792 knlist_init_mtx(struct knlist *knl, struct mtx *lock)
 1793 {
 1794 
 1795         knlist_init(knl, lock, NULL, NULL, NULL);
 1796 }
 1797 
 1798 void
 1799 knlist_destroy(struct knlist *knl)
 1800 {
 1801 
 1802 #ifdef INVARIANTS
 1803         /*
 1804          * if we run across this error, we need to find the offending
 1805          * driver and have it call knlist_clear.
 1806          */
 1807         if (!SLIST_EMPTY(&knl->kl_list))
 1808                 printf("WARNING: destroying knlist w/ knotes on it!\n");
 1809 #endif
 1810 
 1811         knl->kl_lockarg = knl->kl_lock = knl->kl_unlock = NULL;
 1812         SLIST_INIT(&knl->kl_list);
 1813 }
 1814 
 1815 /*
 1816  * Even if we are locked, we may need to drop the lock to allow any influx
 1817  * knotes time to "settle".
 1818  */
 1819 void
 1820 knlist_cleardel(struct knlist *knl, struct thread *td, int islocked, int killkn)
 1821 {
 1822         struct knote *kn, *kn2;
 1823         struct kqueue *kq;
 1824 
 1825         if (islocked)
 1826                 KNL_ASSERT_LOCKED(knl);
 1827         else {
 1828                 KNL_ASSERT_UNLOCKED(knl);
 1829 again:          /* need to reacquire lock since we have dropped it */
 1830                 knl->kl_lock(knl->kl_lockarg);
 1831         }
 1832 
 1833         SLIST_FOREACH_SAFE(kn, &knl->kl_list, kn_selnext, kn2) {
 1834                 kq = kn->kn_kq;
 1835                 KQ_LOCK(kq);
 1836                 if ((kn->kn_status & KN_INFLUX)) {
 1837                         KQ_UNLOCK(kq);
 1838                         continue;
 1839                 }
 1840                 knlist_remove_kq(knl, kn, 1, 1);
 1841                 if (killkn) {
 1842                         kn->kn_status |= KN_INFLUX | KN_DETACHED;
 1843                         KQ_UNLOCK(kq);
 1844                         knote_drop(kn, td);
 1845                 } else {
 1846                         /* Make sure cleared knotes disappear soon */
 1847                         kn->kn_flags |= (EV_EOF | EV_ONESHOT);
 1848                         KQ_UNLOCK(kq);
 1849                 }
 1850                 kq = NULL;
 1851         }
 1852 
 1853         if (!SLIST_EMPTY(&knl->kl_list)) {
 1854                 /* there are still KN_INFLUX remaining */
 1855                 kn = SLIST_FIRST(&knl->kl_list);
 1856                 kq = kn->kn_kq;
 1857                 KQ_LOCK(kq);
 1858                 KASSERT(kn->kn_status & KN_INFLUX,
 1859                     ("knote removed w/o list lock"));
 1860                 knl->kl_unlock(knl->kl_lockarg);
 1861                 kq->kq_state |= KQ_FLUXWAIT;
 1862                 msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqkclr", 0);
 1863                 kq = NULL;
 1864                 goto again;
 1865         }
 1866 
 1867         if (islocked)
 1868                 KNL_ASSERT_LOCKED(knl);
 1869         else {
 1870                 knl->kl_unlock(knl->kl_lockarg);
 1871                 KNL_ASSERT_UNLOCKED(knl);
 1872         }
 1873 }
 1874 
 1875 /*
 1876  * Remove all knotes referencing a specified fd must be called with FILEDESC
 1877  * lock.  This prevents a race where a new fd comes along and occupies the
 1878  * entry and we attach a knote to the fd.
 1879  */
 1880 void
 1881 knote_fdclose(struct thread *td, int fd)
 1882 {
 1883         struct filedesc *fdp = td->td_proc->p_fd;
 1884         struct kqueue *kq;
 1885         struct knote *kn;
 1886         int influx;
 1887 
 1888         FILEDESC_XLOCK_ASSERT(fdp);
 1889 
 1890         /*
 1891          * We shouldn't have to worry about new kevents appearing on fd
 1892          * since filedesc is locked.
 1893          */
 1894         SLIST_FOREACH(kq, &fdp->fd_kqlist, kq_list) {
 1895                 KQ_LOCK(kq);
 1896 
 1897 again:
 1898                 influx = 0;
 1899                 while (kq->kq_knlistsize > fd &&
 1900                     (kn = SLIST_FIRST(&kq->kq_knlist[fd])) != NULL) {
 1901                         if (kn->kn_status & KN_INFLUX) {
 1902                                 /* someone else might be waiting on our knote */
 1903                                 if (influx)
 1904                                         wakeup(kq);
 1905                                 kq->kq_state |= KQ_FLUXWAIT;
 1906                                 msleep(kq, &kq->kq_lock, PSOCK, "kqflxwt", 0);
 1907                                 goto again;
 1908                         }
 1909                         kn->kn_status |= KN_INFLUX;
 1910                         KQ_UNLOCK(kq);
 1911                         if (!(kn->kn_status & KN_DETACHED))
 1912                                 kn->kn_fop->f_detach(kn);
 1913                         knote_drop(kn, td);
 1914                         influx = 1;
 1915                         KQ_LOCK(kq);
 1916                 }
 1917                 KQ_UNLOCK_FLUX(kq);
 1918         }
 1919 }
 1920 
 1921 static int
 1922 knote_attach(struct knote *kn, struct kqueue *kq)
 1923 {
 1924         struct klist *list;
 1925 
 1926         KASSERT(kn->kn_status & KN_INFLUX, ("knote not marked INFLUX"));
 1927         KQ_OWNED(kq);
 1928 
 1929         if (kn->kn_fop->f_isfd) {
 1930                 if (kn->kn_id >= kq->kq_knlistsize)
 1931                         return ENOMEM;
 1932                 list = &kq->kq_knlist[kn->kn_id];
 1933         } else {
 1934                 if (kq->kq_knhash == NULL)
 1935                         return ENOMEM;
 1936                 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
 1937         }
 1938 
 1939         SLIST_INSERT_HEAD(list, kn, kn_link);
 1940 
 1941         return 0;
 1942 }
 1943 
 1944 /*
 1945  * knote must already have been detached using the f_detach method.
 1946  * no lock need to be held, it is assumed that the KN_INFLUX flag is set
 1947  * to prevent other removal.
 1948  */
 1949 static void
 1950 knote_drop(struct knote *kn, struct thread *td)
 1951 {
 1952         struct kqueue *kq;
 1953         struct klist *list;
 1954 
 1955         kq = kn->kn_kq;
 1956 
 1957         KQ_NOTOWNED(kq);
 1958         KASSERT((kn->kn_status & KN_INFLUX) == KN_INFLUX,
 1959             ("knote_drop called without KN_INFLUX set in kn_status"));
 1960 
 1961         KQ_LOCK(kq);
 1962         if (kn->kn_fop->f_isfd)
 1963                 list = &kq->kq_knlist[kn->kn_id];
 1964         else
 1965                 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
 1966 
 1967         if (!SLIST_EMPTY(list))
 1968                 SLIST_REMOVE(list, kn, knote, kn_link);
 1969         if (kn->kn_status & KN_QUEUED)
 1970                 knote_dequeue(kn);
 1971         KQ_UNLOCK_FLUX(kq);
 1972 
 1973         if (kn->kn_fop->f_isfd) {
 1974                 fdrop(kn->kn_fp, td);
 1975                 kn->kn_fp = NULL;
 1976         }
 1977         kqueue_fo_release(kn->kn_kevent.filter);
 1978         kn->kn_fop = NULL;
 1979         knote_free(kn);
 1980 }
 1981 
 1982 static void
 1983 knote_enqueue(struct knote *kn)
 1984 {
 1985         struct kqueue *kq = kn->kn_kq;
 1986 
 1987         KQ_OWNED(kn->kn_kq);
 1988         KASSERT((kn->kn_status & KN_QUEUED) == 0, ("knote already queued"));
 1989 
 1990         TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
 1991         kn->kn_status |= KN_QUEUED;
 1992         kq->kq_count++;
 1993         kqueue_wakeup(kq);
 1994 }
 1995 
 1996 static void
 1997 knote_dequeue(struct knote *kn)
 1998 {
 1999         struct kqueue *kq = kn->kn_kq;
 2000 
 2001         KQ_OWNED(kn->kn_kq);
 2002         KASSERT(kn->kn_status & KN_QUEUED, ("knote not queued"));
 2003 
 2004         TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
 2005         kn->kn_status &= ~KN_QUEUED;
 2006         kq->kq_count--;
 2007 }
 2008 
 2009 static void
 2010 knote_init(void)
 2011 {
 2012 
 2013         knote_zone = uma_zcreate("KNOTE", sizeof(struct knote), NULL, NULL,
 2014             NULL, NULL, UMA_ALIGN_PTR, 0);
 2015 }
 2016 SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL);
 2017 
 2018 static struct knote *
 2019 knote_alloc(int waitok)
 2020 {
 2021         return ((struct knote *)uma_zalloc(knote_zone,
 2022             (waitok ? M_WAITOK : M_NOWAIT)|M_ZERO));
 2023 }
 2024 
 2025 static void
 2026 knote_free(struct knote *kn)
 2027 {
 2028         if (kn != NULL)
 2029                 uma_zfree(knote_zone, kn);
 2030 }
 2031 
 2032 /*
 2033  * Register the kev w/ the kq specified by fd.
 2034  */
 2035 int 
 2036 kqfd_register(int fd, struct kevent *kev, struct thread *td, int waitok)
 2037 {
 2038         struct kqueue *kq;
 2039         struct file *fp;
 2040         int error;
 2041 
 2042         if ((error = fget(td, fd, &fp)) != 0)
 2043                 return (error);
 2044         if ((error = kqueue_acquire(fp, &kq)) != 0)
 2045                 goto noacquire;
 2046 
 2047         error = kqueue_register(kq, kev, td, waitok);
 2048 
 2049         kqueue_release(kq, 0);
 2050 
 2051 noacquire:
 2052         fdrop(fp, td);
 2053 
 2054         return error;
 2055 }

Cache object: aa1c4d6139857b039df76b6b7a53b65a


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.