FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_event.c
1 /*-
2 * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
3 * Copyright 2004 John-Mark Gurney <jmg@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD: releng/6.2/sys/kern/kern_event.c 164286 2006-11-14 20:42:41Z cvs2svn $");
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/lock.h>
35 #include <sys/mutex.h>
36 #include <sys/proc.h>
37 #include <sys/malloc.h>
38 #include <sys/unistd.h>
39 #include <sys/file.h>
40 #include <sys/filedesc.h>
41 #include <sys/filio.h>
42 #include <sys/fcntl.h>
43 #include <sys/kthread.h>
44 #include <sys/selinfo.h>
45 #include <sys/queue.h>
46 #include <sys/event.h>
47 #include <sys/eventvar.h>
48 #include <sys/poll.h>
49 #include <sys/protosw.h>
50 #include <sys/sigio.h>
51 #include <sys/signalvar.h>
52 #include <sys/socket.h>
53 #include <sys/socketvar.h>
54 #include <sys/stat.h>
55 #include <sys/sysctl.h>
56 #include <sys/sysproto.h>
57 #include <sys/syscallsubr.h>
58 #include <sys/taskqueue.h>
59 #include <sys/uio.h>
60
61 #include <vm/uma.h>
62
63 static MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system");
64
65 /*
66 * This lock is used if multiple kq locks are required. This possibly
67 * should be made into a per proc lock.
68 */
69 static struct mtx kq_global;
70 MTX_SYSINIT(kq_global, &kq_global, "kqueue order", MTX_DEF);
71 #define KQ_GLOBAL_LOCK(lck, haslck) do { \
72 if (!haslck) \
73 mtx_lock(lck); \
74 haslck = 1; \
75 } while (0)
76 #define KQ_GLOBAL_UNLOCK(lck, haslck) do { \
77 if (haslck) \
78 mtx_unlock(lck); \
79 haslck = 0; \
80 } while (0)
81
82 TASKQUEUE_DEFINE_THREAD(kqueue);
83
84 static int kevent_copyout(void *arg, struct kevent *kevp, int count);
85 static int kevent_copyin(void *arg, struct kevent *kevp, int count);
86 static int kqueue_aquire(struct file *fp, struct kqueue **kqp);
87 static void kqueue_release(struct kqueue *kq, int locked);
88 static int kqueue_expand(struct kqueue *kq, struct filterops *fops,
89 uintptr_t ident, int waitok);
90 static void kqueue_task(void *arg, int pending);
91 static int kqueue_scan(struct kqueue *kq, int maxevents,
92 struct kevent_copyops *k_ops,
93 const struct timespec *timeout,
94 struct kevent *keva, struct thread *td);
95 static void kqueue_wakeup(struct kqueue *kq);
96 static struct filterops *kqueue_fo_find(int filt);
97 static void kqueue_fo_release(int filt);
98
99 static fo_rdwr_t kqueue_read;
100 static fo_rdwr_t kqueue_write;
101 static fo_ioctl_t kqueue_ioctl;
102 static fo_poll_t kqueue_poll;
103 static fo_kqfilter_t kqueue_kqfilter;
104 static fo_stat_t kqueue_stat;
105 static fo_close_t kqueue_close;
106
107 static struct fileops kqueueops = {
108 .fo_read = kqueue_read,
109 .fo_write = kqueue_write,
110 .fo_ioctl = kqueue_ioctl,
111 .fo_poll = kqueue_poll,
112 .fo_kqfilter = kqueue_kqfilter,
113 .fo_stat = kqueue_stat,
114 .fo_close = kqueue_close,
115 };
116
117 static int knote_attach(struct knote *kn, struct kqueue *kq);
118 static void knote_drop(struct knote *kn, struct thread *td);
119 static void knote_enqueue(struct knote *kn);
120 static void knote_dequeue(struct knote *kn);
121 static void knote_init(void);
122 static struct knote *knote_alloc(int waitok);
123 static void knote_free(struct knote *kn);
124
125 static void filt_kqdetach(struct knote *kn);
126 static int filt_kqueue(struct knote *kn, long hint);
127 static int filt_procattach(struct knote *kn);
128 static void filt_procdetach(struct knote *kn);
129 static int filt_proc(struct knote *kn, long hint);
130 static int filt_fileattach(struct knote *kn);
131 static void filt_timerexpire(void *knx);
132 static int filt_timerattach(struct knote *kn);
133 static void filt_timerdetach(struct knote *kn);
134 static int filt_timer(struct knote *kn, long hint);
135
136 static struct filterops file_filtops =
137 { 1, filt_fileattach, NULL, NULL };
138 static struct filterops kqread_filtops =
139 { 1, NULL, filt_kqdetach, filt_kqueue };
140 /* XXX - move to kern_proc.c? */
141 static struct filterops proc_filtops =
142 { 0, filt_procattach, filt_procdetach, filt_proc };
143 static struct filterops timer_filtops =
144 { 0, filt_timerattach, filt_timerdetach, filt_timer };
145
146 static uma_zone_t knote_zone;
147 static int kq_ncallouts = 0;
148 static int kq_calloutmax = (4 * 1024);
149 SYSCTL_INT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW,
150 &kq_calloutmax, 0, "Maximum number of callouts allocated for kqueue");
151
152 /* XXX - ensure not KN_INFLUX?? */
153 #define KNOTE_ACTIVATE(kn, islock) do { \
154 if ((islock)) \
155 mtx_assert(&(kn)->kn_kq->kq_lock, MA_OWNED); \
156 else \
157 KQ_LOCK((kn)->kn_kq); \
158 (kn)->kn_status |= KN_ACTIVE; \
159 if (((kn)->kn_status & (KN_QUEUED | KN_DISABLED)) == 0) \
160 knote_enqueue((kn)); \
161 if (!(islock)) \
162 KQ_UNLOCK((kn)->kn_kq); \
163 } while(0)
164 #define KQ_LOCK(kq) do { \
165 mtx_lock(&(kq)->kq_lock); \
166 } while (0)
167 #define KQ_FLUX_WAKEUP(kq) do { \
168 if (((kq)->kq_state & KQ_FLUXWAIT) == KQ_FLUXWAIT) { \
169 (kq)->kq_state &= ~KQ_FLUXWAIT; \
170 wakeup((kq)); \
171 } \
172 } while (0)
173 #define KQ_UNLOCK_FLUX(kq) do { \
174 KQ_FLUX_WAKEUP(kq); \
175 mtx_unlock(&(kq)->kq_lock); \
176 } while (0)
177 #define KQ_UNLOCK(kq) do { \
178 mtx_unlock(&(kq)->kq_lock); \
179 } while (0)
180 #define KQ_OWNED(kq) do { \
181 mtx_assert(&(kq)->kq_lock, MA_OWNED); \
182 } while (0)
183 #define KQ_NOTOWNED(kq) do { \
184 mtx_assert(&(kq)->kq_lock, MA_NOTOWNED); \
185 } while (0)
186 #define KN_LIST_LOCK(kn) do { \
187 if (kn->kn_knlist != NULL) \
188 kn->kn_knlist->kl_lock(kn->kn_knlist->kl_lockarg); \
189 } while (0)
190 #define KN_LIST_UNLOCK(kn) do { \
191 if (kn->kn_knlist != NULL) \
192 kn->kn_knlist->kl_unlock(kn->kn_knlist->kl_lockarg); \
193 } while (0)
194 #define KNL_ASSERT_LOCK(knl, islocked) do { \
195 if (islocked) \
196 KNL_ASSERT_LOCKED(knl); \
197 else \
198 KNL_ASSERT_UNLOCKED(knl); \
199 } while (0)
200 #ifdef INVARIANTS
201 #define KNL_ASSERT_LOCKED(knl) do { \
202 if (!knl->kl_locked((knl)->kl_lockarg)) \
203 panic("knlist not locked, but should be"); \
204 } while (0)
205 #define KNL_ASSERT_UNLOCKED(knl) do { \
206 if (knl->kl_locked((knl)->kl_lockarg)) \
207 panic("knlist locked, but should not be"); \
208 } while (0)
209 #else /* !INVARIANTS */
210 #define KNL_ASSERT_LOCKED(knl) do {} while(0)
211 #define KNL_ASSERT_UNLOCKED(knl) do {} while (0)
212 #endif /* INVARIANTS */
213
214 #define KN_HASHSIZE 64 /* XXX should be tunable */
215 #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
216
217 static int
218 filt_nullattach(struct knote *kn)
219 {
220
221 return (ENXIO);
222 };
223
224 struct filterops null_filtops =
225 { 0, filt_nullattach, NULL, NULL };
226
227 /* XXX - make SYSINIT to add these, and move into respective modules. */
228 extern struct filterops sig_filtops;
229 extern struct filterops fs_filtops;
230
231 /*
232 * Table for for all system-defined filters.
233 */
234 static struct mtx filterops_lock;
235 MTX_SYSINIT(kqueue_filterops, &filterops_lock, "protect sysfilt_ops",
236 MTX_DEF);
237 static struct {
238 struct filterops *for_fop;
239 int for_refcnt;
240 } sysfilt_ops[EVFILT_SYSCOUNT] = {
241 { &file_filtops }, /* EVFILT_READ */
242 { &file_filtops }, /* EVFILT_WRITE */
243 { &null_filtops }, /* EVFILT_AIO */
244 { &file_filtops }, /* EVFILT_VNODE */
245 { &proc_filtops }, /* EVFILT_PROC */
246 { &sig_filtops }, /* EVFILT_SIGNAL */
247 { &timer_filtops }, /* EVFILT_TIMER */
248 { &file_filtops }, /* EVFILT_NETDEV */
249 { &fs_filtops }, /* EVFILT_FS */
250 };
251
252 /*
253 * Simple redirection for all cdevsw style objects to call their fo_kqfilter
254 * method.
255 */
256 static int
257 filt_fileattach(struct knote *kn)
258 {
259
260 return (fo_kqfilter(kn->kn_fp, kn));
261 }
262
263 /*ARGSUSED*/
264 static int
265 kqueue_kqfilter(struct file *fp, struct knote *kn)
266 {
267 struct kqueue *kq = kn->kn_fp->f_data;
268
269 if (kn->kn_filter != EVFILT_READ)
270 return (EINVAL);
271
272 kn->kn_status |= KN_KQUEUE;
273 kn->kn_fop = &kqread_filtops;
274 knlist_add(&kq->kq_sel.si_note, kn, 0);
275
276 return (0);
277 }
278
279 static void
280 filt_kqdetach(struct knote *kn)
281 {
282 struct kqueue *kq = kn->kn_fp->f_data;
283
284 knlist_remove(&kq->kq_sel.si_note, kn, 0);
285 }
286
287 /*ARGSUSED*/
288 static int
289 filt_kqueue(struct knote *kn, long hint)
290 {
291 struct kqueue *kq = kn->kn_fp->f_data;
292
293 kn->kn_data = kq->kq_count;
294 return (kn->kn_data > 0);
295 }
296
297 /* XXX - move to kern_proc.c? */
298 static int
299 filt_procattach(struct knote *kn)
300 {
301 struct proc *p;
302 int immediate;
303 int error;
304
305 immediate = 0;
306 p = pfind(kn->kn_id);
307 if (p == NULL && (kn->kn_sfflags & NOTE_EXIT)) {
308 p = zpfind(kn->kn_id);
309 immediate = 1;
310 } else if (p != NULL && (p->p_flag & P_WEXIT)) {
311 immediate = 1;
312 }
313
314 if (p == NULL)
315 return (ESRCH);
316 if ((error = p_cansee(curthread, p)))
317 return (error);
318
319 kn->kn_ptr.p_proc = p;
320 kn->kn_flags |= EV_CLEAR; /* automatically set */
321
322 /*
323 * internal flag indicating registration done by kernel
324 */
325 if (kn->kn_flags & EV_FLAG1) {
326 kn->kn_data = kn->kn_sdata; /* ppid */
327 kn->kn_fflags = NOTE_CHILD;
328 kn->kn_flags &= ~EV_FLAG1;
329 }
330
331 if (immediate == 0)
332 knlist_add(&p->p_klist, kn, 1);
333
334 /*
335 * Immediately activate any exit notes if the target process is a
336 * zombie. This is necessary to handle the case where the target
337 * process, e.g. a child, dies before the kevent is registered.
338 */
339 if (immediate && filt_proc(kn, NOTE_EXIT))
340 KNOTE_ACTIVATE(kn, 0);
341
342 PROC_UNLOCK(p);
343
344 return (0);
345 }
346
347 /*
348 * The knote may be attached to a different process, which may exit,
349 * leaving nothing for the knote to be attached to. So when the process
350 * exits, the knote is marked as DETACHED and also flagged as ONESHOT so
351 * it will be deleted when read out. However, as part of the knote deletion,
352 * this routine is called, so a check is needed to avoid actually performing
353 * a detach, because the original process does not exist any more.
354 */
355 /* XXX - move to kern_proc.c? */
356 static void
357 filt_procdetach(struct knote *kn)
358 {
359 struct proc *p;
360
361 p = kn->kn_ptr.p_proc;
362 knlist_remove(&p->p_klist, kn, 0);
363 kn->kn_ptr.p_proc = NULL;
364 }
365
366 /* XXX - move to kern_proc.c? */
367 static int
368 filt_proc(struct knote *kn, long hint)
369 {
370 struct proc *p = kn->kn_ptr.p_proc;
371 u_int event;
372
373 /*
374 * mask off extra data
375 */
376 event = (u_int)hint & NOTE_PCTRLMASK;
377
378 /*
379 * if the user is interested in this event, record it.
380 */
381 if (kn->kn_sfflags & event)
382 kn->kn_fflags |= event;
383
384 /*
385 * process is gone, so flag the event as finished.
386 */
387 if (event == NOTE_EXIT) {
388 if (!(kn->kn_status & KN_DETACHED))
389 knlist_remove_inevent(&p->p_klist, kn);
390 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
391 kn->kn_ptr.p_proc = NULL;
392 return (1);
393 }
394
395 /*
396 * process forked, and user wants to track the new process,
397 * so attach a new knote to it, and immediately report an
398 * event with the parent's pid.
399 */
400 if ((event == NOTE_FORK) && (kn->kn_sfflags & NOTE_TRACK)) {
401 struct kevent kev;
402 int error;
403
404 /*
405 * register knote with new process.
406 */
407 kev.ident = hint & NOTE_PDATAMASK; /* pid */
408 kev.filter = kn->kn_filter;
409 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1;
410 kev.fflags = kn->kn_sfflags;
411 kev.data = kn->kn_id; /* parent */
412 kev.udata = kn->kn_kevent.udata; /* preserve udata */
413 error = kqueue_register(kn->kn_kq, &kev, NULL, 0);
414 if (error)
415 kn->kn_fflags |= NOTE_TRACKERR;
416 }
417
418 return (kn->kn_fflags != 0);
419 }
420
421 static int
422 timertoticks(intptr_t data)
423 {
424 struct timeval tv;
425 int tticks;
426
427 tv.tv_sec = data / 1000;
428 tv.tv_usec = (data % 1000) * 1000;
429 tticks = tvtohz(&tv);
430
431 return tticks;
432 }
433
434 /* XXX - move to kern_timeout.c? */
435 static void
436 filt_timerexpire(void *knx)
437 {
438 struct knote *kn = knx;
439 struct callout *calloutp;
440
441 kn->kn_data++;
442 KNOTE_ACTIVATE(kn, 0); /* XXX - handle locking */
443
444 if ((kn->kn_flags & EV_ONESHOT) != EV_ONESHOT) {
445 calloutp = (struct callout *)kn->kn_hook;
446 callout_reset(calloutp, timertoticks(kn->kn_sdata),
447 filt_timerexpire, kn);
448 }
449 }
450
451 /*
452 * data contains amount of time to sleep, in milliseconds
453 */
454 /* XXX - move to kern_timeout.c? */
455 static int
456 filt_timerattach(struct knote *kn)
457 {
458 struct callout *calloutp;
459
460 atomic_add_int(&kq_ncallouts, 1);
461
462 if (kq_ncallouts >= kq_calloutmax) {
463 atomic_add_int(&kq_ncallouts, -1);
464 return (ENOMEM);
465 }
466
467 kn->kn_flags |= EV_CLEAR; /* automatically set */
468 kn->kn_status &= ~KN_DETACHED; /* knlist_add usually sets it */
469 MALLOC(calloutp, struct callout *, sizeof(*calloutp),
470 M_KQUEUE, M_WAITOK);
471 callout_init(calloutp, CALLOUT_MPSAFE);
472 kn->kn_hook = calloutp;
473 callout_reset(calloutp, timertoticks(kn->kn_sdata), filt_timerexpire,
474 kn);
475
476 return (0);
477 }
478
479 /* XXX - move to kern_timeout.c? */
480 static void
481 filt_timerdetach(struct knote *kn)
482 {
483 struct callout *calloutp;
484
485 calloutp = (struct callout *)kn->kn_hook;
486 callout_drain(calloutp);
487 FREE(calloutp, M_KQUEUE);
488 atomic_add_int(&kq_ncallouts, -1);
489 kn->kn_status |= KN_DETACHED; /* knlist_remove usually clears it */
490 }
491
492 /* XXX - move to kern_timeout.c? */
493 static int
494 filt_timer(struct knote *kn, long hint)
495 {
496
497 return (kn->kn_data != 0);
498 }
499
500 /*
501 * MPSAFE
502 */
503 int
504 kqueue(struct thread *td, struct kqueue_args *uap)
505 {
506 struct filedesc *fdp;
507 struct kqueue *kq;
508 struct file *fp;
509 int fd, error;
510
511 fdp = td->td_proc->p_fd;
512 error = falloc(td, &fp, &fd);
513 if (error)
514 goto done2;
515
516 /* An extra reference on `nfp' has been held for us by falloc(). */
517 kq = malloc(sizeof *kq, M_KQUEUE, M_WAITOK | M_ZERO);
518 mtx_init(&kq->kq_lock, "kqueue", NULL, MTX_DEF|MTX_DUPOK);
519 TAILQ_INIT(&kq->kq_head);
520 kq->kq_fdp = fdp;
521 knlist_init(&kq->kq_sel.si_note, &kq->kq_lock, NULL, NULL, NULL);
522 TASK_INIT(&kq->kq_task, 0, kqueue_task, kq);
523
524 FILEDESC_LOCK_FAST(fdp);
525 SLIST_INSERT_HEAD(&fdp->fd_kqlist, kq, kq_list);
526 FILEDESC_UNLOCK_FAST(fdp);
527
528 FILE_LOCK(fp);
529 fp->f_flag = FREAD | FWRITE;
530 fp->f_type = DTYPE_KQUEUE;
531 fp->f_ops = &kqueueops;
532 fp->f_data = kq;
533 FILE_UNLOCK(fp);
534 fdrop(fp, td);
535
536 td->td_retval[0] = fd;
537 done2:
538 return (error);
539 }
540
541 #ifndef _SYS_SYSPROTO_H_
542 struct kevent_args {
543 int fd;
544 const struct kevent *changelist;
545 int nchanges;
546 struct kevent *eventlist;
547 int nevents;
548 const struct timespec *timeout;
549 };
550 #endif
551 /*
552 * MPSAFE
553 */
554 int
555 kevent(struct thread *td, struct kevent_args *uap)
556 {
557 struct timespec ts, *tsp;
558 struct kevent_copyops k_ops = { uap,
559 kevent_copyout,
560 kevent_copyin};
561 int error;
562
563 if (uap->timeout != NULL) {
564 error = copyin(uap->timeout, &ts, sizeof(ts));
565 if (error)
566 return (error);
567 tsp = &ts;
568 } else
569 tsp = NULL;
570
571 return (kern_kevent(td, uap->fd, uap->nchanges, uap->nevents,
572 &k_ops, tsp));
573 }
574
575 /*
576 * Copy 'count' items into the destination list pointed to by uap->eventlist.
577 */
578 static int
579 kevent_copyout(void *arg, struct kevent *kevp, int count)
580 {
581 struct kevent_args *uap;
582 int error;
583
584 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
585 uap = (struct kevent_args *)arg;
586
587 error = copyout(kevp, uap->eventlist, count * sizeof *kevp);
588 if (error == 0)
589 uap->eventlist += count;
590 return (error);
591 }
592
593 /*
594 * Copy 'count' items from the list pointed to by uap->changelist.
595 */
596 static int
597 kevent_copyin(void *arg, struct kevent *kevp, int count)
598 {
599 struct kevent_args *uap;
600 int error;
601
602 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
603 uap = (struct kevent_args *)arg;
604
605 error = copyin(uap->changelist, kevp, count * sizeof *kevp);
606 if (error == 0)
607 uap->changelist += count;
608 return (error);
609 }
610
611 int
612 kern_kevent(struct thread *td, int fd, int nchanges, int nevents,
613 struct kevent_copyops *k_ops, const struct timespec *timeout)
614 {
615 struct kevent keva[KQ_NEVENTS];
616 struct kevent *kevp, *changes;
617 struct kqueue *kq;
618 struct file *fp;
619 int i, n, nerrors, error;
620
621 if ((error = fget(td, fd, &fp)) != 0)
622 return (error);
623 if ((error = kqueue_aquire(fp, &kq)) != 0)
624 goto done_norel;
625
626 nerrors = 0;
627
628 while (nchanges > 0) {
629 n = nchanges > KQ_NEVENTS ? KQ_NEVENTS : nchanges;
630 error = k_ops->k_copyin(k_ops->arg, keva, n);
631 if (error)
632 goto done;
633 changes = keva;
634 for (i = 0; i < n; i++) {
635 kevp = &changes[i];
636 kevp->flags &= ~EV_SYSFLAGS;
637 error = kqueue_register(kq, kevp, td, 1);
638 if (error) {
639 if (nevents != 0) {
640 kevp->flags = EV_ERROR;
641 kevp->data = error;
642 (void) k_ops->k_copyout(k_ops->arg,
643 kevp, 1);
644 nevents--;
645 nerrors++;
646 } else {
647 goto done;
648 }
649 }
650 }
651 nchanges -= n;
652 }
653 if (nerrors) {
654 td->td_retval[0] = nerrors;
655 error = 0;
656 goto done;
657 }
658
659 error = kqueue_scan(kq, nevents, k_ops, timeout, keva, td);
660 done:
661 kqueue_release(kq, 0);
662 done_norel:
663 if (fp != NULL)
664 fdrop(fp, td);
665 return (error);
666 }
667
668 int
669 kqueue_add_filteropts(int filt, struct filterops *filtops)
670 {
671 int error;
672
673 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0) {
674 printf(
675 "trying to add a filterop that is out of range: %d is beyond %d\n",
676 ~filt, EVFILT_SYSCOUNT);
677 return EINVAL;
678 }
679 mtx_lock(&filterops_lock);
680 if (sysfilt_ops[~filt].for_fop != &null_filtops &&
681 sysfilt_ops[~filt].for_fop != NULL)
682 error = EEXIST;
683 else {
684 sysfilt_ops[~filt].for_fop = filtops;
685 sysfilt_ops[~filt].for_refcnt = 0;
686 }
687 mtx_unlock(&filterops_lock);
688
689 return (0);
690 }
691
692 int
693 kqueue_del_filteropts(int filt)
694 {
695 int error;
696
697 error = 0;
698 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
699 return EINVAL;
700
701 mtx_lock(&filterops_lock);
702 if (sysfilt_ops[~filt].for_fop == &null_filtops ||
703 sysfilt_ops[~filt].for_fop == NULL)
704 error = EINVAL;
705 else if (sysfilt_ops[~filt].for_refcnt != 0)
706 error = EBUSY;
707 else {
708 sysfilt_ops[~filt].for_fop = &null_filtops;
709 sysfilt_ops[~filt].for_refcnt = 0;
710 }
711 mtx_unlock(&filterops_lock);
712
713 return error;
714 }
715
716 static struct filterops *
717 kqueue_fo_find(int filt)
718 {
719
720 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
721 return NULL;
722
723 mtx_lock(&filterops_lock);
724 sysfilt_ops[~filt].for_refcnt++;
725 if (sysfilt_ops[~filt].for_fop == NULL)
726 sysfilt_ops[~filt].for_fop = &null_filtops;
727 mtx_unlock(&filterops_lock);
728
729 return sysfilt_ops[~filt].for_fop;
730 }
731
732 static void
733 kqueue_fo_release(int filt)
734 {
735
736 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
737 return;
738
739 mtx_lock(&filterops_lock);
740 KASSERT(sysfilt_ops[~filt].for_refcnt > 0,
741 ("filter object refcount not valid on release"));
742 sysfilt_ops[~filt].for_refcnt--;
743 mtx_unlock(&filterops_lock);
744 }
745
746 /*
747 * A ref to kq (obtained via kqueue_aquire) should be held. waitok will
748 * influence if memory allocation should wait. Make sure it is 0 if you
749 * hold any mutexes.
750 */
751 int
752 kqueue_register(struct kqueue *kq, struct kevent *kev, struct thread *td, int waitok)
753 {
754 struct filedesc *fdp;
755 struct filterops *fops;
756 struct file *fp;
757 struct knote *kn, *tkn;
758 int error, filt, event;
759 int haskqglobal;
760 int fd;
761
762 fdp = NULL;
763 fp = NULL;
764 kn = NULL;
765 error = 0;
766 haskqglobal = 0;
767
768 filt = kev->filter;
769 fops = kqueue_fo_find(filt);
770 if (fops == NULL)
771 return EINVAL;
772
773 tkn = knote_alloc(waitok); /* prevent waiting with locks */
774
775 findkn:
776 if (fops->f_isfd) {
777 KASSERT(td != NULL, ("td is NULL"));
778 fdp = td->td_proc->p_fd;
779 FILEDESC_LOCK(fdp);
780 /* validate descriptor */
781 fd = kev->ident;
782 if (fd < 0 || fd >= fdp->fd_nfiles ||
783 (fp = fdp->fd_ofiles[fd]) == NULL) {
784 FILEDESC_UNLOCK(fdp);
785 error = EBADF;
786 goto done;
787 }
788 fhold(fp);
789
790 if ((kev->flags & EV_ADD) == EV_ADD && kqueue_expand(kq, fops,
791 kev->ident, 0) != 0) {
792 /* unlock and try again */
793 FILEDESC_UNLOCK(fdp);
794 fdrop(fp, td);
795 fp = NULL;
796 error = kqueue_expand(kq, fops, kev->ident, waitok);
797 if (error)
798 goto done;
799 goto findkn;
800 }
801
802 if (fp->f_type == DTYPE_KQUEUE) {
803 /*
804 * if we add some inteligence about what we are doing,
805 * we should be able to support events on ourselves.
806 * We need to know when we are doing this to prevent
807 * getting both the knlist lock and the kq lock since
808 * they are the same thing.
809 */
810 if (fp->f_data == kq) {
811 FILEDESC_UNLOCK(fdp);
812 error = EINVAL;
813 goto done;
814 }
815
816 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
817 }
818
819 FILEDESC_UNLOCK(fdp);
820 KQ_LOCK(kq);
821 if (kev->ident < kq->kq_knlistsize) {
822 SLIST_FOREACH(kn, &kq->kq_knlist[kev->ident], kn_link)
823 if (kev->filter == kn->kn_filter)
824 break;
825 }
826 } else {
827 if ((kev->flags & EV_ADD) == EV_ADD)
828 kqueue_expand(kq, fops, kev->ident, waitok);
829
830 KQ_LOCK(kq);
831 if (kq->kq_knhashmask != 0) {
832 struct klist *list;
833
834 list = &kq->kq_knhash[
835 KN_HASH((u_long)kev->ident, kq->kq_knhashmask)];
836 SLIST_FOREACH(kn, list, kn_link)
837 if (kev->ident == kn->kn_id &&
838 kev->filter == kn->kn_filter)
839 break;
840 }
841 }
842
843 /* knote is in the process of changing, wait for it to stablize. */
844 if (kn != NULL && (kn->kn_status & KN_INFLUX) == KN_INFLUX) {
845 if (fp != NULL) {
846 fdrop(fp, td);
847 fp = NULL;
848 }
849 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
850 kq->kq_state |= KQ_FLUXWAIT;
851 msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqflxwt", 0);
852 goto findkn;
853 }
854
855 if (kn == NULL && ((kev->flags & EV_ADD) == 0)) {
856 KQ_UNLOCK(kq);
857 error = ENOENT;
858 goto done;
859 }
860
861 /*
862 * kn now contains the matching knote, or NULL if no match
863 */
864 if (kev->flags & EV_ADD) {
865 if (kn == NULL) {
866 kn = tkn;
867 tkn = NULL;
868 if (kn == NULL) {
869 KQ_UNLOCK(kq);
870 error = ENOMEM;
871 goto done;
872 }
873 kn->kn_fp = fp;
874 kn->kn_kq = kq;
875 kn->kn_fop = fops;
876 /*
877 * apply reference counts to knote structure, and
878 * do not release it at the end of this routine.
879 */
880 fops = NULL;
881 fp = NULL;
882
883 kn->kn_sfflags = kev->fflags;
884 kn->kn_sdata = kev->data;
885 kev->fflags = 0;
886 kev->data = 0;
887 kn->kn_kevent = *kev;
888 kn->kn_kevent.flags &= ~(EV_ADD | EV_DELETE |
889 EV_ENABLE | EV_DISABLE);
890 kn->kn_status = KN_INFLUX|KN_DETACHED;
891
892 error = knote_attach(kn, kq);
893 KQ_UNLOCK(kq);
894 if (error != 0) {
895 tkn = kn;
896 goto done;
897 }
898
899 if ((error = kn->kn_fop->f_attach(kn)) != 0) {
900 knote_drop(kn, td);
901 goto done;
902 }
903 KN_LIST_LOCK(kn);
904 } else {
905 /*
906 * The user may change some filter values after the
907 * initial EV_ADD, but doing so will not reset any
908 * filter which has already been triggered.
909 */
910 kn->kn_status |= KN_INFLUX;
911 KQ_UNLOCK(kq);
912 KN_LIST_LOCK(kn);
913 kn->kn_sfflags = kev->fflags;
914 kn->kn_sdata = kev->data;
915 kn->kn_kevent.udata = kev->udata;
916 }
917
918 /*
919 * We can get here with kn->kn_knlist == NULL.
920 * This can happen when the initial attach event decides that
921 * the event is "completed" already. i.e. filt_procattach
922 * is called on a zombie process. It will call filt_proc
923 * which will remove it from the list, and NULL kn_knlist.
924 */
925 event = kn->kn_fop->f_event(kn, 0);
926 KQ_LOCK(kq);
927 if (event)
928 KNOTE_ACTIVATE(kn, 1);
929 kn->kn_status &= ~KN_INFLUX;
930 KN_LIST_UNLOCK(kn);
931 } else if (kev->flags & EV_DELETE) {
932 kn->kn_status |= KN_INFLUX;
933 KQ_UNLOCK(kq);
934 if (!(kn->kn_status & KN_DETACHED))
935 kn->kn_fop->f_detach(kn);
936 knote_drop(kn, td);
937 goto done;
938 }
939
940 if ((kev->flags & EV_DISABLE) &&
941 ((kn->kn_status & KN_DISABLED) == 0)) {
942 kn->kn_status |= KN_DISABLED;
943 }
944
945 if ((kev->flags & EV_ENABLE) && (kn->kn_status & KN_DISABLED)) {
946 kn->kn_status &= ~KN_DISABLED;
947 if ((kn->kn_status & KN_ACTIVE) &&
948 ((kn->kn_status & KN_QUEUED) == 0))
949 knote_enqueue(kn);
950 }
951 KQ_UNLOCK_FLUX(kq);
952
953 done:
954 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
955 if (fp != NULL)
956 fdrop(fp, td);
957 if (tkn != NULL)
958 knote_free(tkn);
959 if (fops != NULL)
960 kqueue_fo_release(filt);
961 return (error);
962 }
963
964 static int
965 kqueue_aquire(struct file *fp, struct kqueue **kqp)
966 {
967 int error;
968 struct kqueue *kq;
969
970 error = 0;
971
972 FILE_LOCK(fp);
973 do {
974 kq = fp->f_data;
975 if (fp->f_type != DTYPE_KQUEUE || kq == NULL) {
976 error = EBADF;
977 break;
978 }
979 *kqp = kq;
980 KQ_LOCK(kq);
981 if ((kq->kq_state & KQ_CLOSING) == KQ_CLOSING) {
982 KQ_UNLOCK(kq);
983 error = EBADF;
984 break;
985 }
986 kq->kq_refcnt++;
987 KQ_UNLOCK(kq);
988 } while (0);
989 FILE_UNLOCK(fp);
990
991 return error;
992 }
993
994 static void
995 kqueue_release(struct kqueue *kq, int locked)
996 {
997 if (locked)
998 KQ_OWNED(kq);
999 else
1000 KQ_LOCK(kq);
1001 kq->kq_refcnt--;
1002 if (kq->kq_refcnt == 1)
1003 wakeup(&kq->kq_refcnt);
1004 if (!locked)
1005 KQ_UNLOCK(kq);
1006 }
1007
1008 static void
1009 kqueue_schedtask(struct kqueue *kq)
1010 {
1011
1012 KQ_OWNED(kq);
1013 KASSERT(((kq->kq_state & KQ_TASKDRAIN) != KQ_TASKDRAIN),
1014 ("scheduling kqueue task while draining"));
1015
1016 if ((kq->kq_state & KQ_TASKSCHED) != KQ_TASKSCHED) {
1017 taskqueue_enqueue(taskqueue_kqueue, &kq->kq_task);
1018 kq->kq_state |= KQ_TASKSCHED;
1019 }
1020 }
1021
1022 /*
1023 * Expand the kq to make sure we have storage for fops/ident pair.
1024 *
1025 * Return 0 on success (or no work necessary), return errno on failure.
1026 *
1027 * Not calling hashinit w/ waitok (proper malloc flag) should be safe.
1028 * If kqueue_register is called from a non-fd context, there usually/should
1029 * be no locks held.
1030 */
1031 static int
1032 kqueue_expand(struct kqueue *kq, struct filterops *fops, uintptr_t ident,
1033 int waitok)
1034 {
1035 struct klist *list, *tmp_knhash;
1036 u_long tmp_knhashmask;
1037 int size;
1038 int fd;
1039 int mflag = waitok ? M_WAITOK : M_NOWAIT;
1040
1041 KQ_NOTOWNED(kq);
1042
1043 if (fops->f_isfd) {
1044 fd = ident;
1045 if (kq->kq_knlistsize <= fd) {
1046 size = kq->kq_knlistsize;
1047 while (size <= fd)
1048 size += KQEXTENT;
1049 MALLOC(list, struct klist *,
1050 size * sizeof list, M_KQUEUE, mflag);
1051 if (list == NULL)
1052 return ENOMEM;
1053 KQ_LOCK(kq);
1054 if (kq->kq_knlistsize > fd) {
1055 FREE(list, M_KQUEUE);
1056 list = NULL;
1057 } else {
1058 if (kq->kq_knlist != NULL) {
1059 bcopy(kq->kq_knlist, list,
1060 kq->kq_knlistsize * sizeof list);
1061 FREE(kq->kq_knlist, M_KQUEUE);
1062 kq->kq_knlist = NULL;
1063 }
1064 bzero((caddr_t)list +
1065 kq->kq_knlistsize * sizeof list,
1066 (size - kq->kq_knlistsize) * sizeof list);
1067 kq->kq_knlistsize = size;
1068 kq->kq_knlist = list;
1069 }
1070 KQ_UNLOCK(kq);
1071 }
1072 } else {
1073 if (kq->kq_knhashmask == 0) {
1074 tmp_knhash = hashinit(KN_HASHSIZE, M_KQUEUE,
1075 &tmp_knhashmask);
1076 if (tmp_knhash == NULL)
1077 return ENOMEM;
1078 KQ_LOCK(kq);
1079 if (kq->kq_knhashmask == 0) {
1080 kq->kq_knhash = tmp_knhash;
1081 kq->kq_knhashmask = tmp_knhashmask;
1082 } else {
1083 free(tmp_knhash, M_KQUEUE);
1084 }
1085 KQ_UNLOCK(kq);
1086 }
1087 }
1088
1089 KQ_NOTOWNED(kq);
1090 return 0;
1091 }
1092
1093 static void
1094 kqueue_task(void *arg, int pending)
1095 {
1096 struct kqueue *kq;
1097 int haskqglobal;
1098
1099 haskqglobal = 0;
1100 kq = arg;
1101
1102 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
1103 KQ_LOCK(kq);
1104
1105 KNOTE_LOCKED(&kq->kq_sel.si_note, 0);
1106
1107 kq->kq_state &= ~KQ_TASKSCHED;
1108 if ((kq->kq_state & KQ_TASKDRAIN) == KQ_TASKDRAIN) {
1109 wakeup(&kq->kq_state);
1110 }
1111 KQ_UNLOCK(kq);
1112 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1113 }
1114
1115 /*
1116 * Scan, update kn_data (if not ONESHOT), and copyout triggered events.
1117 * We treat KN_MARKER knotes as if they are INFLUX.
1118 */
1119 static int
1120 kqueue_scan(struct kqueue *kq, int maxevents, struct kevent_copyops *k_ops,
1121 const struct timespec *tsp, struct kevent *keva, struct thread *td)
1122 {
1123 struct kevent *kevp;
1124 struct timeval atv, rtv, ttv;
1125 struct knote *kn, *marker;
1126 int count, timeout, nkev, error;
1127 int haskqglobal;
1128
1129 count = maxevents;
1130 nkev = 0;
1131 error = 0;
1132 haskqglobal = 0;
1133
1134 if (maxevents == 0)
1135 goto done_nl;
1136
1137 if (tsp != NULL) {
1138 TIMESPEC_TO_TIMEVAL(&atv, tsp);
1139 if (itimerfix(&atv)) {
1140 error = EINVAL;
1141 goto done_nl;
1142 }
1143 if (tsp->tv_sec == 0 && tsp->tv_nsec == 0)
1144 timeout = -1;
1145 else
1146 timeout = atv.tv_sec > 24 * 60 * 60 ?
1147 24 * 60 * 60 * hz : tvtohz(&atv);
1148 getmicrouptime(&rtv);
1149 timevaladd(&atv, &rtv);
1150 } else {
1151 atv.tv_sec = 0;
1152 atv.tv_usec = 0;
1153 timeout = 0;
1154 }
1155 marker = knote_alloc(1);
1156 if (marker == NULL) {
1157 error = ENOMEM;
1158 goto done_nl;
1159 }
1160 marker->kn_status = KN_MARKER;
1161 KQ_LOCK(kq);
1162 goto start;
1163
1164 retry:
1165 if (atv.tv_sec || atv.tv_usec) {
1166 getmicrouptime(&rtv);
1167 if (timevalcmp(&rtv, &atv, >=))
1168 goto done;
1169 ttv = atv;
1170 timevalsub(&ttv, &rtv);
1171 timeout = ttv.tv_sec > 24 * 60 * 60 ?
1172 24 * 60 * 60 * hz : tvtohz(&ttv);
1173 }
1174
1175 start:
1176 kevp = keva;
1177 if (kq->kq_count == 0) {
1178 if (timeout < 0) {
1179 error = EWOULDBLOCK;
1180 } else {
1181 kq->kq_state |= KQ_SLEEP;
1182 error = msleep(kq, &kq->kq_lock, PSOCK | PCATCH,
1183 "kqread", timeout);
1184 }
1185 if (error == 0)
1186 goto retry;
1187 /* don't restart after signals... */
1188 if (error == ERESTART)
1189 error = EINTR;
1190 else if (error == EWOULDBLOCK)
1191 error = 0;
1192 goto done;
1193 }
1194
1195 TAILQ_INSERT_TAIL(&kq->kq_head, marker, kn_tqe);
1196 while (count) {
1197 KQ_OWNED(kq);
1198 kn = TAILQ_FIRST(&kq->kq_head);
1199
1200 if ((kn->kn_status == KN_MARKER && kn != marker) ||
1201 (kn->kn_status & KN_INFLUX) == KN_INFLUX) {
1202 kq->kq_state |= KQ_FLUXWAIT;
1203 error = msleep(kq, &kq->kq_lock, PSOCK,
1204 "kqflxwt", 0);
1205 continue;
1206 }
1207
1208 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
1209 if ((kn->kn_status & KN_DISABLED) == KN_DISABLED) {
1210 kn->kn_status &= ~KN_QUEUED;
1211 kq->kq_count--;
1212 continue;
1213 }
1214 if (kn == marker) {
1215 KQ_FLUX_WAKEUP(kq);
1216 if (count == maxevents)
1217 goto retry;
1218 goto done;
1219 }
1220 KASSERT((kn->kn_status & KN_INFLUX) == 0,
1221 ("KN_INFLUX set when not suppose to be"));
1222
1223 if ((kn->kn_flags & EV_ONESHOT) == EV_ONESHOT) {
1224 kn->kn_status &= ~KN_QUEUED;
1225 kn->kn_status |= KN_INFLUX;
1226 kq->kq_count--;
1227 KQ_UNLOCK(kq);
1228 /*
1229 * We don't need to lock the list since we've marked
1230 * it _INFLUX.
1231 */
1232 *kevp = kn->kn_kevent;
1233 if (!(kn->kn_status & KN_DETACHED))
1234 kn->kn_fop->f_detach(kn);
1235 knote_drop(kn, td);
1236 KQ_LOCK(kq);
1237 kn = NULL;
1238 } else {
1239 kn->kn_status |= KN_INFLUX;
1240 KQ_UNLOCK(kq);
1241 if ((kn->kn_status & KN_KQUEUE) == KN_KQUEUE)
1242 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
1243 KN_LIST_LOCK(kn);
1244 if (kn->kn_fop->f_event(kn, 0) == 0) {
1245 KQ_LOCK(kq);
1246 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1247 kn->kn_status &=
1248 ~(KN_QUEUED | KN_ACTIVE | KN_INFLUX);
1249 kq->kq_count--;
1250 KN_LIST_UNLOCK(kn);
1251 continue;
1252 }
1253 *kevp = kn->kn_kevent;
1254 KQ_LOCK(kq);
1255 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1256 if (kn->kn_flags & EV_CLEAR) {
1257 kn->kn_data = 0;
1258 kn->kn_fflags = 0;
1259 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE);
1260 kq->kq_count--;
1261 } else
1262 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
1263
1264 kn->kn_status &= ~(KN_INFLUX);
1265 KN_LIST_UNLOCK(kn);
1266 }
1267
1268 /* we are returning a copy to the user */
1269 kevp++;
1270 nkev++;
1271 count--;
1272
1273 if (nkev == KQ_NEVENTS) {
1274 KQ_UNLOCK_FLUX(kq);
1275 error = k_ops->k_copyout(k_ops->arg, keva, nkev);
1276 nkev = 0;
1277 kevp = keva;
1278 KQ_LOCK(kq);
1279 if (error)
1280 break;
1281 }
1282 }
1283 TAILQ_REMOVE(&kq->kq_head, marker, kn_tqe);
1284 done:
1285 KQ_OWNED(kq);
1286 KQ_UNLOCK_FLUX(kq);
1287 knote_free(marker);
1288 done_nl:
1289 KQ_NOTOWNED(kq);
1290 if (nkev != 0)
1291 error = k_ops->k_copyout(k_ops->arg, keva, nkev);
1292 td->td_retval[0] = maxevents - count;
1293 return (error);
1294 }
1295
1296 /*
1297 * XXX
1298 * This could be expanded to call kqueue_scan, if desired.
1299 */
1300 /*ARGSUSED*/
1301 static int
1302 kqueue_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
1303 int flags, struct thread *td)
1304 {
1305 return (ENXIO);
1306 }
1307
1308 /*ARGSUSED*/
1309 static int
1310 kqueue_write(struct file *fp, struct uio *uio, struct ucred *active_cred,
1311 int flags, struct thread *td)
1312 {
1313 return (ENXIO);
1314 }
1315
1316 /*ARGSUSED*/
1317 static int
1318 kqueue_ioctl(struct file *fp, u_long cmd, void *data,
1319 struct ucred *active_cred, struct thread *td)
1320 {
1321 /*
1322 * Enabling sigio causes two major problems:
1323 * 1) infinite recursion:
1324 * Synopsys: kevent is being used to track signals and have FIOASYNC
1325 * set. On receipt of a signal this will cause a kqueue to recurse
1326 * into itself over and over. Sending the sigio causes the kqueue
1327 * to become ready, which in turn posts sigio again, forever.
1328 * Solution: this can be solved by setting a flag in the kqueue that
1329 * we have a SIGIO in progress.
1330 * 2) locking problems:
1331 * Synopsys: Kqueue is a leaf subsystem, but adding signalling puts
1332 * us above the proc and pgrp locks.
1333 * Solution: Post a signal using an async mechanism, being sure to
1334 * record a generation count in the delivery so that we do not deliver
1335 * a signal to the wrong process.
1336 *
1337 * Note, these two mechanisms are somewhat mutually exclusive!
1338 */
1339 #if 0
1340 struct kqueue *kq;
1341
1342 kq = fp->f_data;
1343 switch (cmd) {
1344 case FIOASYNC:
1345 if (*(int *)data) {
1346 kq->kq_state |= KQ_ASYNC;
1347 } else {
1348 kq->kq_state &= ~KQ_ASYNC;
1349 }
1350 return (0);
1351
1352 case FIOSETOWN:
1353 return (fsetown(*(int *)data, &kq->kq_sigio));
1354
1355 case FIOGETOWN:
1356 *(int *)data = fgetown(&kq->kq_sigio);
1357 return (0);
1358 }
1359 #endif
1360
1361 return (ENOTTY);
1362 }
1363
1364 /*ARGSUSED*/
1365 static int
1366 kqueue_poll(struct file *fp, int events, struct ucred *active_cred,
1367 struct thread *td)
1368 {
1369 struct kqueue *kq;
1370 int revents = 0;
1371 int error;
1372
1373 if ((error = kqueue_aquire(fp, &kq)))
1374 return POLLERR;
1375
1376 KQ_LOCK(kq);
1377 if (events & (POLLIN | POLLRDNORM)) {
1378 if (kq->kq_count) {
1379 revents |= events & (POLLIN | POLLRDNORM);
1380 } else {
1381 selrecord(td, &kq->kq_sel);
1382 kq->kq_state |= KQ_SEL;
1383 }
1384 }
1385 kqueue_release(kq, 1);
1386 KQ_UNLOCK(kq);
1387 return (revents);
1388 }
1389
1390 /*ARGSUSED*/
1391 static int
1392 kqueue_stat(struct file *fp, struct stat *st, struct ucred *active_cred,
1393 struct thread *td)
1394 {
1395
1396 bzero((void *)st, sizeof *st);
1397 /*
1398 * We no longer return kq_count because the unlocked value is useless.
1399 * If you spent all this time getting the count, why not spend your
1400 * syscall better by calling kevent?
1401 *
1402 * XXX - This is needed for libc_r.
1403 */
1404 st->st_mode = S_IFIFO;
1405 return (0);
1406 }
1407
1408 /*ARGSUSED*/
1409 static int
1410 kqueue_close(struct file *fp, struct thread *td)
1411 {
1412 struct kqueue *kq = fp->f_data;
1413 struct filedesc *fdp;
1414 struct knote *kn;
1415 int i;
1416 int error;
1417
1418 if ((error = kqueue_aquire(fp, &kq)))
1419 return error;
1420
1421 KQ_LOCK(kq);
1422
1423 KASSERT((kq->kq_state & KQ_CLOSING) != KQ_CLOSING,
1424 ("kqueue already closing"));
1425 kq->kq_state |= KQ_CLOSING;
1426 if (kq->kq_refcnt > 1)
1427 msleep(&kq->kq_refcnt, &kq->kq_lock, PSOCK, "kqclose", 0);
1428
1429 KASSERT(kq->kq_refcnt == 1, ("other refs are out there!"));
1430 fdp = kq->kq_fdp;
1431
1432 KASSERT(knlist_empty(&kq->kq_sel.si_note),
1433 ("kqueue's knlist not empty"));
1434
1435 for (i = 0; i < kq->kq_knlistsize; i++) {
1436 while ((kn = SLIST_FIRST(&kq->kq_knlist[i])) != NULL) {
1437 KASSERT((kn->kn_status & KN_INFLUX) == 0,
1438 ("KN_INFLUX set when not suppose to be"));
1439 kn->kn_status |= KN_INFLUX;
1440 KQ_UNLOCK(kq);
1441 if (!(kn->kn_status & KN_DETACHED))
1442 kn->kn_fop->f_detach(kn);
1443 knote_drop(kn, td);
1444 KQ_LOCK(kq);
1445 }
1446 }
1447 if (kq->kq_knhashmask != 0) {
1448 for (i = 0; i <= kq->kq_knhashmask; i++) {
1449 while ((kn = SLIST_FIRST(&kq->kq_knhash[i])) != NULL) {
1450 KASSERT((kn->kn_status & KN_INFLUX) == 0,
1451 ("KN_INFLUX set when not suppose to be"));
1452 kn->kn_status |= KN_INFLUX;
1453 KQ_UNLOCK(kq);
1454 if (!(kn->kn_status & KN_DETACHED))
1455 kn->kn_fop->f_detach(kn);
1456 knote_drop(kn, td);
1457 KQ_LOCK(kq);
1458 }
1459 }
1460 }
1461
1462 if ((kq->kq_state & KQ_TASKSCHED) == KQ_TASKSCHED) {
1463 kq->kq_state |= KQ_TASKDRAIN;
1464 msleep(&kq->kq_state, &kq->kq_lock, PSOCK, "kqtqdr", 0);
1465 }
1466
1467 if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
1468 kq->kq_state &= ~KQ_SEL;
1469 selwakeuppri(&kq->kq_sel, PSOCK);
1470 }
1471
1472 KQ_UNLOCK(kq);
1473
1474 FILEDESC_LOCK_FAST(fdp);
1475 SLIST_REMOVE(&fdp->fd_kqlist, kq, kqueue, kq_list);
1476 FILEDESC_UNLOCK_FAST(fdp);
1477
1478 knlist_destroy(&kq->kq_sel.si_note);
1479 mtx_destroy(&kq->kq_lock);
1480 kq->kq_fdp = NULL;
1481
1482 if (kq->kq_knhash != NULL)
1483 free(kq->kq_knhash, M_KQUEUE);
1484 if (kq->kq_knlist != NULL)
1485 free(kq->kq_knlist, M_KQUEUE);
1486
1487 funsetown(&kq->kq_sigio);
1488 free(kq, M_KQUEUE);
1489 fp->f_data = NULL;
1490
1491 return (0);
1492 }
1493
1494 static void
1495 kqueue_wakeup(struct kqueue *kq)
1496 {
1497 KQ_OWNED(kq);
1498
1499 if ((kq->kq_state & KQ_SLEEP) == KQ_SLEEP) {
1500 kq->kq_state &= ~KQ_SLEEP;
1501 wakeup(kq);
1502 }
1503 if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
1504 kq->kq_state &= ~KQ_SEL;
1505 selwakeuppri(&kq->kq_sel, PSOCK);
1506 }
1507 if (!knlist_empty(&kq->kq_sel.si_note))
1508 kqueue_schedtask(kq);
1509 if ((kq->kq_state & KQ_ASYNC) == KQ_ASYNC) {
1510 pgsigio(&kq->kq_sigio, SIGIO, 0);
1511 }
1512 }
1513
1514 /*
1515 * Walk down a list of knotes, activating them if their event has triggered.
1516 *
1517 * There is a possibility to optimize in the case of one kq watching another.
1518 * Instead of scheduling a task to wake it up, you could pass enough state
1519 * down the chain to make up the parent kqueue. Make this code functional
1520 * first.
1521 */
1522 void
1523 knote(struct knlist *list, long hint, int islocked)
1524 {
1525 struct kqueue *kq;
1526 struct knote *kn;
1527
1528 if (list == NULL)
1529 return;
1530
1531 KNL_ASSERT_LOCK(list, islocked);
1532
1533 if (!islocked)
1534 list->kl_lock(list->kl_lockarg);
1535
1536 /*
1537 * If we unlock the list lock (and set KN_INFLUX), we can eliminate
1538 * the kqueue scheduling, but this will introduce four
1539 * lock/unlock's for each knote to test. If we do, continue to use
1540 * SLIST_FOREACH, SLIST_FOREACH_SAFE is not safe in our case, it is
1541 * only safe if you want to remove the current item, which we are
1542 * not doing.
1543 */
1544 SLIST_FOREACH(kn, &list->kl_list, kn_selnext) {
1545 kq = kn->kn_kq;
1546 if ((kn->kn_status & KN_INFLUX) != KN_INFLUX) {
1547 KQ_LOCK(kq);
1548 if ((kn->kn_status & KN_INFLUX) != KN_INFLUX) {
1549 kn->kn_status |= KN_HASKQLOCK;
1550 if (kn->kn_fop->f_event(kn, hint))
1551 KNOTE_ACTIVATE(kn, 1);
1552 kn->kn_status &= ~KN_HASKQLOCK;
1553 }
1554 KQ_UNLOCK(kq);
1555 }
1556 kq = NULL;
1557 }
1558 if (!islocked)
1559 list->kl_unlock(list->kl_lockarg);
1560 }
1561
1562 /*
1563 * add a knote to a knlist
1564 */
1565 void
1566 knlist_add(struct knlist *knl, struct knote *kn, int islocked)
1567 {
1568 KNL_ASSERT_LOCK(knl, islocked);
1569 KQ_NOTOWNED(kn->kn_kq);
1570 KASSERT((kn->kn_status & (KN_INFLUX|KN_DETACHED)) ==
1571 (KN_INFLUX|KN_DETACHED), ("knote not KN_INFLUX and KN_DETACHED"));
1572 if (!islocked)
1573 knl->kl_lock(knl->kl_lockarg);
1574 SLIST_INSERT_HEAD(&knl->kl_list, kn, kn_selnext);
1575 if (!islocked)
1576 knl->kl_unlock(knl->kl_lockarg);
1577 KQ_LOCK(kn->kn_kq);
1578 kn->kn_knlist = knl;
1579 kn->kn_status &= ~KN_DETACHED;
1580 KQ_UNLOCK(kn->kn_kq);
1581 }
1582
1583 static void
1584 knlist_remove_kq(struct knlist *knl, struct knote *kn, int knlislocked, int kqislocked)
1585 {
1586 KASSERT(!(!!kqislocked && !knlislocked), ("kq locked w/o knl locked"));
1587 KNL_ASSERT_LOCK(knl, knlislocked);
1588 mtx_assert(&kn->kn_kq->kq_lock, kqislocked ? MA_OWNED : MA_NOTOWNED);
1589 if (!kqislocked)
1590 KASSERT((kn->kn_status & (KN_INFLUX|KN_DETACHED)) == KN_INFLUX,
1591 ("knlist_remove called w/o knote being KN_INFLUX or already removed"));
1592 if (!knlislocked)
1593 knl->kl_lock(knl->kl_lockarg);
1594 SLIST_REMOVE(&knl->kl_list, kn, knote, kn_selnext);
1595 kn->kn_knlist = NULL;
1596 if (!knlislocked)
1597 knl->kl_unlock(knl->kl_lockarg);
1598 if (!kqislocked)
1599 KQ_LOCK(kn->kn_kq);
1600 kn->kn_status |= KN_DETACHED;
1601 if (!kqislocked)
1602 KQ_UNLOCK(kn->kn_kq);
1603 }
1604
1605 /*
1606 * remove all knotes from a specified klist
1607 */
1608 void
1609 knlist_remove(struct knlist *knl, struct knote *kn, int islocked)
1610 {
1611
1612 knlist_remove_kq(knl, kn, islocked, 0);
1613 }
1614
1615 /*
1616 * remove knote from a specified klist while in f_event handler.
1617 */
1618 void
1619 knlist_remove_inevent(struct knlist *knl, struct knote *kn)
1620 {
1621
1622 knlist_remove_kq(knl, kn, 1,
1623 (kn->kn_status & KN_HASKQLOCK) == KN_HASKQLOCK);
1624 }
1625
1626 int
1627 knlist_empty(struct knlist *knl)
1628 {
1629 KNL_ASSERT_LOCKED(knl);
1630 return SLIST_EMPTY(&knl->kl_list);
1631 }
1632
1633 static struct mtx knlist_lock;
1634 MTX_SYSINIT(knlist_lock, &knlist_lock, "knlist lock for lockless objects",
1635 MTX_DEF);
1636 static void knlist_mtx_lock(void *arg);
1637 static void knlist_mtx_unlock(void *arg);
1638 static int knlist_mtx_locked(void *arg);
1639
1640 static void
1641 knlist_mtx_lock(void *arg)
1642 {
1643 mtx_lock((struct mtx *)arg);
1644 }
1645
1646 static void
1647 knlist_mtx_unlock(void *arg)
1648 {
1649 mtx_unlock((struct mtx *)arg);
1650 }
1651
1652 static int
1653 knlist_mtx_locked(void *arg)
1654 {
1655 return (mtx_owned((struct mtx *)arg));
1656 }
1657
1658 void
1659 knlist_init(struct knlist *knl, void *lock, void (*kl_lock)(void *),
1660 void (*kl_unlock)(void *), int (*kl_locked)(void *))
1661 {
1662
1663 if (lock == NULL)
1664 knl->kl_lockarg = &knlist_lock;
1665 else
1666 knl->kl_lockarg = lock;
1667
1668 if (kl_lock == NULL)
1669 knl->kl_lock = knlist_mtx_lock;
1670 else
1671 knl->kl_lock = kl_lock;
1672 if (kl_unlock == NULL)
1673 knl->kl_unlock = knlist_mtx_unlock;
1674 else
1675 knl->kl_unlock = kl_unlock;
1676 if (kl_locked == NULL)
1677 knl->kl_locked = knlist_mtx_locked;
1678 else
1679 knl->kl_locked = kl_locked;
1680
1681 SLIST_INIT(&knl->kl_list);
1682 }
1683
1684 void
1685 knlist_destroy(struct knlist *knl)
1686 {
1687
1688 #ifdef INVARIANTS
1689 /*
1690 * if we run across this error, we need to find the offending
1691 * driver and have it call knlist_clear.
1692 */
1693 if (!SLIST_EMPTY(&knl->kl_list))
1694 printf("WARNING: destroying knlist w/ knotes on it!\n");
1695 #endif
1696
1697 knl->kl_lockarg = knl->kl_lock = knl->kl_unlock = NULL;
1698 SLIST_INIT(&knl->kl_list);
1699 }
1700
1701 /*
1702 * Even if we are locked, we may need to drop the lock to allow any influx
1703 * knotes time to "settle".
1704 */
1705 void
1706 knlist_cleardel(struct knlist *knl, struct thread *td, int islocked, int killkn)
1707 {
1708 struct knote *kn, *kn2;
1709 struct kqueue *kq;
1710
1711 if (islocked)
1712 KNL_ASSERT_LOCKED(knl);
1713 else {
1714 KNL_ASSERT_UNLOCKED(knl);
1715 again: /* need to reaquire lock since we have dropped it */
1716 knl->kl_lock(knl->kl_lockarg);
1717 }
1718
1719 SLIST_FOREACH_SAFE(kn, &knl->kl_list, kn_selnext, kn2) {
1720 kq = kn->kn_kq;
1721 KQ_LOCK(kq);
1722 if ((kn->kn_status & KN_INFLUX)) {
1723 KQ_UNLOCK(kq);
1724 continue;
1725 }
1726 knlist_remove_kq(knl, kn, 1, 1);
1727 if (killkn) {
1728 kn->kn_status |= KN_INFLUX | KN_DETACHED;
1729 KQ_UNLOCK(kq);
1730 knote_drop(kn, td);
1731 } else {
1732 /* Make sure cleared knotes disappear soon */
1733 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
1734 KQ_UNLOCK(kq);
1735 }
1736 kq = NULL;
1737 }
1738
1739 if (!SLIST_EMPTY(&knl->kl_list)) {
1740 /* there are still KN_INFLUX remaining */
1741 kn = SLIST_FIRST(&knl->kl_list);
1742 kq = kn->kn_kq;
1743 KQ_LOCK(kq);
1744 KASSERT(kn->kn_status & KN_INFLUX,
1745 ("knote removed w/o list lock"));
1746 knl->kl_unlock(knl->kl_lockarg);
1747 kq->kq_state |= KQ_FLUXWAIT;
1748 msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqkclr", 0);
1749 kq = NULL;
1750 goto again;
1751 }
1752
1753 if (islocked)
1754 KNL_ASSERT_LOCKED(knl);
1755 else {
1756 knl->kl_unlock(knl->kl_lockarg);
1757 KNL_ASSERT_UNLOCKED(knl);
1758 }
1759 }
1760
1761 /*
1762 * remove all knotes referencing a specified fd
1763 * must be called with FILEDESC lock. This prevents a race where a new fd
1764 * comes along and occupies the entry and we attach a knote to the fd.
1765 */
1766 void
1767 knote_fdclose(struct thread *td, int fd)
1768 {
1769 struct filedesc *fdp = td->td_proc->p_fd;
1770 struct kqueue *kq;
1771 struct knote *kn;
1772 int influx;
1773
1774 FILEDESC_LOCK_ASSERT(fdp, MA_OWNED);
1775
1776 /*
1777 * We shouldn't have to worry about new kevents appearing on fd
1778 * since filedesc is locked.
1779 */
1780 SLIST_FOREACH(kq, &fdp->fd_kqlist, kq_list) {
1781 KQ_LOCK(kq);
1782
1783 again:
1784 influx = 0;
1785 while (kq->kq_knlistsize > fd &&
1786 (kn = SLIST_FIRST(&kq->kq_knlist[fd])) != NULL) {
1787 if (kn->kn_status & KN_INFLUX) {
1788 /* someone else might be waiting on our knote */
1789 if (influx)
1790 wakeup(kq);
1791 kq->kq_state |= KQ_FLUXWAIT;
1792 msleep(kq, &kq->kq_lock, PSOCK, "kqflxwt", 0);
1793 goto again;
1794 }
1795 kn->kn_status |= KN_INFLUX;
1796 KQ_UNLOCK(kq);
1797 if (!(kn->kn_status & KN_DETACHED))
1798 kn->kn_fop->f_detach(kn);
1799 knote_drop(kn, td);
1800 influx = 1;
1801 KQ_LOCK(kq);
1802 }
1803 KQ_UNLOCK_FLUX(kq);
1804 }
1805 }
1806
1807 static int
1808 knote_attach(struct knote *kn, struct kqueue *kq)
1809 {
1810 struct klist *list;
1811
1812 KASSERT(kn->kn_status & KN_INFLUX, ("knote not marked INFLUX"));
1813 KQ_OWNED(kq);
1814
1815 if (kn->kn_fop->f_isfd) {
1816 if (kn->kn_id >= kq->kq_knlistsize)
1817 return ENOMEM;
1818 list = &kq->kq_knlist[kn->kn_id];
1819 } else {
1820 if (kq->kq_knhash == NULL)
1821 return ENOMEM;
1822 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
1823 }
1824
1825 SLIST_INSERT_HEAD(list, kn, kn_link);
1826
1827 return 0;
1828 }
1829
1830 /*
1831 * knote must already have been detatched using the f_detach method.
1832 * no lock need to be held, it is assumed that the KN_INFLUX flag is set
1833 * to prevent other removal.
1834 */
1835 static void
1836 knote_drop(struct knote *kn, struct thread *td)
1837 {
1838 struct kqueue *kq;
1839 struct klist *list;
1840
1841 kq = kn->kn_kq;
1842
1843 KQ_NOTOWNED(kq);
1844 KASSERT((kn->kn_status & KN_INFLUX) == KN_INFLUX,
1845 ("knote_drop called without KN_INFLUX set in kn_status"));
1846
1847 KQ_LOCK(kq);
1848 if (kn->kn_fop->f_isfd)
1849 list = &kq->kq_knlist[kn->kn_id];
1850 else
1851 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
1852
1853 SLIST_REMOVE(list, kn, knote, kn_link);
1854 if (kn->kn_status & KN_QUEUED)
1855 knote_dequeue(kn);
1856 KQ_UNLOCK_FLUX(kq);
1857
1858 if (kn->kn_fop->f_isfd) {
1859 fdrop(kn->kn_fp, td);
1860 kn->kn_fp = NULL;
1861 }
1862 kqueue_fo_release(kn->kn_kevent.filter);
1863 kn->kn_fop = NULL;
1864 knote_free(kn);
1865 }
1866
1867 static void
1868 knote_enqueue(struct knote *kn)
1869 {
1870 struct kqueue *kq = kn->kn_kq;
1871
1872 KQ_OWNED(kn->kn_kq);
1873 KASSERT((kn->kn_status & KN_QUEUED) == 0, ("knote already queued"));
1874
1875 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
1876 kn->kn_status |= KN_QUEUED;
1877 kq->kq_count++;
1878 kqueue_wakeup(kq);
1879 }
1880
1881 static void
1882 knote_dequeue(struct knote *kn)
1883 {
1884 struct kqueue *kq = kn->kn_kq;
1885
1886 KQ_OWNED(kn->kn_kq);
1887 KASSERT(kn->kn_status & KN_QUEUED, ("knote not queued"));
1888
1889 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
1890 kn->kn_status &= ~KN_QUEUED;
1891 kq->kq_count--;
1892 }
1893
1894 static void
1895 knote_init(void)
1896 {
1897
1898 knote_zone = uma_zcreate("KNOTE", sizeof(struct knote), NULL, NULL,
1899 NULL, NULL, UMA_ALIGN_PTR, 0);
1900 }
1901 SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL)
1902
1903 static struct knote *
1904 knote_alloc(int waitok)
1905 {
1906 return ((struct knote *)uma_zalloc(knote_zone,
1907 (waitok ? M_WAITOK : M_NOWAIT)|M_ZERO));
1908 }
1909
1910 static void
1911 knote_free(struct knote *kn)
1912 {
1913 if (kn != NULL)
1914 uma_zfree(knote_zone, kn);
1915 }
Cache object: 0251c89daaadc34e9e7a3b65c7383d5b
|