The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/vfs_aio.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1997 John S. Dyson.  All rights reserved.
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. John S. Dyson's name may not be used to endorse or promote products
   10  *    derived from this software without specific prior written permission.
   11  *
   12  * DISCLAIMER:  This code isn't warranted to do anything useful.  Anything
   13  * bad that happens because of using this software isn't the responsibility
   14  * of the author.  This software is distributed AS-IS.
   15  */
   16 
   17 /*
   18  * This file contains support for the POSIX 1003.1B AIO/LIO facility.
   19  */
   20 
   21 #include <sys/cdefs.h>
   22 __FBSDID("$FreeBSD: releng/6.1/sys/kern/vfs_aio.c 158179 2006-04-30 16:44:43Z cvs2svn $");
   23 
   24 #include <sys/param.h>
   25 #include <sys/systm.h>
   26 #include <sys/malloc.h>
   27 #include <sys/bio.h>
   28 #include <sys/buf.h>
   29 #include <sys/eventhandler.h>
   30 #include <sys/sysproto.h>
   31 #include <sys/filedesc.h>
   32 #include <sys/kernel.h>
   33 #include <sys/module.h>
   34 #include <sys/kthread.h>
   35 #include <sys/fcntl.h>
   36 #include <sys/file.h>
   37 #include <sys/limits.h>
   38 #include <sys/lock.h>
   39 #include <sys/mutex.h>
   40 #include <sys/unistd.h>
   41 #include <sys/proc.h>
   42 #include <sys/resourcevar.h>
   43 #include <sys/signalvar.h>
   44 #include <sys/protosw.h>
   45 #include <sys/socketvar.h>
   46 #include <sys/syscall.h>
   47 #include <sys/sysent.h>
   48 #include <sys/sysctl.h>
   49 #include <sys/sx.h>
   50 #include <sys/vnode.h>
   51 #include <sys/conf.h>
   52 #include <sys/event.h>
   53 
   54 #include <posix4/posix4.h>
   55 #include <vm/vm.h>
   56 #include <vm/vm_extern.h>
   57 #include <vm/pmap.h>
   58 #include <vm/vm_map.h>
   59 #include <vm/uma.h>
   60 #include <sys/aio.h>
   61 
   62 #include "opt_vfs_aio.h"
   63 
   64 NET_NEEDS_GIANT("aio");
   65 
   66 /*
   67  * Counter for allocating reference ids to new jobs.  Wrapped to 1 on
   68  * overflow.
   69  */
   70 static  long jobrefid;
   71 
   72 #define JOBST_NULL              0x0
   73 #define JOBST_JOBQGLOBAL        0x2
   74 #define JOBST_JOBRUNNING        0x3
   75 #define JOBST_JOBFINISHED       0x4
   76 #define JOBST_JOBQBUF           0x5
   77 #define JOBST_JOBBFINISHED      0x6
   78 
   79 #ifndef MAX_AIO_PER_PROC
   80 #define MAX_AIO_PER_PROC        32
   81 #endif
   82 
   83 #ifndef MAX_AIO_QUEUE_PER_PROC
   84 #define MAX_AIO_QUEUE_PER_PROC  256 /* Bigger than AIO_LISTIO_MAX */
   85 #endif
   86 
   87 #ifndef MAX_AIO_PROCS
   88 #define MAX_AIO_PROCS           32
   89 #endif
   90 
   91 #ifndef MAX_AIO_QUEUE
   92 #define MAX_AIO_QUEUE           1024 /* Bigger than AIO_LISTIO_MAX */
   93 #endif
   94 
   95 #ifndef TARGET_AIO_PROCS
   96 #define TARGET_AIO_PROCS        4
   97 #endif
   98 
   99 #ifndef MAX_BUF_AIO
  100 #define MAX_BUF_AIO             16
  101 #endif
  102 
  103 #ifndef AIOD_TIMEOUT_DEFAULT
  104 #define AIOD_TIMEOUT_DEFAULT    (10 * hz)
  105 #endif
  106 
  107 #ifndef AIOD_LIFETIME_DEFAULT
  108 #define AIOD_LIFETIME_DEFAULT   (30 * hz)
  109 #endif
  110 
  111 static SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, "Async IO management");
  112 
  113 static int max_aio_procs = MAX_AIO_PROCS;
  114 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_procs,
  115         CTLFLAG_RW, &max_aio_procs, 0,
  116         "Maximum number of kernel threads to use for handling async IO ");
  117 
  118 static int num_aio_procs = 0;
  119 SYSCTL_INT(_vfs_aio, OID_AUTO, num_aio_procs,
  120         CTLFLAG_RD, &num_aio_procs, 0,
  121         "Number of presently active kernel threads for async IO");
  122 
  123 /*
  124  * The code will adjust the actual number of AIO processes towards this
  125  * number when it gets a chance.
  126  */
  127 static int target_aio_procs = TARGET_AIO_PROCS;
  128 SYSCTL_INT(_vfs_aio, OID_AUTO, target_aio_procs, CTLFLAG_RW, &target_aio_procs,
  129         0, "Preferred number of ready kernel threads for async IO");
  130 
  131 static int max_queue_count = MAX_AIO_QUEUE;
  132 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue, CTLFLAG_RW, &max_queue_count, 0,
  133     "Maximum number of aio requests to queue, globally");
  134 
  135 static int num_queue_count = 0;
  136 SYSCTL_INT(_vfs_aio, OID_AUTO, num_queue_count, CTLFLAG_RD, &num_queue_count, 0,
  137     "Number of queued aio requests");
  138 
  139 static int num_buf_aio = 0;
  140 SYSCTL_INT(_vfs_aio, OID_AUTO, num_buf_aio, CTLFLAG_RD, &num_buf_aio, 0,
  141     "Number of aio requests presently handled by the buf subsystem");
  142 
  143 /* Number of async I/O thread in the process of being started */
  144 /* XXX This should be local to _aio_aqueue() */
  145 static int num_aio_resv_start = 0;
  146 
  147 static int aiod_timeout;
  148 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_timeout, CTLFLAG_RW, &aiod_timeout, 0,
  149     "Timeout value for synchronous aio operations");
  150 
  151 static int aiod_lifetime;
  152 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime, CTLFLAG_RW, &aiod_lifetime, 0,
  153     "Maximum lifetime for idle aiod");
  154 
  155 static int unloadable = 0;
  156 SYSCTL_INT(_vfs_aio, OID_AUTO, unloadable, CTLFLAG_RW, &unloadable, 0,
  157     "Allow unload of aio (not recommended)");
  158 
  159 
  160 static int max_aio_per_proc = MAX_AIO_PER_PROC;
  161 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_per_proc, CTLFLAG_RW, &max_aio_per_proc,
  162     0, "Maximum active aio requests per process (stored in the process)");
  163 
  164 static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC;
  165 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW,
  166     &max_aio_queue_per_proc, 0,
  167     "Maximum queued aio requests per process (stored in the process)");
  168 
  169 static int max_buf_aio = MAX_BUF_AIO;
  170 SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW, &max_buf_aio, 0,
  171     "Maximum buf aio requests per process (stored in the process)");
  172 
  173 struct aiocblist {
  174         TAILQ_ENTRY(aiocblist) list;    /* List of jobs */
  175         TAILQ_ENTRY(aiocblist) plist;   /* List of jobs for proc */
  176         int     jobflags;
  177         int     jobstate;
  178         int     inputcharge;
  179         int     outputcharge;
  180         struct  buf *bp;                /* Buffer pointer */
  181         struct  proc *userproc;         /* User process */ /* Not td! */
  182         struct  ucred *cred;            /* Active credential when created */
  183         struct  file *fd_file;          /* Pointer to file structure */
  184         struct  aio_liojob *lio;        /* Optional lio job */
  185         struct  aiocb *uuaiocb;         /* Pointer in userspace of aiocb */
  186         struct  knlist klist;           /* list of knotes */
  187         struct  aiocb uaiocb;           /* Kernel I/O control block */
  188 };
  189 
  190 /* jobflags */
  191 #define AIOCBLIST_RUNDOWN       0x4
  192 #define AIOCBLIST_DONE          0x10
  193 
  194 /*
  195  * AIO process info
  196  */
  197 #define AIOP_FREE       0x1                     /* proc on free queue */
  198 
  199 struct aiothreadlist {
  200         int aiothreadflags;                     /* AIO proc flags */
  201         TAILQ_ENTRY(aiothreadlist) list;        /* List of processes */
  202         struct thread *aiothread;               /* The AIO thread */
  203 };
  204 
  205 /*
  206  * data-structure for lio signal management
  207  */
  208 struct aio_liojob {
  209         int     lioj_flags;
  210         int     lioj_buffer_count;
  211         int     lioj_buffer_finished_count;
  212         int     lioj_queue_count;
  213         int     lioj_queue_finished_count;
  214         struct  sigevent lioj_signal;   /* signal on all I/O done */
  215         TAILQ_ENTRY(aio_liojob) lioj_list;
  216 };
  217 #define LIOJ_SIGNAL             0x1     /* signal on all done (lio) */
  218 #define LIOJ_SIGNAL_POSTED      0x2     /* signal has been posted */
  219 
  220 /*
  221  * per process aio data structure
  222  */
  223 struct kaioinfo {
  224         int     kaio_flags;             /* per process kaio flags */
  225         int     kaio_maxactive_count;   /* maximum number of AIOs */
  226         int     kaio_active_count;      /* number of currently used AIOs */
  227         int     kaio_qallowed_count;    /* maxiumu size of AIO queue */
  228         int     kaio_queue_count;       /* size of AIO queue */
  229         int     kaio_ballowed_count;    /* maximum number of buffers */
  230         int     kaio_queue_finished_count; /* number of daemon jobs finished */
  231         int     kaio_buffer_count;      /* number of physio buffers */
  232         int     kaio_buffer_finished_count; /* count of I/O done */
  233         TAILQ_HEAD(,aio_liojob) kaio_liojoblist; /* list of lio jobs */
  234         TAILQ_HEAD(,aiocblist) kaio_jobqueue;   /* job queue for process */
  235         TAILQ_HEAD(,aiocblist) kaio_jobdone;    /* done queue for process */
  236         TAILQ_HEAD(,aiocblist) kaio_bufqueue;   /* buffer job queue for process */
  237         TAILQ_HEAD(,aiocblist) kaio_bufdone;    /* buffer done queue for process */
  238         TAILQ_HEAD(,aiocblist) kaio_sockqueue;  /* queue for aios waiting on sockets */
  239 };
  240 
  241 #define KAIO_RUNDOWN    0x1     /* process is being run down */
  242 #define KAIO_WAKEUP     0x2     /* wakeup process when there is a significant event */
  243 
  244 static TAILQ_HEAD(,aiothreadlist) aio_freeproc;         /* Idle daemons */
  245 static struct mtx aio_freeproc_mtx;
  246 
  247 static TAILQ_HEAD(,aiocblist) aio_jobs;                 /* Async job list */
  248 
  249 static void     aio_init_aioinfo(struct proc *p);
  250 static void     aio_onceonly(void);
  251 static int      aio_free_entry(struct aiocblist *aiocbe);
  252 static void     aio_process(struct aiocblist *aiocbe);
  253 static int      aio_newproc(void);
  254 static int      aio_aqueue(struct thread *td, struct aiocb *job, int type);
  255 static void     aio_physwakeup(struct buf *bp);
  256 static void     aio_proc_rundown(void *arg, struct proc *p);
  257 static int      aio_fphysio(struct aiocblist *aiocbe);
  258 static int      aio_qphysio(struct proc *p, struct aiocblist *iocb);
  259 static void     aio_daemon(void *uproc);
  260 static void     aio_swake_cb(struct socket *, struct sockbuf *);
  261 static int      aio_unload(void);
  262 static int      filt_aioattach(struct knote *kn);
  263 static void     filt_aiodetach(struct knote *kn);
  264 static int      filt_aio(struct knote *kn, long hint);
  265 
  266 /*
  267  * Zones for:
  268  *      kaio    Per process async io info
  269  *      aiop    async io thread data
  270  *      aiocb   async io jobs
  271  *      aiol    list io job pointer - internal to aio_suspend XXX
  272  *      aiolio  list io jobs
  273  */
  274 static uma_zone_t kaio_zone, aiop_zone, aiocb_zone, aiol_zone, aiolio_zone;
  275 
  276 /* kqueue filters for aio */
  277 static struct filterops aio_filtops =
  278         { 0, filt_aioattach, filt_aiodetach, filt_aio };
  279 
  280 static eventhandler_tag exit_tag, exec_tag;
  281 
  282 /*
  283  * Main operations function for use as a kernel module.
  284  */
  285 static int
  286 aio_modload(struct module *module, int cmd, void *arg)
  287 {
  288         int error = 0;
  289 
  290         switch (cmd) {
  291         case MOD_LOAD:
  292                 aio_onceonly();
  293                 break;
  294         case MOD_UNLOAD:
  295                 error = aio_unload();
  296                 break;
  297         case MOD_SHUTDOWN:
  298                 break;
  299         default:
  300                 error = EINVAL;
  301                 break;
  302         }
  303         return (error);
  304 }
  305 
  306 static moduledata_t aio_mod = {
  307         "aio",
  308         &aio_modload,
  309         NULL
  310 };
  311 
  312 SYSCALL_MODULE_HELPER(aio_return);
  313 SYSCALL_MODULE_HELPER(aio_suspend);
  314 SYSCALL_MODULE_HELPER(aio_cancel);
  315 SYSCALL_MODULE_HELPER(aio_error);
  316 SYSCALL_MODULE_HELPER(aio_read);
  317 SYSCALL_MODULE_HELPER(aio_write);
  318 SYSCALL_MODULE_HELPER(aio_waitcomplete);
  319 SYSCALL_MODULE_HELPER(lio_listio);
  320 
  321 DECLARE_MODULE(aio, aio_mod,
  322         SI_SUB_VFS, SI_ORDER_ANY);
  323 MODULE_VERSION(aio, 1);
  324 
  325 /*
  326  * Startup initialization
  327  */
  328 static void
  329 aio_onceonly(void)
  330 {
  331 
  332         /* XXX: should probably just use so->callback */
  333         aio_swake = &aio_swake_cb;
  334         exit_tag = EVENTHANDLER_REGISTER(process_exit, aio_proc_rundown, NULL,
  335             EVENTHANDLER_PRI_ANY);
  336         exec_tag = EVENTHANDLER_REGISTER(process_exec, aio_proc_rundown, NULL,
  337             EVENTHANDLER_PRI_ANY);
  338         kqueue_add_filteropts(EVFILT_AIO, &aio_filtops);
  339         TAILQ_INIT(&aio_freeproc);
  340         mtx_init(&aio_freeproc_mtx, "aio_freeproc", NULL, MTX_DEF);
  341         TAILQ_INIT(&aio_jobs);
  342         kaio_zone = uma_zcreate("AIO", sizeof(struct kaioinfo), NULL, NULL,
  343             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  344         aiop_zone = uma_zcreate("AIOP", sizeof(struct aiothreadlist), NULL,
  345             NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  346         aiocb_zone = uma_zcreate("AIOCB", sizeof(struct aiocblist), NULL, NULL,
  347             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  348         aiol_zone = uma_zcreate("AIOL", AIO_LISTIO_MAX*sizeof(intptr_t) , NULL,
  349             NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  350         aiolio_zone = uma_zcreate("AIOLIO", sizeof(struct aio_liojob), NULL,
  351             NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  352         aiod_timeout = AIOD_TIMEOUT_DEFAULT;
  353         aiod_lifetime = AIOD_LIFETIME_DEFAULT;
  354         jobrefid = 1;
  355         async_io_version = _POSIX_VERSION;
  356         p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, AIO_LISTIO_MAX);
  357         p31b_setcfg(CTL_P1003_1B_AIO_MAX, MAX_AIO_QUEUE);
  358         p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, 0);
  359 }
  360 
  361 /*
  362  * Callback for unload of AIO when used as a module.
  363  */
  364 static int
  365 aio_unload(void)
  366 {
  367         int error;
  368 
  369         /*
  370          * XXX: no unloads by default, it's too dangerous.
  371          * perhaps we could do it if locked out callers and then
  372          * did an aio_proc_rundown() on each process.
  373          */
  374         if (!unloadable)
  375                 return (EOPNOTSUPP);
  376 
  377         error = kqueue_del_filteropts(EVFILT_AIO);
  378         if (error)
  379                 return error;
  380 
  381         async_io_version = 0;
  382         aio_swake = NULL;
  383         EVENTHANDLER_DEREGISTER(process_exit, exit_tag);
  384         EVENTHANDLER_DEREGISTER(process_exec, exec_tag);
  385         p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, -1);
  386         p31b_setcfg(CTL_P1003_1B_AIO_MAX, -1);
  387         p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, -1);
  388         return (0);
  389 }
  390 
  391 /*
  392  * Init the per-process aioinfo structure.  The aioinfo limits are set
  393  * per-process for user limit (resource) management.
  394  */
  395 static void
  396 aio_init_aioinfo(struct proc *p)
  397 {
  398         struct kaioinfo *ki;
  399 
  400         ki = uma_zalloc(kaio_zone, M_WAITOK);
  401         ki->kaio_flags = 0;
  402         ki->kaio_maxactive_count = max_aio_per_proc;
  403         ki->kaio_active_count = 0;
  404         ki->kaio_qallowed_count = max_aio_queue_per_proc;
  405         ki->kaio_queue_count = 0;
  406         ki->kaio_ballowed_count = max_buf_aio;
  407         ki->kaio_buffer_count = 0;
  408         ki->kaio_buffer_finished_count = 0;
  409         TAILQ_INIT(&ki->kaio_jobdone);
  410         TAILQ_INIT(&ki->kaio_jobqueue);
  411         TAILQ_INIT(&ki->kaio_bufdone);
  412         TAILQ_INIT(&ki->kaio_bufqueue);
  413         TAILQ_INIT(&ki->kaio_liojoblist);
  414         TAILQ_INIT(&ki->kaio_sockqueue);
  415         PROC_LOCK(p);
  416         if (p->p_aioinfo == NULL) {
  417                 p->p_aioinfo = ki;
  418                 PROC_UNLOCK(p);
  419         } else {
  420                 PROC_UNLOCK(p);
  421                 uma_zfree(kaio_zone, ki);
  422         }
  423 
  424         while (num_aio_procs < target_aio_procs)
  425                 aio_newproc();
  426 }
  427 
  428 /*
  429  * Free a job entry.  Wait for completion if it is currently active, but don't
  430  * delay forever.  If we delay, we return a flag that says that we have to
  431  * restart the queue scan.
  432  */
  433 static int
  434 aio_free_entry(struct aiocblist *aiocbe)
  435 {
  436         struct kaioinfo *ki;
  437         struct aio_liojob *lj;
  438         struct proc *p;
  439         int error;
  440         int s;
  441 
  442         if (aiocbe->jobstate == JOBST_NULL)
  443                 panic("aio_free_entry: freeing already free job");
  444 
  445         p = aiocbe->userproc;
  446         ki = p->p_aioinfo;
  447         lj = aiocbe->lio;
  448         if (ki == NULL)
  449                 panic("aio_free_entry: missing p->p_aioinfo");
  450 
  451         while (aiocbe->jobstate == JOBST_JOBRUNNING) {
  452                 aiocbe->jobflags |= AIOCBLIST_RUNDOWN;
  453                 tsleep(aiocbe, PRIBIO, "jobwai", 0);
  454         }
  455         if (aiocbe->bp == NULL) {
  456                 if (ki->kaio_queue_count <= 0)
  457                         panic("aio_free_entry: process queue size <= 0");
  458                 if (num_queue_count <= 0)
  459                         panic("aio_free_entry: system wide queue size <= 0");
  460 
  461                 if (lj) {
  462                         lj->lioj_queue_count--;
  463                         if (aiocbe->jobflags & AIOCBLIST_DONE)
  464                                 lj->lioj_queue_finished_count--;
  465                 }
  466                 ki->kaio_queue_count--;
  467                 if (aiocbe->jobflags & AIOCBLIST_DONE)
  468                         ki->kaio_queue_finished_count--;
  469                 num_queue_count--;
  470         } else {
  471                 if (lj) {
  472                         lj->lioj_buffer_count--;
  473                         if (aiocbe->jobflags & AIOCBLIST_DONE)
  474                                 lj->lioj_buffer_finished_count--;
  475                 }
  476                 if (aiocbe->jobflags & AIOCBLIST_DONE)
  477                         ki->kaio_buffer_finished_count--;
  478                 ki->kaio_buffer_count--;
  479                 num_buf_aio--;
  480         }
  481 
  482         /* aiocbe is going away, we need to destroy any knotes */
  483         /* XXXKSE Note the thread here is used to eventually find the
  484          * owning process again, but it is also used to do a fo_close
  485          * and that requires the thread. (but does it require the
  486          * OWNING thread? (or maybe the running thread?)
  487          * There is a semantic problem here...
  488          */
  489         knlist_delete(&aiocbe->klist, FIRST_THREAD_IN_PROC(p), 0); /* XXXKSE */
  490 
  491         if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags & KAIO_RUNDOWN)
  492             && ((ki->kaio_buffer_count == 0) && (ki->kaio_queue_count == 0)))) {
  493                 ki->kaio_flags &= ~KAIO_WAKEUP;
  494                 wakeup(p);
  495         }
  496 
  497         if (aiocbe->jobstate == JOBST_JOBQBUF) {
  498                 if ((error = aio_fphysio(aiocbe)) != 0)
  499                         return (error);
  500                 if (aiocbe->jobstate != JOBST_JOBBFINISHED)
  501                         panic("aio_free_entry: invalid physio finish-up state");
  502                 s = splbio();
  503                 TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist);
  504                 splx(s);
  505         } else if (aiocbe->jobstate == JOBST_JOBQGLOBAL) {
  506                 s = splnet();
  507                 TAILQ_REMOVE(&aio_jobs, aiocbe, list);
  508                 TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
  509                 splx(s);
  510         } else if (aiocbe->jobstate == JOBST_JOBFINISHED)
  511                 TAILQ_REMOVE(&ki->kaio_jobdone, aiocbe, plist);
  512         else if (aiocbe->jobstate == JOBST_JOBBFINISHED) {
  513                 s = splbio();
  514                 TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist);
  515                 splx(s);
  516                 if (aiocbe->bp) {
  517                         vunmapbuf(aiocbe->bp);
  518                         relpbuf(aiocbe->bp, NULL);
  519                         aiocbe->bp = NULL;
  520                 }
  521         }
  522         if (lj && (lj->lioj_buffer_count == 0) && (lj->lioj_queue_count == 0)) {
  523                 TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
  524                 uma_zfree(aiolio_zone, lj);
  525         }
  526         aiocbe->jobstate = JOBST_NULL;
  527         fdrop(aiocbe->fd_file, curthread);
  528         crfree(aiocbe->cred);
  529         uma_zfree(aiocb_zone, aiocbe);
  530         return (0);
  531 }
  532 
  533 /*
  534  * Rundown the jobs for a given process.
  535  */
  536 static void
  537 aio_proc_rundown(void *arg, struct proc *p)
  538 {
  539         int s;
  540         struct kaioinfo *ki;
  541         struct aio_liojob *lj, *ljn;
  542         struct aiocblist *aiocbe, *aiocbn;
  543         struct file *fp;
  544         struct socket *so;
  545 
  546         ki = p->p_aioinfo;
  547         if (ki == NULL)
  548                 return;
  549 
  550         mtx_lock(&Giant);
  551         ki->kaio_flags |= LIOJ_SIGNAL_POSTED;
  552         while ((ki->kaio_active_count > 0) || (ki->kaio_buffer_count >
  553             ki->kaio_buffer_finished_count)) {
  554                 ki->kaio_flags |= KAIO_RUNDOWN;
  555                 if (tsleep(p, PRIBIO, "kaiowt", aiod_timeout))
  556                         break;
  557         }
  558 
  559         /*
  560          * Move any aio ops that are waiting on socket I/O to the normal job
  561          * queues so they are cleaned up with any others.
  562          */
  563         s = splnet();
  564         for (aiocbe = TAILQ_FIRST(&ki->kaio_sockqueue); aiocbe; aiocbe =
  565             aiocbn) {
  566                 aiocbn = TAILQ_NEXT(aiocbe, plist);
  567                 fp = aiocbe->fd_file;
  568                 if (fp != NULL) {
  569                         so = fp->f_data;
  570                         TAILQ_REMOVE(&so->so_aiojobq, aiocbe, list);
  571                         if (TAILQ_EMPTY(&so->so_aiojobq)) {
  572                                 SOCKBUF_LOCK(&so->so_snd);
  573                                 so->so_snd.sb_flags &= ~SB_AIO;
  574                                 SOCKBUF_UNLOCK(&so->so_snd);
  575                                 SOCKBUF_LOCK(&so->so_rcv);
  576                                 so->so_rcv.sb_flags &= ~SB_AIO;
  577                                 SOCKBUF_UNLOCK(&so->so_rcv);
  578                         }
  579                 }
  580                 TAILQ_REMOVE(&ki->kaio_sockqueue, aiocbe, plist);
  581                 TAILQ_INSERT_HEAD(&aio_jobs, aiocbe, list);
  582                 TAILQ_INSERT_HEAD(&ki->kaio_jobqueue, aiocbe, plist);
  583         }
  584         splx(s);
  585 
  586 restart1:
  587         for (aiocbe = TAILQ_FIRST(&ki->kaio_jobdone); aiocbe; aiocbe = aiocbn) {
  588                 aiocbn = TAILQ_NEXT(aiocbe, plist);
  589                 if (aio_free_entry(aiocbe))
  590                         goto restart1;
  591         }
  592 
  593 restart2:
  594         for (aiocbe = TAILQ_FIRST(&ki->kaio_jobqueue); aiocbe; aiocbe =
  595             aiocbn) {
  596                 aiocbn = TAILQ_NEXT(aiocbe, plist);
  597                 if (aio_free_entry(aiocbe))
  598                         goto restart2;
  599         }
  600 
  601 /*
  602  * Note the use of lots of splbio here, trying to avoid splbio for long chains
  603  * of I/O.  Probably unnecessary.
  604  */
  605 restart3:
  606         s = splbio();
  607         while (TAILQ_FIRST(&ki->kaio_bufqueue)) {
  608                 ki->kaio_flags |= KAIO_WAKEUP;
  609                 tsleep(p, PRIBIO, "aioprn", 0);
  610                 splx(s);
  611                 goto restart3;
  612         }
  613         splx(s);
  614 
  615 restart4:
  616         s = splbio();
  617         for (aiocbe = TAILQ_FIRST(&ki->kaio_bufdone); aiocbe; aiocbe = aiocbn) {
  618                 aiocbn = TAILQ_NEXT(aiocbe, plist);
  619                 if (aio_free_entry(aiocbe)) {
  620                         splx(s);
  621                         goto restart4;
  622                 }
  623         }
  624         splx(s);
  625 
  626         /*
  627          * If we've slept, jobs might have moved from one queue to another.
  628          * Retry rundown if we didn't manage to empty the queues.
  629          */
  630         if (TAILQ_FIRST(&ki->kaio_jobdone) != NULL ||
  631             TAILQ_FIRST(&ki->kaio_jobqueue) != NULL ||
  632             TAILQ_FIRST(&ki->kaio_bufqueue) != NULL ||
  633             TAILQ_FIRST(&ki->kaio_bufdone) != NULL)
  634                 goto restart1;
  635 
  636         for (lj = TAILQ_FIRST(&ki->kaio_liojoblist); lj; lj = ljn) {
  637                 ljn = TAILQ_NEXT(lj, lioj_list);
  638                 if ((lj->lioj_buffer_count == 0) && (lj->lioj_queue_count ==
  639                     0)) {
  640                         TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
  641                         uma_zfree(aiolio_zone, lj);
  642                 } else {
  643 #ifdef DIAGNOSTIC
  644                         printf("LIO job not cleaned up: B:%d, BF:%d, Q:%d, "
  645                             "QF:%d\n", lj->lioj_buffer_count,
  646                             lj->lioj_buffer_finished_count,
  647                             lj->lioj_queue_count,
  648                             lj->lioj_queue_finished_count);
  649 #endif
  650                 }
  651         }
  652 
  653         uma_zfree(kaio_zone, ki);
  654         p->p_aioinfo = NULL;
  655         mtx_unlock(&Giant);
  656 }
  657 
  658 /*
  659  * Select a job to run (called by an AIO daemon).
  660  */
  661 static struct aiocblist *
  662 aio_selectjob(struct aiothreadlist *aiop)
  663 {
  664         int s;
  665         struct aiocblist *aiocbe;
  666         struct kaioinfo *ki;
  667         struct proc *userp;
  668 
  669         s = splnet();
  670         for (aiocbe = TAILQ_FIRST(&aio_jobs); aiocbe; aiocbe =
  671             TAILQ_NEXT(aiocbe, list)) {
  672                 userp = aiocbe->userproc;
  673                 ki = userp->p_aioinfo;
  674 
  675                 if (ki->kaio_active_count < ki->kaio_maxactive_count) {
  676                         TAILQ_REMOVE(&aio_jobs, aiocbe, list);
  677                         splx(s);
  678                         return (aiocbe);
  679                 }
  680         }
  681         splx(s);
  682 
  683         return (NULL);
  684 }
  685 
  686 /*
  687  * The AIO processing activity.  This is the code that does the I/O request for
  688  * the non-physio version of the operations.  The normal vn operations are used,
  689  * and this code should work in all instances for every type of file, including
  690  * pipes, sockets, fifos, and regular files.
  691  */
  692 static void
  693 aio_process(struct aiocblist *aiocbe)
  694 {
  695         struct ucred *td_savedcred;
  696         struct thread *td;
  697         struct proc *mycp;
  698         struct aiocb *cb;
  699         struct file *fp;
  700         struct uio auio;
  701         struct iovec aiov;
  702         int cnt;
  703         int error;
  704         int oublock_st, oublock_end;
  705         int inblock_st, inblock_end;
  706 
  707         td = curthread;
  708         td_savedcred = td->td_ucred;
  709         td->td_ucred = aiocbe->cred;
  710         mycp = td->td_proc;
  711         cb = &aiocbe->uaiocb;
  712         fp = aiocbe->fd_file;
  713 
  714         aiov.iov_base = (void *)(uintptr_t)cb->aio_buf;
  715         aiov.iov_len = cb->aio_nbytes;
  716 
  717         auio.uio_iov = &aiov;
  718         auio.uio_iovcnt = 1;
  719         auio.uio_offset = cb->aio_offset;
  720         auio.uio_resid = cb->aio_nbytes;
  721         cnt = cb->aio_nbytes;
  722         auio.uio_segflg = UIO_USERSPACE;
  723         auio.uio_td = td;
  724 
  725         inblock_st = mycp->p_stats->p_ru.ru_inblock;
  726         oublock_st = mycp->p_stats->p_ru.ru_oublock;
  727         /*
  728          * _aio_aqueue() acquires a reference to the file that is
  729          * released in aio_free_entry().
  730          */
  731         if (cb->aio_lio_opcode == LIO_READ) {
  732                 auio.uio_rw = UIO_READ;
  733                 error = fo_read(fp, &auio, fp->f_cred, FOF_OFFSET, td);
  734         } else {
  735                 auio.uio_rw = UIO_WRITE;
  736                 error = fo_write(fp, &auio, fp->f_cred, FOF_OFFSET, td);
  737         }
  738         inblock_end = mycp->p_stats->p_ru.ru_inblock;
  739         oublock_end = mycp->p_stats->p_ru.ru_oublock;
  740 
  741         aiocbe->inputcharge = inblock_end - inblock_st;
  742         aiocbe->outputcharge = oublock_end - oublock_st;
  743 
  744         if ((error) && (auio.uio_resid != cnt)) {
  745                 if (error == ERESTART || error == EINTR || error == EWOULDBLOCK)
  746                         error = 0;
  747                 if ((error == EPIPE) && (cb->aio_lio_opcode == LIO_WRITE)) {
  748                         PROC_LOCK(aiocbe->userproc);
  749                         psignal(aiocbe->userproc, SIGPIPE);
  750                         PROC_UNLOCK(aiocbe->userproc);
  751                 }
  752         }
  753 
  754         cnt -= auio.uio_resid;
  755         cb->_aiocb_private.error = error;
  756         cb->_aiocb_private.status = cnt;
  757         td->td_ucred = td_savedcred;
  758 }
  759 
  760 /*
  761  * The AIO daemon, most of the actual work is done in aio_process,
  762  * but the setup (and address space mgmt) is done in this routine.
  763  */
  764 static void
  765 aio_daemon(void *uproc)
  766 {
  767         int s;
  768         struct aio_liojob *lj;
  769         struct aiocb *cb;
  770         struct aiocblist *aiocbe;
  771         struct aiothreadlist *aiop;
  772         struct kaioinfo *ki;
  773         struct proc *curcp, *mycp, *userp;
  774         struct vmspace *myvm, *tmpvm;
  775         struct thread *td = curthread;
  776         struct pgrp *newpgrp;
  777         struct session *newsess;
  778 
  779         /*
  780          * Local copies of curproc (cp) and vmspace (myvm)
  781          */
  782         mycp = td->td_proc;
  783         myvm = mycp->p_vmspace;
  784 
  785         KASSERT(mycp->p_textvp == NULL, ("kthread has a textvp"));
  786 
  787         /*
  788          * Allocate and ready the aio control info.  There is one aiop structure
  789          * per daemon.
  790          */
  791         aiop = uma_zalloc(aiop_zone, M_WAITOK);
  792         aiop->aiothread = td;
  793         aiop->aiothreadflags |= AIOP_FREE;
  794 
  795         /*
  796          * Place thread (lightweight process) onto the AIO free thread list.
  797          */
  798         mtx_lock(&aio_freeproc_mtx);
  799         TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
  800         mtx_unlock(&aio_freeproc_mtx);
  801 
  802         /*
  803          * Get rid of our current filedescriptors.  AIOD's don't need any
  804          * filedescriptors, except as temporarily inherited from the client.
  805          */
  806         fdfree(td);
  807 
  808         /* The daemon resides in its own pgrp. */
  809         MALLOC(newpgrp, struct pgrp *, sizeof(struct pgrp), M_PGRP,
  810                 M_WAITOK | M_ZERO);
  811         MALLOC(newsess, struct session *, sizeof(struct session), M_SESSION,
  812                 M_WAITOK | M_ZERO);
  813 
  814         sx_xlock(&proctree_lock);
  815         enterpgrp(mycp, mycp->p_pid, newpgrp, newsess);
  816         sx_xunlock(&proctree_lock);
  817         mtx_lock(&Giant);
  818 
  819         /*
  820          * Wakeup parent process.  (Parent sleeps to keep from blasting away
  821          * and creating too many daemons.)
  822          */
  823         wakeup(mycp);
  824 
  825         for (;;) {
  826                 /*
  827                  * curcp is the current daemon process context.
  828                  * userp is the current user process context.
  829                  */
  830                 curcp = mycp;
  831 
  832                 /*
  833                  * Take daemon off of free queue
  834                  */
  835                 mtx_lock(&aio_freeproc_mtx);
  836                 if (aiop->aiothreadflags & AIOP_FREE) {
  837                         TAILQ_REMOVE(&aio_freeproc, aiop, list);
  838                         aiop->aiothreadflags &= ~AIOP_FREE;
  839                 }
  840                 mtx_unlock(&aio_freeproc_mtx);
  841 
  842                 /*
  843                  * Check for jobs.
  844                  */
  845                 while ((aiocbe = aio_selectjob(aiop)) != NULL) {
  846                         cb = &aiocbe->uaiocb;
  847                         userp = aiocbe->userproc;
  848 
  849                         aiocbe->jobstate = JOBST_JOBRUNNING;
  850 
  851                         /*
  852                          * Connect to process address space for user program.
  853                          */
  854                         if (userp != curcp) {
  855                                 /*
  856                                  * Save the current address space that we are
  857                                  * connected to.
  858                                  */
  859                                 tmpvm = mycp->p_vmspace;
  860 
  861                                 /*
  862                                  * Point to the new user address space, and
  863                                  * refer to it.
  864                                  */
  865                                 mycp->p_vmspace = userp->p_vmspace;
  866                                 atomic_add_int(&mycp->p_vmspace->vm_refcnt, 1);
  867 
  868                                 /* Activate the new mapping. */
  869                                 pmap_activate(FIRST_THREAD_IN_PROC(mycp));
  870 
  871                                 /*
  872                                  * If the old address space wasn't the daemons
  873                                  * own address space, then we need to remove the
  874                                  * daemon's reference from the other process
  875                                  * that it was acting on behalf of.
  876                                  */
  877                                 if (tmpvm != myvm) {
  878                                         vmspace_free(tmpvm);
  879                                 }
  880                                 curcp = userp;
  881                         }
  882 
  883                         ki = userp->p_aioinfo;
  884                         lj = aiocbe->lio;
  885 
  886                         /* Account for currently active jobs. */
  887                         ki->kaio_active_count++;
  888 
  889                         /* Do the I/O function. */
  890                         aio_process(aiocbe);
  891 
  892                         /* Decrement the active job count. */
  893                         ki->kaio_active_count--;
  894 
  895                         /*
  896                          * Increment the completion count for wakeup/signal
  897                          * comparisons.
  898                          */
  899                         aiocbe->jobflags |= AIOCBLIST_DONE;
  900                         ki->kaio_queue_finished_count++;
  901                         if (lj)
  902                                 lj->lioj_queue_finished_count++;
  903                         if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags
  904                             & KAIO_RUNDOWN) && (ki->kaio_active_count == 0))) {
  905                                 ki->kaio_flags &= ~KAIO_WAKEUP;
  906                                 wakeup(userp);
  907                         }
  908 
  909                         s = splbio();
  910                         if (lj && (lj->lioj_flags &
  911                             (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) == LIOJ_SIGNAL) {
  912                                 if ((lj->lioj_queue_finished_count ==
  913                                     lj->lioj_queue_count) &&
  914                                     (lj->lioj_buffer_finished_count ==
  915                                     lj->lioj_buffer_count)) {
  916                                         PROC_LOCK(userp);
  917                                         psignal(userp,
  918                                             lj->lioj_signal.sigev_signo);
  919                                         PROC_UNLOCK(userp);
  920                                         lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
  921                                 }
  922                         }
  923                         splx(s);
  924 
  925                         aiocbe->jobstate = JOBST_JOBFINISHED;
  926 
  927                         s = splnet();
  928                         TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
  929                         TAILQ_INSERT_TAIL(&ki->kaio_jobdone, aiocbe, plist);
  930                         splx(s);
  931                         KNOTE_UNLOCKED(&aiocbe->klist, 0);
  932 
  933                         if (aiocbe->jobflags & AIOCBLIST_RUNDOWN) {
  934                                 wakeup(aiocbe);
  935                                 aiocbe->jobflags &= ~AIOCBLIST_RUNDOWN;
  936                         }
  937 
  938                         if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) {
  939                                 PROC_LOCK(userp);
  940                                 psignal(userp, cb->aio_sigevent.sigev_signo);
  941                                 PROC_UNLOCK(userp);
  942                         }
  943                 }
  944 
  945                 /*
  946                  * Disconnect from user address space.
  947                  */
  948                 if (curcp != mycp) {
  949                         /* Get the user address space to disconnect from. */
  950                         tmpvm = mycp->p_vmspace;
  951 
  952                         /* Get original address space for daemon. */
  953                         mycp->p_vmspace = myvm;
  954 
  955                         /* Activate the daemon's address space. */
  956                         pmap_activate(FIRST_THREAD_IN_PROC(mycp));
  957 #ifdef DIAGNOSTIC
  958                         if (tmpvm == myvm) {
  959                                 printf("AIOD: vmspace problem -- %d\n",
  960                                     mycp->p_pid);
  961                         }
  962 #endif
  963                         /* Remove our vmspace reference. */
  964                         vmspace_free(tmpvm);
  965 
  966                         curcp = mycp;
  967                 }
  968 
  969                 mtx_lock(&aio_freeproc_mtx);
  970                 TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
  971                 aiop->aiothreadflags |= AIOP_FREE;
  972 
  973                 /*
  974                  * If daemon is inactive for a long time, allow it to exit,
  975                  * thereby freeing resources.
  976                  */
  977                 if (msleep(aiop->aiothread, &aio_freeproc_mtx, PDROP | PRIBIO,
  978                     "aiordy", aiod_lifetime)) {
  979                         s = splnet();
  980                         if (TAILQ_EMPTY(&aio_jobs)) {
  981                                 mtx_lock(&aio_freeproc_mtx);
  982                                 if ((aiop->aiothreadflags & AIOP_FREE) &&
  983                                     (num_aio_procs > target_aio_procs)) {
  984                                         TAILQ_REMOVE(&aio_freeproc, aiop, list);
  985                                         mtx_unlock(&aio_freeproc_mtx);
  986                                         splx(s);
  987                                         uma_zfree(aiop_zone, aiop);
  988                                         num_aio_procs--;
  989 #ifdef DIAGNOSTIC
  990                                         if (mycp->p_vmspace->vm_refcnt <= 1) {
  991                                                 printf("AIOD: bad vm refcnt for"
  992                                                     " exiting daemon: %d\n",
  993                                                     mycp->p_vmspace->vm_refcnt);
  994                                         }
  995 #endif
  996                                         kthread_exit(0);
  997                                 }
  998                                 mtx_unlock(&aio_freeproc_mtx);
  999                         }
 1000                         splx(s);
 1001                 }
 1002         }
 1003 }
 1004 
 1005 /*
 1006  * Create a new AIO daemon.  This is mostly a kernel-thread fork routine.  The
 1007  * AIO daemon modifies its environment itself.
 1008  */
 1009 static int
 1010 aio_newproc(void)
 1011 {
 1012         int error;
 1013         struct proc *p;
 1014 
 1015         error = kthread_create(aio_daemon, curproc, &p, RFNOWAIT, 0, "aiod%d",
 1016             num_aio_procs);
 1017         if (error)
 1018                 return (error);
 1019 
 1020         /*
 1021          * Wait until daemon is started, but continue on just in case to
 1022          * handle error conditions.
 1023          */
 1024         error = tsleep(p, PZERO, "aiosta", aiod_timeout);
 1025 
 1026         num_aio_procs++;
 1027 
 1028         return (error);
 1029 }
 1030 
 1031 /*
 1032  * Try the high-performance, low-overhead physio method for eligible
 1033  * VCHR devices.  This method doesn't use an aio helper thread, and
 1034  * thus has very low overhead.
 1035  *
 1036  * Assumes that the caller, _aio_aqueue(), has incremented the file
 1037  * structure's reference count, preventing its deallocation for the
 1038  * duration of this call.
 1039  */
 1040 static int
 1041 aio_qphysio(struct proc *p, struct aiocblist *aiocbe)
 1042 {
 1043         int error;
 1044         struct aiocb *cb;
 1045         struct file *fp;
 1046         struct buf *bp;
 1047         struct vnode *vp;
 1048         struct kaioinfo *ki;
 1049         struct aio_liojob *lj;
 1050         int s;
 1051         int notify;
 1052 
 1053         cb = &aiocbe->uaiocb;
 1054         fp = aiocbe->fd_file;
 1055 
 1056         if (fp->f_type != DTYPE_VNODE)
 1057                 return (-1);
 1058 
 1059         vp = fp->f_vnode;
 1060 
 1061         /*
 1062          * If its not a disk, we don't want to return a positive error.
 1063          * It causes the aio code to not fall through to try the thread
 1064          * way when you're talking to a regular file.
 1065          */
 1066         if (!vn_isdisk(vp, &error)) {
 1067                 if (error == ENOTBLK)
 1068                         return (-1);
 1069                 else
 1070                         return (error);
 1071         }
 1072 
 1073         if (vp->v_bufobj.bo_bsize == 0)
 1074                 return (-1);
 1075 
 1076         if (cb->aio_nbytes % vp->v_bufobj.bo_bsize)
 1077                 return (-1);
 1078 
 1079         if (cb->aio_nbytes >
 1080             MAXPHYS - (((vm_offset_t) cb->aio_buf) & PAGE_MASK))
 1081                 return (-1);
 1082 
 1083         ki = p->p_aioinfo;
 1084         if (ki->kaio_buffer_count >= ki->kaio_ballowed_count)
 1085                 return (-1);
 1086 
 1087         ki->kaio_buffer_count++;
 1088 
 1089         lj = aiocbe->lio;
 1090         if (lj)
 1091                 lj->lioj_buffer_count++;
 1092 
 1093         /* Create and build a buffer header for a transfer. */
 1094         bp = (struct buf *)getpbuf(NULL);
 1095         BUF_KERNPROC(bp);
 1096 
 1097         /*
 1098          * Get a copy of the kva from the physical buffer.
 1099          */
 1100         error = 0;
 1101 
 1102         bp->b_bcount = cb->aio_nbytes;
 1103         bp->b_bufsize = cb->aio_nbytes;
 1104         bp->b_iodone = aio_physwakeup;
 1105         bp->b_saveaddr = bp->b_data;
 1106         bp->b_data = (void *)(uintptr_t)cb->aio_buf;
 1107         bp->b_offset = cb->aio_offset;
 1108         bp->b_iooffset = cb->aio_offset;
 1109         bp->b_blkno = btodb(cb->aio_offset);
 1110         bp->b_iocmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ;
 1111 
 1112         /*
 1113          * Bring buffer into kernel space.
 1114          */
 1115         if (vmapbuf(bp) < 0) {
 1116                 error = EFAULT;
 1117                 goto doerror;
 1118         }
 1119 
 1120         s = splbio();
 1121         aiocbe->bp = bp;
 1122         bp->b_caller1 = (void *)aiocbe;
 1123         TAILQ_INSERT_TAIL(&ki->kaio_bufqueue, aiocbe, plist);
 1124         aiocbe->jobstate = JOBST_JOBQBUF;
 1125         cb->_aiocb_private.status = cb->aio_nbytes;
 1126         num_buf_aio++;
 1127         bp->b_error = 0;
 1128 
 1129         splx(s);
 1130 
 1131         /* Perform transfer. */
 1132         dev_strategy(vp->v_rdev, bp);
 1133 
 1134         notify = 0;
 1135         s = splbio();
 1136 
 1137         /*
 1138          * If we had an error invoking the request, or an error in processing
 1139          * the request before we have returned, we process it as an error in
 1140          * transfer.  Note that such an I/O error is not indicated immediately,
 1141          * but is returned using the aio_error mechanism.  In this case,
 1142          * aio_suspend will return immediately.
 1143          */
 1144         if (bp->b_error || (bp->b_ioflags & BIO_ERROR)) {
 1145                 struct aiocb *job = aiocbe->uuaiocb;
 1146 
 1147                 aiocbe->uaiocb._aiocb_private.status = 0;
 1148                 suword(&job->_aiocb_private.status, 0);
 1149                 aiocbe->uaiocb._aiocb_private.error = bp->b_error;
 1150                 suword(&job->_aiocb_private.error, bp->b_error);
 1151 
 1152                 ki->kaio_buffer_finished_count++;
 1153 
 1154                 if (aiocbe->jobstate != JOBST_JOBBFINISHED) {
 1155                         aiocbe->jobstate = JOBST_JOBBFINISHED;
 1156                         aiocbe->jobflags |= AIOCBLIST_DONE;
 1157                         TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist);
 1158                         TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist);
 1159                         notify = 1;
 1160                 }
 1161         }
 1162         splx(s);
 1163         if (notify)
 1164                 KNOTE_UNLOCKED(&aiocbe->klist, 0);
 1165         return (0);
 1166 
 1167 doerror:
 1168         ki->kaio_buffer_count--;
 1169         if (lj)
 1170                 lj->lioj_buffer_count--;
 1171         aiocbe->bp = NULL;
 1172         relpbuf(bp, NULL);
 1173         return (error);
 1174 }
 1175 
 1176 /*
 1177  * This waits/tests physio completion.
 1178  */
 1179 static int
 1180 aio_fphysio(struct aiocblist *iocb)
 1181 {
 1182         int s;
 1183         struct buf *bp;
 1184         int error;
 1185 
 1186         bp = iocb->bp;
 1187 
 1188         s = splbio();
 1189         while ((bp->b_flags & B_DONE) == 0) {
 1190                 if (tsleep(bp, PRIBIO, "physstr", aiod_timeout)) {
 1191                         if ((bp->b_flags & B_DONE) == 0) {
 1192                                 splx(s);
 1193                                 return (EINPROGRESS);
 1194                         } else
 1195                                 break;
 1196                 }
 1197         }
 1198         splx(s);
 1199 
 1200         /* Release mapping into kernel space. */
 1201         vunmapbuf(bp);
 1202         iocb->bp = 0;
 1203 
 1204         error = 0;
 1205 
 1206         /* Check for an error. */
 1207         if (bp->b_ioflags & BIO_ERROR)
 1208                 error = bp->b_error;
 1209 
 1210         relpbuf(bp, NULL);
 1211         return (error);
 1212 }
 1213 
 1214 /*
 1215  * Wake up aio requests that may be serviceable now.
 1216  */
 1217 static void
 1218 aio_swake_cb(struct socket *so, struct sockbuf *sb)
 1219 {
 1220         struct aiocblist *cb,*cbn;
 1221         struct proc *p;
 1222         struct kaioinfo *ki = NULL;
 1223         int opcode, wakecount = 0;
 1224         struct aiothreadlist *aiop;
 1225 
 1226         if (sb == &so->so_snd) {
 1227                 opcode = LIO_WRITE;
 1228                 SOCKBUF_LOCK(&so->so_snd);
 1229                 so->so_snd.sb_flags &= ~SB_AIO;
 1230                 SOCKBUF_UNLOCK(&so->so_snd);
 1231         } else {
 1232                 opcode = LIO_READ;
 1233                 SOCKBUF_LOCK(&so->so_rcv);
 1234                 so->so_rcv.sb_flags &= ~SB_AIO;
 1235                 SOCKBUF_UNLOCK(&so->so_rcv);
 1236         }
 1237 
 1238         for (cb = TAILQ_FIRST(&so->so_aiojobq); cb; cb = cbn) {
 1239                 cbn = TAILQ_NEXT(cb, list);
 1240                 if (opcode == cb->uaiocb.aio_lio_opcode) {
 1241                         p = cb->userproc;
 1242                         ki = p->p_aioinfo;
 1243                         TAILQ_REMOVE(&so->so_aiojobq, cb, list);
 1244                         TAILQ_REMOVE(&ki->kaio_sockqueue, cb, plist);
 1245                         TAILQ_INSERT_TAIL(&aio_jobs, cb, list);
 1246                         TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, cb, plist);
 1247                         wakecount++;
 1248                         if (cb->jobstate != JOBST_JOBQGLOBAL)
 1249                                 panic("invalid queue value");
 1250                 }
 1251         }
 1252 
 1253         while (wakecount--) {
 1254                 mtx_lock(&aio_freeproc_mtx);
 1255                 if ((aiop = TAILQ_FIRST(&aio_freeproc)) != 0) {
 1256                         TAILQ_REMOVE(&aio_freeproc, aiop, list);
 1257                         aiop->aiothreadflags &= ~AIOP_FREE;
 1258                         wakeup(aiop->aiothread);
 1259                 }
 1260                 mtx_unlock(&aio_freeproc_mtx);
 1261         }
 1262 }
 1263 
 1264 /*
 1265  * Queue a new AIO request.  Choosing either the threaded or direct physio VCHR
 1266  * technique is done in this code.
 1267  */
 1268 static int
 1269 _aio_aqueue(struct thread *td, struct aiocb *job, struct aio_liojob *lj, int type)
 1270 {
 1271         struct proc *p = td->td_proc;
 1272         struct filedesc *fdp;
 1273         struct file *fp;
 1274         unsigned int fd;
 1275         struct socket *so;
 1276         int s;
 1277         int error;
 1278         int opcode;
 1279         struct aiocblist *aiocbe;
 1280         struct aiothreadlist *aiop;
 1281         struct kaioinfo *ki;
 1282         struct kevent kev;
 1283         struct kqueue *kq;
 1284         struct file *kq_fp;
 1285         struct sockbuf *sb;
 1286 
 1287         aiocbe = uma_zalloc(aiocb_zone, M_WAITOK);
 1288         aiocbe->inputcharge = 0;
 1289         aiocbe->outputcharge = 0;
 1290         /* XXX - need a lock */
 1291         knlist_init(&aiocbe->klist, NULL, NULL, NULL, NULL);
 1292 
 1293         suword(&job->_aiocb_private.status, -1);
 1294         suword(&job->_aiocb_private.error, 0);
 1295         suword(&job->_aiocb_private.kernelinfo, -1);
 1296 
 1297         error = copyin(job, &aiocbe->uaiocb, sizeof(aiocbe->uaiocb));
 1298         if (error) {
 1299                 suword(&job->_aiocb_private.error, error);
 1300                 uma_zfree(aiocb_zone, aiocbe);
 1301                 return (error);
 1302         }
 1303         if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL &&
 1304                 !_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) {
 1305                 uma_zfree(aiocb_zone, aiocbe);
 1306                 return (EINVAL);
 1307         }
 1308 
 1309         /* Save userspace address of the job info. */
 1310         aiocbe->uuaiocb = job;
 1311 
 1312         /* Get the opcode. */
 1313         if (type != LIO_NOP)
 1314                 aiocbe->uaiocb.aio_lio_opcode = type;
 1315         opcode = aiocbe->uaiocb.aio_lio_opcode;
 1316 
 1317         /* Get the fd info for process. */
 1318         fdp = p->p_fd;
 1319 
 1320         /*
 1321          * Range check file descriptor.
 1322          */
 1323         FILEDESC_LOCK(fdp);
 1324         fd = aiocbe->uaiocb.aio_fildes;
 1325         if (fd >= fdp->fd_nfiles) {
 1326                 FILEDESC_UNLOCK(fdp);
 1327                 uma_zfree(aiocb_zone, aiocbe);
 1328                 if (type == 0)
 1329                         suword(&job->_aiocb_private.error, EBADF);
 1330                 return (EBADF);
 1331         }
 1332 
 1333         fp = aiocbe->fd_file = fdp->fd_ofiles[fd];
 1334         if ((fp == NULL) ||
 1335             ((opcode == LIO_WRITE) && ((fp->f_flag & FWRITE) == 0)) ||
 1336             ((opcode == LIO_READ) && ((fp->f_flag & FREAD) == 0))) {
 1337                 FILEDESC_UNLOCK(fdp);
 1338                 uma_zfree(aiocb_zone, aiocbe);
 1339                 if (type == 0)
 1340                         suword(&job->_aiocb_private.error, EBADF);
 1341                 return (EBADF);
 1342         }
 1343         fhold(fp);
 1344         FILEDESC_UNLOCK(fdp);
 1345 
 1346         if (aiocbe->uaiocb.aio_offset == -1LL) {
 1347                 error = EINVAL;
 1348                 goto aqueue_fail;
 1349         }
 1350         error = suword(&job->_aiocb_private.kernelinfo, jobrefid);
 1351         if (error) {
 1352                 error = EINVAL;
 1353                 goto aqueue_fail;
 1354         }
 1355         aiocbe->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jobrefid;
 1356         if (jobrefid == LONG_MAX)
 1357                 jobrefid = 1;
 1358         else
 1359                 jobrefid++;
 1360 
 1361         if (opcode == LIO_NOP) {
 1362                 fdrop(fp, td);
 1363                 uma_zfree(aiocb_zone, aiocbe);
 1364                 if (type == 0) {
 1365                         suword(&job->_aiocb_private.error, 0);
 1366                         suword(&job->_aiocb_private.status, 0);
 1367                         suword(&job->_aiocb_private.kernelinfo, 0);
 1368                 }
 1369                 return (0);
 1370         }
 1371         if ((opcode != LIO_READ) && (opcode != LIO_WRITE)) {
 1372                 if (type == 0)
 1373                         suword(&job->_aiocb_private.status, 0);
 1374                 error = EINVAL;
 1375                 goto aqueue_fail;
 1376         }
 1377 
 1378         if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_KEVENT) {
 1379                 kev.ident = aiocbe->uaiocb.aio_sigevent.sigev_notify_kqueue;
 1380                 kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sigval_ptr;
 1381         } else
 1382                 goto no_kqueue;
 1383         if ((u_int)kev.ident >= fdp->fd_nfiles ||
 1384             (kq_fp = fdp->fd_ofiles[kev.ident]) == NULL ||
 1385             (kq_fp->f_type != DTYPE_KQUEUE)) {
 1386                 error = EBADF;
 1387                 goto aqueue_fail;
 1388         }
 1389         kq = kq_fp->f_data;
 1390         kev.ident = (uintptr_t)aiocbe->uuaiocb;
 1391         kev.filter = EVFILT_AIO;
 1392         kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
 1393         kev.data = (intptr_t)aiocbe;
 1394         error = kqueue_register(kq, &kev, td, 1);
 1395 aqueue_fail:
 1396         if (error) {
 1397                 fdrop(fp, td);
 1398                 uma_zfree(aiocb_zone, aiocbe);
 1399                 if (type == 0)
 1400                         suword(&job->_aiocb_private.error, error);
 1401                 goto done;
 1402         }
 1403 no_kqueue:
 1404 
 1405         suword(&job->_aiocb_private.error, EINPROGRESS);
 1406         aiocbe->uaiocb._aiocb_private.error = EINPROGRESS;
 1407         aiocbe->userproc = p;
 1408         aiocbe->cred = crhold(td->td_ucred);
 1409         aiocbe->jobflags = 0;
 1410         aiocbe->lio = lj;
 1411         ki = p->p_aioinfo;
 1412 
 1413         if (fp->f_type == DTYPE_SOCKET) {
 1414                 /*
 1415                  * Alternate queueing for socket ops: Reach down into the
 1416                  * descriptor to get the socket data.  Then check to see if the
 1417                  * socket is ready to be read or written (based on the requested
 1418                  * operation).
 1419                  *
 1420                  * If it is not ready for io, then queue the aiocbe on the
 1421                  * socket, and set the flags so we get a call when sbnotify()
 1422                  * happens.
 1423                  *
 1424                  * Note if opcode is neither LIO_WRITE nor LIO_READ we lock
 1425                  * and unlock the snd sockbuf for no reason.
 1426                  */
 1427                 so = fp->f_data;
 1428                 sb = (opcode == LIO_READ) ? &so->so_rcv : &so->so_snd;
 1429                 SOCKBUF_LOCK(sb);
 1430                 s = splnet();
 1431                 if (((opcode == LIO_READ) && (!soreadable(so))) || ((opcode ==
 1432                     LIO_WRITE) && (!sowriteable(so)))) {
 1433                         TAILQ_INSERT_TAIL(&so->so_aiojobq, aiocbe, list);
 1434                         TAILQ_INSERT_TAIL(&ki->kaio_sockqueue, aiocbe, plist);
 1435                         sb->sb_flags |= SB_AIO;
 1436                         aiocbe->jobstate = JOBST_JOBQGLOBAL; /* XXX */
 1437                         ki->kaio_queue_count++;
 1438                         num_queue_count++;
 1439                         SOCKBUF_UNLOCK(sb);
 1440                         splx(s);
 1441                         error = 0;
 1442                         goto done;
 1443                 }
 1444                 SOCKBUF_UNLOCK(sb);
 1445                 splx(s);
 1446         }
 1447 
 1448         if ((error = aio_qphysio(p, aiocbe)) == 0)
 1449                 goto done;
 1450         if (error > 0) {
 1451                 suword(&job->_aiocb_private.status, 0);
 1452                 aiocbe->uaiocb._aiocb_private.error = error;
 1453                 suword(&job->_aiocb_private.error, error);
 1454                 goto done;
 1455         }
 1456 
 1457         /* No buffer for daemon I/O. */
 1458         aiocbe->bp = NULL;
 1459 
 1460         ki->kaio_queue_count++;
 1461         if (lj)
 1462                 lj->lioj_queue_count++;
 1463         s = splnet();
 1464         TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist);
 1465         TAILQ_INSERT_TAIL(&aio_jobs, aiocbe, list);
 1466         splx(s);
 1467         aiocbe->jobstate = JOBST_JOBQGLOBAL;
 1468 
 1469         num_queue_count++;
 1470         error = 0;
 1471 
 1472         /*
 1473          * If we don't have a free AIO process, and we are below our quota, then
 1474          * start one.  Otherwise, depend on the subsequent I/O completions to
 1475          * pick-up this job.  If we don't sucessfully create the new process
 1476          * (thread) due to resource issues, we return an error for now (EAGAIN),
 1477          * which is likely not the correct thing to do.
 1478          */
 1479         mtx_lock(&aio_freeproc_mtx);
 1480 retryproc:
 1481         if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
 1482                 TAILQ_REMOVE(&aio_freeproc, aiop, list);
 1483                 aiop->aiothreadflags &= ~AIOP_FREE;
 1484                 wakeup(aiop->aiothread);
 1485         } else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) &&
 1486             ((ki->kaio_active_count + num_aio_resv_start) <
 1487             ki->kaio_maxactive_count)) {
 1488                 num_aio_resv_start++;
 1489                 mtx_unlock(&aio_freeproc_mtx);
 1490                 if ((error = aio_newproc()) == 0) {
 1491                         mtx_lock(&aio_freeproc_mtx);
 1492                         num_aio_resv_start--;
 1493                         goto retryproc;
 1494                 }
 1495                 mtx_lock(&aio_freeproc_mtx);
 1496                 num_aio_resv_start--;
 1497         }
 1498         mtx_unlock(&aio_freeproc_mtx);
 1499 done:
 1500         return (error);
 1501 }
 1502 
 1503 /*
 1504  * This routine queues an AIO request, checking for quotas.
 1505  */
 1506 static int
 1507 aio_aqueue(struct thread *td, struct aiocb *job, int type)
 1508 {
 1509         struct proc *p = td->td_proc;
 1510         struct kaioinfo *ki;
 1511 
 1512         if (p->p_aioinfo == NULL)
 1513                 aio_init_aioinfo(p);
 1514 
 1515         if (num_queue_count >= max_queue_count)
 1516                 return (EAGAIN);
 1517 
 1518         ki = p->p_aioinfo;
 1519         if (ki->kaio_queue_count >= ki->kaio_qallowed_count)
 1520                 return (EAGAIN);
 1521 
 1522         return _aio_aqueue(td, job, NULL, type);
 1523 }
 1524 
 1525 /*
 1526  * Support the aio_return system call, as a side-effect, kernel resources are
 1527  * released.
 1528  */
 1529 int
 1530 aio_return(struct thread *td, struct aio_return_args *uap)
 1531 {
 1532         struct proc *p = td->td_proc;
 1533         int s;
 1534         long jobref;
 1535         struct aiocblist *cb, *ncb;
 1536         struct aiocb *ujob;
 1537         struct kaioinfo *ki;
 1538 
 1539         ujob = uap->aiocbp;
 1540         jobref = fuword(&ujob->_aiocb_private.kernelinfo);
 1541         if (jobref == -1 || jobref == 0)
 1542                 return (EINVAL);
 1543 
 1544         ki = p->p_aioinfo;
 1545         if (ki == NULL)
 1546                 return (EINVAL);
 1547         PROC_LOCK(p);
 1548         TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
 1549                 if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo) ==
 1550                     jobref) {
 1551                         if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
 1552                                 p->p_stats->p_ru.ru_oublock +=
 1553                                     cb->outputcharge;
 1554                                 cb->outputcharge = 0;
 1555                         } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
 1556                                 p->p_stats->p_ru.ru_inblock += cb->inputcharge;
 1557                                 cb->inputcharge = 0;
 1558                         }
 1559                         goto done;
 1560                 }
 1561         }
 1562         s = splbio();
 1563         for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = ncb) {
 1564                 ncb = TAILQ_NEXT(cb, plist);
 1565                 if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo)
 1566                     == jobref) {
 1567                         break;
 1568                 }
 1569         }
 1570         splx(s);
 1571  done:
 1572         PROC_UNLOCK(p);
 1573         if (cb != NULL) {
 1574                 if (ujob == cb->uuaiocb) {
 1575                         td->td_retval[0] =
 1576                             cb->uaiocb._aiocb_private.status;
 1577                 } else
 1578                         td->td_retval[0] = EFAULT;
 1579                 aio_free_entry(cb);
 1580                 return (0);
 1581         }
 1582         return (EINVAL);
 1583 }
 1584 
 1585 /*
 1586  * Allow a process to wakeup when any of the I/O requests are completed.
 1587  */
 1588 int
 1589 aio_suspend(struct thread *td, struct aio_suspend_args *uap)
 1590 {
 1591         struct proc *p = td->td_proc;
 1592         struct timeval atv;
 1593         struct timespec ts;
 1594         struct aiocb *const *cbptr, *cbp;
 1595         struct kaioinfo *ki;
 1596         struct aiocblist *cb;
 1597         int i;
 1598         int njoblist;
 1599         int error, s, timo;
 1600         long *ijoblist;
 1601         struct aiocb **ujoblist;
 1602 
 1603         if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
 1604                 return (EINVAL);
 1605 
 1606         timo = 0;
 1607         if (uap->timeout) {
 1608                 /* Get timespec struct. */
 1609                 if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0)
 1610                         return (error);
 1611 
 1612                 if (ts.tv_nsec < 0 || ts.tv_nsec >= 1000000000)
 1613                         return (EINVAL);
 1614 
 1615                 TIMESPEC_TO_TIMEVAL(&atv, &ts);
 1616                 if (itimerfix(&atv))
 1617                         return (EINVAL);
 1618                 timo = tvtohz(&atv);
 1619         }
 1620 
 1621         ki = p->p_aioinfo;
 1622         if (ki == NULL)
 1623                 return (EAGAIN);
 1624 
 1625         njoblist = 0;
 1626         ijoblist = uma_zalloc(aiol_zone, M_WAITOK);
 1627         ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
 1628         cbptr = uap->aiocbp;
 1629 
 1630         for (i = 0; i < uap->nent; i++) {
 1631                 cbp = (struct aiocb *)(intptr_t)fuword(&cbptr[i]);
 1632                 if (cbp == 0)
 1633                         continue;
 1634                 ujoblist[njoblist] = cbp;
 1635                 ijoblist[njoblist] = fuword(&cbp->_aiocb_private.kernelinfo);
 1636                 njoblist++;
 1637         }
 1638 
 1639         if (njoblist == 0) {
 1640                 uma_zfree(aiol_zone, ijoblist);
 1641                 uma_zfree(aiol_zone, ujoblist);
 1642                 return (0);
 1643         }
 1644 
 1645         error = 0;
 1646         for (;;) {
 1647                 PROC_LOCK(p);
 1648                 TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
 1649                         for (i = 0; i < njoblist; i++) {
 1650                                 if (((intptr_t)
 1651                                     cb->uaiocb._aiocb_private.kernelinfo) ==
 1652                                     ijoblist[i]) {
 1653                                         PROC_UNLOCK(p);
 1654                                         if (ujoblist[i] != cb->uuaiocb)
 1655                                                 error = EINVAL;
 1656                                         uma_zfree(aiol_zone, ijoblist);
 1657                                         uma_zfree(aiol_zone, ujoblist);
 1658                                         return (error);
 1659                                 }
 1660                         }
 1661                 }
 1662 
 1663                 s = splbio();
 1664                 for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb =
 1665                     TAILQ_NEXT(cb, plist)) {
 1666                         for (i = 0; i < njoblist; i++) {
 1667                                 if (((intptr_t)
 1668                                     cb->uaiocb._aiocb_private.kernelinfo) ==
 1669                                     ijoblist[i]) {
 1670                                         PROC_UNLOCK(p);
 1671                                         splx(s);
 1672                                         if (ujoblist[i] != cb->uuaiocb)
 1673                                                 error = EINVAL;
 1674                                         uma_zfree(aiol_zone, ijoblist);
 1675                                         uma_zfree(aiol_zone, ujoblist);
 1676                                         return (error);
 1677                                 }
 1678                         }
 1679                 }
 1680 
 1681                 ki->kaio_flags |= KAIO_WAKEUP;
 1682                 error = msleep(p, &p->p_mtx, PDROP | PRIBIO | PCATCH, "aiospn",
 1683                     timo);
 1684                 splx(s);
 1685 
 1686                 if (error == ERESTART || error == EINTR) {
 1687                         uma_zfree(aiol_zone, ijoblist);
 1688                         uma_zfree(aiol_zone, ujoblist);
 1689                         return (EINTR);
 1690                 } else if (error == EWOULDBLOCK) {
 1691                         uma_zfree(aiol_zone, ijoblist);
 1692                         uma_zfree(aiol_zone, ujoblist);
 1693                         return (EAGAIN);
 1694                 }
 1695         }
 1696 
 1697 /* NOTREACHED */
 1698         return (EINVAL);
 1699 }
 1700 
 1701 /*
 1702  * aio_cancel cancels any non-physio aio operations not currently in
 1703  * progress.
 1704  */
 1705 int
 1706 aio_cancel(struct thread *td, struct aio_cancel_args *uap)
 1707 {
 1708         struct proc *p = td->td_proc;
 1709         struct kaioinfo *ki;
 1710         struct aiocblist *cbe, *cbn;
 1711         struct file *fp;
 1712         struct filedesc *fdp;
 1713         struct socket *so;
 1714         struct proc *po;
 1715         int s,error;
 1716         int cancelled=0;
 1717         int notcancelled=0;
 1718         struct vnode *vp;
 1719 
 1720         fdp = p->p_fd;
 1721         if ((u_int)uap->fd >= fdp->fd_nfiles ||
 1722             (fp = fdp->fd_ofiles[uap->fd]) == NULL)
 1723                 return (EBADF);
 1724 
 1725         if (fp->f_type == DTYPE_VNODE) {
 1726                 vp = fp->f_vnode;
 1727 
 1728                 if (vn_isdisk(vp,&error)) {
 1729                         td->td_retval[0] = AIO_NOTCANCELED;
 1730                         return (0);
 1731                 }
 1732         } else if (fp->f_type == DTYPE_SOCKET) {
 1733                 so = fp->f_data;
 1734 
 1735                 s = splnet();
 1736 
 1737                 for (cbe = TAILQ_FIRST(&so->so_aiojobq); cbe; cbe = cbn) {
 1738                         cbn = TAILQ_NEXT(cbe, list);
 1739                         if ((uap->aiocbp == NULL) ||
 1740                                 (uap->aiocbp == cbe->uuaiocb) ) {
 1741                                 po = cbe->userproc;
 1742                                 ki = po->p_aioinfo;
 1743                                 TAILQ_REMOVE(&so->so_aiojobq, cbe, list);
 1744                                 TAILQ_REMOVE(&ki->kaio_sockqueue, cbe, plist);
 1745                                 TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe, plist);
 1746                                 if (ki->kaio_flags & KAIO_WAKEUP) {
 1747                                         wakeup(po);
 1748                                 }
 1749                                 cbe->jobstate = JOBST_JOBFINISHED;
 1750                                 cbe->uaiocb._aiocb_private.status=-1;
 1751                                 cbe->uaiocb._aiocb_private.error=ECANCELED;
 1752                                 cancelled++;
 1753 /* XXX cancelled, knote? */
 1754                                 if (cbe->uaiocb.aio_sigevent.sigev_notify ==
 1755                                     SIGEV_SIGNAL) {
 1756                                         PROC_LOCK(cbe->userproc);
 1757                                         psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo);
 1758                                         PROC_UNLOCK(cbe->userproc);
 1759                                 }
 1760                                 if (uap->aiocbp)
 1761                                         break;
 1762                         }
 1763                 }
 1764                 splx(s);
 1765 
 1766                 if ((cancelled) && (uap->aiocbp)) {
 1767                         td->td_retval[0] = AIO_CANCELED;
 1768                         return (0);
 1769                 }
 1770         }
 1771         ki=p->p_aioinfo;
 1772         if (ki == NULL)
 1773                 goto done;
 1774         s = splnet();
 1775 
 1776         for (cbe = TAILQ_FIRST(&ki->kaio_jobqueue); cbe; cbe = cbn) {
 1777                 cbn = TAILQ_NEXT(cbe, plist);
 1778 
 1779                 if ((uap->fd == cbe->uaiocb.aio_fildes) &&
 1780                     ((uap->aiocbp == NULL ) ||
 1781                      (uap->aiocbp == cbe->uuaiocb))) {
 1782 
 1783                         if (cbe->jobstate == JOBST_JOBQGLOBAL) {
 1784                                 TAILQ_REMOVE(&aio_jobs, cbe, list);
 1785                                 TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist);
 1786                                 TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe,
 1787                                     plist);
 1788                                 cancelled++;
 1789                                 ki->kaio_queue_finished_count++;
 1790                                 cbe->jobstate = JOBST_JOBFINISHED;
 1791                                 cbe->uaiocb._aiocb_private.status = -1;
 1792                                 cbe->uaiocb._aiocb_private.error = ECANCELED;
 1793 /* XXX cancelled, knote? */
 1794                                 if (cbe->uaiocb.aio_sigevent.sigev_notify ==
 1795                                     SIGEV_SIGNAL) {
 1796                                         PROC_LOCK(cbe->userproc);
 1797                                         psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo);
 1798                                         PROC_UNLOCK(cbe->userproc);
 1799                                 }
 1800                         } else {
 1801                                 notcancelled++;
 1802                         }
 1803                 }
 1804         }
 1805         splx(s);
 1806 done:
 1807         if (notcancelled) {
 1808                 td->td_retval[0] = AIO_NOTCANCELED;
 1809                 return (0);
 1810         }
 1811         if (cancelled) {
 1812                 td->td_retval[0] = AIO_CANCELED;
 1813                 return (0);
 1814         }
 1815         td->td_retval[0] = AIO_ALLDONE;
 1816 
 1817         return (0);
 1818 }
 1819 
 1820 /*
 1821  * aio_error is implemented in the kernel level for compatibility purposes only.
 1822  * For a user mode async implementation, it would be best to do it in a userland
 1823  * subroutine.
 1824  */
 1825 int
 1826 aio_error(struct thread *td, struct aio_error_args *uap)
 1827 {
 1828         struct proc *p = td->td_proc;
 1829         int s;
 1830         struct aiocblist *cb;
 1831         struct kaioinfo *ki;
 1832         long jobref;
 1833 
 1834         ki = p->p_aioinfo;
 1835         if (ki == NULL)
 1836                 return (EINVAL);
 1837 
 1838         jobref = fuword(&uap->aiocbp->_aiocb_private.kernelinfo);
 1839         if ((jobref == -1) || (jobref == 0))
 1840                 return (EINVAL);
 1841 
 1842         PROC_LOCK(p);
 1843         TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
 1844                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1845                     jobref) {
 1846                         PROC_UNLOCK(p);
 1847                         td->td_retval[0] = cb->uaiocb._aiocb_private.error;
 1848                         return (0);
 1849                 }
 1850         }
 1851 
 1852         s = splnet();
 1853 
 1854         for (cb = TAILQ_FIRST(&ki->kaio_jobqueue); cb; cb = TAILQ_NEXT(cb,
 1855             plist)) {
 1856                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1857                     jobref) {
 1858                         PROC_UNLOCK(p);
 1859                         td->td_retval[0] = EINPROGRESS;
 1860                         splx(s);
 1861                         return (0);
 1862                 }
 1863         }
 1864 
 1865         for (cb = TAILQ_FIRST(&ki->kaio_sockqueue); cb; cb = TAILQ_NEXT(cb,
 1866             plist)) {
 1867                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1868                     jobref) {
 1869                         PROC_UNLOCK(p);
 1870                         td->td_retval[0] = EINPROGRESS;
 1871                         splx(s);
 1872                         return (0);
 1873                 }
 1874         }
 1875         splx(s);
 1876 
 1877         s = splbio();
 1878         for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = TAILQ_NEXT(cb,
 1879             plist)) {
 1880                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1881                     jobref) {
 1882                         PROC_UNLOCK(p);
 1883                         td->td_retval[0] = cb->uaiocb._aiocb_private.error;
 1884                         splx(s);
 1885                         return (0);
 1886                 }
 1887         }
 1888 
 1889         for (cb = TAILQ_FIRST(&ki->kaio_bufqueue); cb; cb = TAILQ_NEXT(cb,
 1890             plist)) {
 1891                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1892                     jobref) {
 1893                         PROC_UNLOCK(p);
 1894                         td->td_retval[0] = EINPROGRESS;
 1895                         splx(s);
 1896                         return (0);
 1897                 }
 1898         }
 1899         splx(s);
 1900         PROC_UNLOCK(p);
 1901 
 1902 #if (0)
 1903         /*
 1904          * Hack for lio.
 1905          */
 1906         status = fuword(&uap->aiocbp->_aiocb_private.status);
 1907         if (status == -1)
 1908                 return fuword(&uap->aiocbp->_aiocb_private.error);
 1909 #endif
 1910         return (EINVAL);
 1911 }
 1912 
 1913 /* syscall - asynchronous read from a file (REALTIME) */
 1914 int
 1915 aio_read(struct thread *td, struct aio_read_args *uap)
 1916 {
 1917 
 1918         return aio_aqueue(td, uap->aiocbp, LIO_READ);
 1919 }
 1920 
 1921 /* syscall - asynchronous write to a file (REALTIME) */
 1922 int
 1923 aio_write(struct thread *td, struct aio_write_args *uap)
 1924 {
 1925 
 1926         return aio_aqueue(td, uap->aiocbp, LIO_WRITE);
 1927 }
 1928 
 1929 /* syscall - list directed I/O (REALTIME) */
 1930 int
 1931 lio_listio(struct thread *td, struct lio_listio_args *uap)
 1932 {
 1933         struct proc *p = td->td_proc;
 1934         int nent, nentqueued;
 1935         struct aiocb *iocb, * const *cbptr;
 1936         struct aiocblist *cb;
 1937         struct kaioinfo *ki;
 1938         struct aio_liojob *lj;
 1939         int error, runningcode;
 1940         int nerror;
 1941         int i;
 1942         int s;
 1943 
 1944         if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
 1945                 return (EINVAL);
 1946 
 1947         nent = uap->nent;
 1948         if (nent < 0 || nent > AIO_LISTIO_MAX)
 1949                 return (EINVAL);
 1950 
 1951         if (p->p_aioinfo == NULL)
 1952                 aio_init_aioinfo(p);
 1953 
 1954         if ((nent + num_queue_count) > max_queue_count)
 1955                 return (EAGAIN);
 1956 
 1957         ki = p->p_aioinfo;
 1958         if ((nent + ki->kaio_queue_count) > ki->kaio_qallowed_count)
 1959                 return (EAGAIN);
 1960 
 1961         lj = uma_zalloc(aiolio_zone, M_WAITOK);
 1962         if (!lj)
 1963                 return (EAGAIN);
 1964 
 1965         lj->lioj_flags = 0;
 1966         lj->lioj_buffer_count = 0;
 1967         lj->lioj_buffer_finished_count = 0;
 1968         lj->lioj_queue_count = 0;
 1969         lj->lioj_queue_finished_count = 0;
 1970 
 1971         /*
 1972          * Setup signal.
 1973          */
 1974         if (uap->sig && (uap->mode == LIO_NOWAIT)) {
 1975                 error = copyin(uap->sig, &lj->lioj_signal,
 1976                     sizeof(lj->lioj_signal));
 1977                 if (error) {
 1978                         uma_zfree(aiolio_zone, lj);
 1979                         return (error);
 1980                 }
 1981                 if (!_SIG_VALID(lj->lioj_signal.sigev_signo)) {
 1982                         uma_zfree(aiolio_zone, lj);
 1983                         return (EINVAL);
 1984                 }
 1985                 lj->lioj_flags |= LIOJ_SIGNAL;
 1986         }
 1987         TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list);
 1988         /*
 1989          * Get pointers to the list of I/O requests.
 1990          */
 1991         nerror = 0;
 1992         nentqueued = 0;
 1993         cbptr = uap->acb_list;
 1994         for (i = 0; i < uap->nent; i++) {
 1995                 iocb = (struct aiocb *)(intptr_t)fuword(&cbptr[i]);
 1996                 if (((intptr_t)iocb != -1) && ((intptr_t)iocb != 0)) {
 1997                         error = _aio_aqueue(td, iocb, lj, 0);
 1998                         if (error == 0)
 1999                                 nentqueued++;
 2000                         else
 2001                                 nerror++;
 2002                 }
 2003         }
 2004 
 2005         /*
 2006          * If we haven't queued any, then just return error.
 2007          */
 2008         if (nentqueued == 0)
 2009                 return (0);
 2010 
 2011         /*
 2012          * Calculate the appropriate error return.
 2013          */
 2014         runningcode = 0;
 2015         if (nerror)
 2016                 runningcode = EIO;
 2017 
 2018         if (uap->mode == LIO_WAIT) {
 2019                 int command, found;
 2020                 long jobref;
 2021 
 2022                 for (;;) {
 2023                         found = 0;
 2024                         for (i = 0; i < uap->nent; i++) {
 2025                                 /*
 2026                                  * Fetch address of the control buf pointer in
 2027                                  * user space.
 2028                                  */
 2029                                 iocb = (struct aiocb *)
 2030                                     (intptr_t)fuword(&cbptr[i]);
 2031                                 if (((intptr_t)iocb == -1) || ((intptr_t)iocb
 2032                                     == 0))
 2033                                         continue;
 2034 
 2035                                 /*
 2036                                  * Fetch the associated command from user space.
 2037                                  */
 2038                                 command = fuword(&iocb->aio_lio_opcode);
 2039                                 if (command == LIO_NOP) {
 2040                                         found++;
 2041                                         continue;
 2042                                 }
 2043 
 2044                                 jobref =
 2045                                     fuword(&iocb->_aiocb_private.kernelinfo);
 2046 
 2047                                 TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
 2048                                         if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo)
 2049                                             == jobref) {
 2050                                                 if (cb->uaiocb.aio_lio_opcode
 2051                                                     == LIO_WRITE) {
 2052                                                         p->p_stats->p_ru.ru_oublock
 2053                                                             +=
 2054                                                             cb->outputcharge;
 2055                                                         cb->outputcharge = 0;
 2056                                                 } else if (cb->uaiocb.aio_lio_opcode
 2057                                                     == LIO_READ) {
 2058                                                         p->p_stats->p_ru.ru_inblock
 2059                                                             += cb->inputcharge;
 2060                                                         cb->inputcharge = 0;
 2061                                                 }
 2062                                                 found++;
 2063                                                 break;
 2064                                         }
 2065                                 }
 2066 
 2067                                 s = splbio();
 2068                                 TAILQ_FOREACH(cb, &ki->kaio_bufdone, plist) {
 2069                                         if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo)
 2070                                             == jobref) {
 2071                                                 found++;
 2072                                                 break;
 2073                                         }
 2074                                 }
 2075                                 splx(s);
 2076                         }
 2077 
 2078                         /*
 2079                          * If all I/Os have been disposed of, then we can
 2080                          * return.
 2081                          */
 2082                         if (found == nentqueued)
 2083                                 return (runningcode);
 2084 
 2085                         ki->kaio_flags |= KAIO_WAKEUP;
 2086                         error = tsleep(p, PRIBIO | PCATCH, "aiospn", 0);
 2087 
 2088                         if (error == EINTR)
 2089                                 return (EINTR);
 2090                         else if (error == EWOULDBLOCK)
 2091                                 return (EAGAIN);
 2092                 }
 2093         }
 2094 
 2095         return (runningcode);
 2096 }
 2097 
 2098 /*
 2099  * Interrupt handler for physio, performs the necessary process wakeups, and
 2100  * signals.
 2101  */
 2102 static void
 2103 aio_physwakeup(struct buf *bp)
 2104 {
 2105         struct aiocblist *aiocbe;
 2106         struct proc *p;
 2107         struct kaioinfo *ki;
 2108         struct aio_liojob *lj;
 2109 
 2110         mtx_lock(&Giant);
 2111         bp->b_flags |= B_DONE;
 2112         wakeup(bp);
 2113 
 2114         aiocbe = (struct aiocblist *)bp->b_caller1;
 2115         if (aiocbe) {
 2116                 p = aiocbe->userproc;
 2117 
 2118                 aiocbe->jobstate = JOBST_JOBBFINISHED;
 2119                 aiocbe->uaiocb._aiocb_private.status -= bp->b_resid;
 2120                 aiocbe->uaiocb._aiocb_private.error = 0;
 2121                 aiocbe->jobflags |= AIOCBLIST_DONE;
 2122 
 2123                 if (bp->b_ioflags & BIO_ERROR)
 2124                         aiocbe->uaiocb._aiocb_private.error = bp->b_error;
 2125 
 2126                 lj = aiocbe->lio;
 2127                 if (lj) {
 2128                         lj->lioj_buffer_finished_count++;
 2129 
 2130                         /*
 2131                          * wakeup/signal if all of the interrupt jobs are done.
 2132                          */
 2133                         if (lj->lioj_buffer_finished_count ==
 2134                             lj->lioj_buffer_count &&
 2135                             lj->lioj_queue_finished_count ==
 2136                             lj->lioj_queue_count) {
 2137                                 /*
 2138                                  * Post a signal if it is called for.
 2139                                  */
 2140                                 if ((lj->lioj_flags &
 2141                                     (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) ==
 2142                                     LIOJ_SIGNAL) {
 2143                                         PROC_LOCK(p);
 2144                                         psignal(p, lj->lioj_signal.sigev_signo);
 2145                                         PROC_UNLOCK(p);
 2146                                         lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
 2147                                 }
 2148                         }
 2149                 }
 2150 
 2151                 ki = p->p_aioinfo;
 2152                 if (ki) {
 2153                         ki->kaio_buffer_finished_count++;
 2154                         TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist);
 2155                         TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist);
 2156 
 2157                         KNOTE_UNLOCKED(&aiocbe->klist, 0);
 2158                         /* Do the wakeup. */
 2159                         if (ki->kaio_flags & (KAIO_RUNDOWN|KAIO_WAKEUP)) {
 2160                                 ki->kaio_flags &= ~KAIO_WAKEUP;
 2161                                 wakeup(p);
 2162                         }
 2163                 }
 2164 
 2165                 if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL) {
 2166                         PROC_LOCK(p);
 2167                         psignal(p, aiocbe->uaiocb.aio_sigevent.sigev_signo);
 2168                         PROC_UNLOCK(p);
 2169                 }
 2170         }
 2171         mtx_unlock(&Giant);
 2172 }
 2173 
 2174 /* syscall - wait for the next completion of an aio request */
 2175 int
 2176 aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
 2177 {
 2178         struct proc *p = td->td_proc;
 2179         struct timeval atv;
 2180         struct timespec ts;
 2181         struct kaioinfo *ki;
 2182         struct aiocblist *cb = NULL;
 2183         int error, s, timo;
 2184 
 2185         suword(uap->aiocbp, (int)NULL);
 2186 
 2187         timo = 0;
 2188         if (uap->timeout) {
 2189                 /* Get timespec struct. */
 2190                 error = copyin(uap->timeout, &ts, sizeof(ts));
 2191                 if (error)
 2192                         return (error);
 2193 
 2194                 if ((ts.tv_nsec < 0) || (ts.tv_nsec >= 1000000000))
 2195                         return (EINVAL);
 2196 
 2197                 TIMESPEC_TO_TIMEVAL(&atv, &ts);
 2198                 if (itimerfix(&atv))
 2199                         return (EINVAL);
 2200                 timo = tvtohz(&atv);
 2201         }
 2202 
 2203         ki = p->p_aioinfo;
 2204         if (ki == NULL)
 2205                 return (EAGAIN);
 2206 
 2207         for (;;) {
 2208                 PROC_LOCK(p);
 2209                 if ((cb = TAILQ_FIRST(&ki->kaio_jobdone)) != 0) {
 2210                         PROC_UNLOCK(p);
 2211                         suword(uap->aiocbp, (uintptr_t)cb->uuaiocb);
 2212                         td->td_retval[0] = cb->uaiocb._aiocb_private.status;
 2213                         if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
 2214                                 p->p_stats->p_ru.ru_oublock +=
 2215                                     cb->outputcharge;
 2216                                 cb->outputcharge = 0;
 2217                         } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
 2218                                 p->p_stats->p_ru.ru_inblock += cb->inputcharge;
 2219                                 cb->inputcharge = 0;
 2220                         }
 2221                         error = cb->uaiocb._aiocb_private.error;
 2222                         aio_free_entry(cb);
 2223                         return (error);
 2224                 }
 2225 
 2226                 s = splbio();
 2227                 if ((cb = TAILQ_FIRST(&ki->kaio_bufdone)) != 0 ) {
 2228                         PROC_UNLOCK(p);
 2229                         splx(s);
 2230                         suword(uap->aiocbp, (uintptr_t)cb->uuaiocb);
 2231                         error = cb->uaiocb._aiocb_private.error;
 2232                         td->td_retval[0] = cb->uaiocb._aiocb_private.status;
 2233                         aio_free_entry(cb);
 2234                         return (error);
 2235                 }
 2236 
 2237                 ki->kaio_flags |= KAIO_WAKEUP;
 2238                 error = msleep(p, &p->p_mtx, PDROP | PRIBIO | PCATCH, "aiowc",
 2239                     timo);
 2240                 splx(s);
 2241 
 2242                 if (error == ERESTART)
 2243                         return (EINTR);
 2244                 else if (error < 0)
 2245                         return (error);
 2246                 else if (error == EINTR)
 2247                         return (EINTR);
 2248                 else if (error == EWOULDBLOCK)
 2249                         return (EAGAIN);
 2250         }
 2251 }
 2252 
 2253 /* kqueue attach function */
 2254 static int
 2255 filt_aioattach(struct knote *kn)
 2256 {
 2257         struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
 2258 
 2259         /*
 2260          * The aiocbe pointer must be validated before using it, so
 2261          * registration is restricted to the kernel; the user cannot
 2262          * set EV_FLAG1.
 2263          */
 2264         if ((kn->kn_flags & EV_FLAG1) == 0)
 2265                 return (EPERM);
 2266         kn->kn_flags &= ~EV_FLAG1;
 2267 
 2268         knlist_add(&aiocbe->klist, kn, 0);
 2269 
 2270         return (0);
 2271 }
 2272 
 2273 /* kqueue detach function */
 2274 static void
 2275 filt_aiodetach(struct knote *kn)
 2276 {
 2277         struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
 2278 
 2279         knlist_remove(&aiocbe->klist, kn, 0);
 2280 }
 2281 
 2282 /* kqueue filter function */
 2283 /*ARGSUSED*/
 2284 static int
 2285 filt_aio(struct knote *kn, long hint)
 2286 {
 2287         struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
 2288 
 2289         kn->kn_data = aiocbe->uaiocb._aiocb_private.error;
 2290         if (aiocbe->jobstate != JOBST_JOBFINISHED &&
 2291             aiocbe->jobstate != JOBST_JOBBFINISHED)
 2292                 return (0);
 2293         kn->kn_flags |= EV_EOF;
 2294         return (1);
 2295 }

Cache object: 333288f2ad8caaf9fb6f90f44daeb957


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.