The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/vfs_aio.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1997 John S. Dyson.  All rights reserved.
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. John S. Dyson's name may not be used to endorse or promote products
   10  *    derived from this software without specific prior written permission.
   11  *
   12  * DISCLAIMER:  This code isn't warranted to do anything useful.  Anything
   13  * bad that happens because of using this software isn't the responsibility
   14  * of the author.  This software is distributed AS-IS.
   15  */
   16 
   17 /*
   18  * This file contains support for the POSIX 1003.1B AIO/LIO facility.
   19  */
   20 
   21 #include <sys/cdefs.h>
   22 __FBSDID("$FreeBSD: releng/6.3/sys/kern/vfs_aio.c 173886 2007-11-24 19:45:58Z cvs2svn $");
   23 
   24 #include <sys/param.h>
   25 #include <sys/systm.h>
   26 #include <sys/malloc.h>
   27 #include <sys/bio.h>
   28 #include <sys/buf.h>
   29 #include <sys/eventhandler.h>
   30 #include <sys/sysproto.h>
   31 #include <sys/filedesc.h>
   32 #include <sys/kernel.h>
   33 #include <sys/module.h>
   34 #include <sys/kthread.h>
   35 #include <sys/fcntl.h>
   36 #include <sys/file.h>
   37 #include <sys/limits.h>
   38 #include <sys/lock.h>
   39 #include <sys/mutex.h>
   40 #include <sys/unistd.h>
   41 #include <sys/proc.h>
   42 #include <sys/resourcevar.h>
   43 #include <sys/signalvar.h>
   44 #include <sys/protosw.h>
   45 #include <sys/socketvar.h>
   46 #include <sys/syscall.h>
   47 #include <sys/sysent.h>
   48 #include <sys/sysctl.h>
   49 #include <sys/sx.h>
   50 #include <sys/vnode.h>
   51 #include <sys/conf.h>
   52 #include <sys/event.h>
   53 
   54 #include <posix4/posix4.h>
   55 #include <vm/vm.h>
   56 #include <vm/vm_extern.h>
   57 #include <vm/pmap.h>
   58 #include <vm/vm_map.h>
   59 #include <vm/uma.h>
   60 #include <sys/aio.h>
   61 
   62 #include "opt_vfs_aio.h"
   63 
   64 NET_NEEDS_GIANT("aio");
   65 
   66 /*
   67  * Counter for allocating reference ids to new jobs.  Wrapped to 1 on
   68  * overflow.
   69  */
   70 static  long jobrefid;
   71 
   72 #define JOBST_NULL              0x0
   73 #define JOBST_JOBQGLOBAL        0x2
   74 #define JOBST_JOBRUNNING        0x3
   75 #define JOBST_JOBFINISHED       0x4
   76 #define JOBST_JOBQBUF           0x5
   77 #define JOBST_JOBBFINISHED      0x6
   78 
   79 #ifndef MAX_AIO_PER_PROC
   80 #define MAX_AIO_PER_PROC        32
   81 #endif
   82 
   83 #ifndef MAX_AIO_QUEUE_PER_PROC
   84 #define MAX_AIO_QUEUE_PER_PROC  256 /* Bigger than AIO_LISTIO_MAX */
   85 #endif
   86 
   87 #ifndef MAX_AIO_PROCS
   88 #define MAX_AIO_PROCS           32
   89 #endif
   90 
   91 #ifndef MAX_AIO_QUEUE
   92 #define MAX_AIO_QUEUE           1024 /* Bigger than AIO_LISTIO_MAX */
   93 #endif
   94 
   95 #ifndef TARGET_AIO_PROCS
   96 #define TARGET_AIO_PROCS        4
   97 #endif
   98 
   99 #ifndef MAX_BUF_AIO
  100 #define MAX_BUF_AIO             16
  101 #endif
  102 
  103 #ifndef AIOD_TIMEOUT_DEFAULT
  104 #define AIOD_TIMEOUT_DEFAULT    (10 * hz)
  105 #endif
  106 
  107 #ifndef AIOD_LIFETIME_DEFAULT
  108 #define AIOD_LIFETIME_DEFAULT   (30 * hz)
  109 #endif
  110 
  111 static SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, "Async IO management");
  112 
  113 static int max_aio_procs = MAX_AIO_PROCS;
  114 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_procs,
  115         CTLFLAG_RW, &max_aio_procs, 0,
  116         "Maximum number of kernel threads to use for handling async IO ");
  117 
  118 static int num_aio_procs = 0;
  119 SYSCTL_INT(_vfs_aio, OID_AUTO, num_aio_procs,
  120         CTLFLAG_RD, &num_aio_procs, 0,
  121         "Number of presently active kernel threads for async IO");
  122 
  123 /*
  124  * The code will adjust the actual number of AIO processes towards this
  125  * number when it gets a chance.
  126  */
  127 static int target_aio_procs = TARGET_AIO_PROCS;
  128 SYSCTL_INT(_vfs_aio, OID_AUTO, target_aio_procs, CTLFLAG_RW, &target_aio_procs,
  129         0, "Preferred number of ready kernel threads for async IO");
  130 
  131 static int max_queue_count = MAX_AIO_QUEUE;
  132 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue, CTLFLAG_RW, &max_queue_count, 0,
  133     "Maximum number of aio requests to queue, globally");
  134 
  135 static int num_queue_count = 0;
  136 SYSCTL_INT(_vfs_aio, OID_AUTO, num_queue_count, CTLFLAG_RD, &num_queue_count, 0,
  137     "Number of queued aio requests");
  138 
  139 static int num_buf_aio = 0;
  140 SYSCTL_INT(_vfs_aio, OID_AUTO, num_buf_aio, CTLFLAG_RD, &num_buf_aio, 0,
  141     "Number of aio requests presently handled by the buf subsystem");
  142 
  143 /* Number of async I/O thread in the process of being started */
  144 /* XXX This should be local to _aio_aqueue() */
  145 static int num_aio_resv_start = 0;
  146 
  147 static int aiod_timeout;
  148 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_timeout, CTLFLAG_RW, &aiod_timeout, 0,
  149     "Timeout value for synchronous aio operations");
  150 
  151 static int aiod_lifetime;
  152 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime, CTLFLAG_RW, &aiod_lifetime, 0,
  153     "Maximum lifetime for idle aiod");
  154 
  155 static int unloadable = 0;
  156 SYSCTL_INT(_vfs_aio, OID_AUTO, unloadable, CTLFLAG_RW, &unloadable, 0,
  157     "Allow unload of aio (not recommended)");
  158 
  159 
  160 static int max_aio_per_proc = MAX_AIO_PER_PROC;
  161 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_per_proc, CTLFLAG_RW, &max_aio_per_proc,
  162     0, "Maximum active aio requests per process (stored in the process)");
  163 
  164 static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC;
  165 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW,
  166     &max_aio_queue_per_proc, 0,
  167     "Maximum queued aio requests per process (stored in the process)");
  168 
  169 static int max_buf_aio = MAX_BUF_AIO;
  170 SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW, &max_buf_aio, 0,
  171     "Maximum buf aio requests per process (stored in the process)");
  172 
  173 struct aiocblist {
  174         TAILQ_ENTRY(aiocblist) list;    /* List of jobs */
  175         TAILQ_ENTRY(aiocblist) plist;   /* List of jobs for proc */
  176         int     jobflags;
  177         int     jobstate;
  178         int     inputcharge;
  179         int     outputcharge;
  180         struct  buf *bp;                /* Buffer pointer */
  181         struct  proc *userproc;         /* User process */ /* Not td! */
  182         struct  ucred *cred;            /* Active credential when created */
  183         struct  file *fd_file;          /* Pointer to file structure */
  184         struct  aio_liojob *lio;        /* Optional lio job */
  185         struct  aiocb *uuaiocb;         /* Pointer in userspace of aiocb */
  186         struct  knlist klist;           /* list of knotes */
  187         struct  aiocb uaiocb;           /* Kernel I/O control block */
  188 };
  189 
  190 /* jobflags */
  191 #define AIOCBLIST_RUNDOWN       0x4
  192 #define AIOCBLIST_DONE          0x10
  193 
  194 /*
  195  * AIO process info
  196  */
  197 #define AIOP_FREE       0x1                     /* proc on free queue */
  198 
  199 struct aiothreadlist {
  200         int aiothreadflags;                     /* AIO proc flags */
  201         TAILQ_ENTRY(aiothreadlist) list;        /* List of processes */
  202         struct thread *aiothread;               /* The AIO thread */
  203 };
  204 
  205 /*
  206  * data-structure for lio signal management
  207  */
  208 struct aio_liojob {
  209         int     lioj_flags;
  210         int     lioj_buffer_count;
  211         int     lioj_buffer_finished_count;
  212         int     lioj_queue_count;
  213         int     lioj_queue_finished_count;
  214         struct  sigevent lioj_signal;   /* signal on all I/O done */
  215         TAILQ_ENTRY(aio_liojob) lioj_list;
  216 };
  217 #define LIOJ_SIGNAL             0x1     /* signal on all done (lio) */
  218 #define LIOJ_SIGNAL_POSTED      0x2     /* signal has been posted */
  219 
  220 /*
  221  * per process aio data structure
  222  */
  223 struct kaioinfo {
  224         int     kaio_flags;             /* per process kaio flags */
  225         int     kaio_maxactive_count;   /* maximum number of AIOs */
  226         int     kaio_active_count;      /* number of currently used AIOs */
  227         int     kaio_qallowed_count;    /* maxiumu size of AIO queue */
  228         int     kaio_queue_count;       /* size of AIO queue */
  229         int     kaio_ballowed_count;    /* maximum number of buffers */
  230         int     kaio_queue_finished_count; /* number of daemon jobs finished */
  231         int     kaio_buffer_count;      /* number of physio buffers */
  232         int     kaio_buffer_finished_count; /* count of I/O done */
  233         TAILQ_HEAD(,aio_liojob) kaio_liojoblist; /* list of lio jobs */
  234         TAILQ_HEAD(,aiocblist) kaio_jobqueue;   /* job queue for process */
  235         TAILQ_HEAD(,aiocblist) kaio_jobdone;    /* done queue for process */
  236         TAILQ_HEAD(,aiocblist) kaio_bufqueue;   /* buffer job queue for process */
  237         TAILQ_HEAD(,aiocblist) kaio_bufdone;    /* buffer done queue for process */
  238         TAILQ_HEAD(,aiocblist) kaio_sockqueue;  /* queue for aios waiting on sockets */
  239 };
  240 
  241 #define KAIO_RUNDOWN    0x1     /* process is being run down */
  242 #define KAIO_WAKEUP     0x2     /* wakeup process when there is a significant event */
  243 
  244 static TAILQ_HEAD(,aiothreadlist) aio_freeproc;         /* Idle daemons */
  245 static struct mtx aio_freeproc_mtx;
  246 
  247 static TAILQ_HEAD(,aiocblist) aio_jobs;                 /* Async job list */
  248 
  249 static void     aio_init_aioinfo(struct proc *p);
  250 static void     aio_onceonly(void);
  251 static int      aio_free_entry(struct aiocblist *aiocbe);
  252 static void     aio_process(struct aiocblist *aiocbe);
  253 static int      aio_newproc(void);
  254 static int      aio_aqueue(struct thread *td, struct aiocb *job, int type);
  255 static void     aio_physwakeup(struct buf *bp);
  256 static void     aio_proc_rundown(void *arg, struct proc *p);
  257 static int      aio_fphysio(struct aiocblist *aiocbe);
  258 static int      aio_qphysio(struct proc *p, struct aiocblist *iocb);
  259 static void     aio_daemon(void *uproc);
  260 static void     aio_swake_cb(struct socket *, struct sockbuf *);
  261 static int      aio_unload(void);
  262 static int      filt_aioattach(struct knote *kn);
  263 static void     filt_aiodetach(struct knote *kn);
  264 static int      filt_aio(struct knote *kn, long hint);
  265 
  266 /*
  267  * Zones for:
  268  *      kaio    Per process async io info
  269  *      aiop    async io thread data
  270  *      aiocb   async io jobs
  271  *      aiol    list io job pointer - internal to aio_suspend XXX
  272  *      aiolio  list io jobs
  273  */
  274 static uma_zone_t kaio_zone, aiop_zone, aiocb_zone, aiol_zone, aiolio_zone;
  275 
  276 /* kqueue filters for aio */
  277 static struct filterops aio_filtops =
  278         { 0, filt_aioattach, filt_aiodetach, filt_aio };
  279 
  280 static eventhandler_tag exit_tag, exec_tag;
  281 
  282 /*
  283  * Main operations function for use as a kernel module.
  284  */
  285 static int
  286 aio_modload(struct module *module, int cmd, void *arg)
  287 {
  288         int error = 0;
  289 
  290         switch (cmd) {
  291         case MOD_LOAD:
  292                 aio_onceonly();
  293                 break;
  294         case MOD_UNLOAD:
  295                 error = aio_unload();
  296                 break;
  297         case MOD_SHUTDOWN:
  298                 break;
  299         default:
  300                 error = EINVAL;
  301                 break;
  302         }
  303         return (error);
  304 }
  305 
  306 static moduledata_t aio_mod = {
  307         "aio",
  308         &aio_modload,
  309         NULL
  310 };
  311 
  312 SYSCALL_MODULE_HELPER(aio_return);
  313 SYSCALL_MODULE_HELPER(aio_suspend);
  314 SYSCALL_MODULE_HELPER(aio_cancel);
  315 SYSCALL_MODULE_HELPER(aio_error);
  316 SYSCALL_MODULE_HELPER(aio_read);
  317 SYSCALL_MODULE_HELPER(aio_write);
  318 SYSCALL_MODULE_HELPER(aio_waitcomplete);
  319 SYSCALL_MODULE_HELPER(lio_listio);
  320 
  321 DECLARE_MODULE(aio, aio_mod,
  322         SI_SUB_VFS, SI_ORDER_ANY);
  323 MODULE_VERSION(aio, 1);
  324 
  325 /*
  326  * Startup initialization
  327  */
  328 static void
  329 aio_onceonly(void)
  330 {
  331 
  332         /* XXX: should probably just use so->callback */
  333         aio_swake = &aio_swake_cb;
  334         exit_tag = EVENTHANDLER_REGISTER(process_exit, aio_proc_rundown, NULL,
  335             EVENTHANDLER_PRI_ANY);
  336         exec_tag = EVENTHANDLER_REGISTER(process_exec, aio_proc_rundown, NULL,
  337             EVENTHANDLER_PRI_ANY);
  338         kqueue_add_filteropts(EVFILT_AIO, &aio_filtops);
  339         TAILQ_INIT(&aio_freeproc);
  340         mtx_init(&aio_freeproc_mtx, "aio_freeproc", NULL, MTX_DEF);
  341         TAILQ_INIT(&aio_jobs);
  342         kaio_zone = uma_zcreate("AIO", sizeof(struct kaioinfo), NULL, NULL,
  343             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  344         aiop_zone = uma_zcreate("AIOP", sizeof(struct aiothreadlist), NULL,
  345             NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  346         aiocb_zone = uma_zcreate("AIOCB", sizeof(struct aiocblist), NULL, NULL,
  347             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  348         aiol_zone = uma_zcreate("AIOL", AIO_LISTIO_MAX*sizeof(intptr_t) , NULL,
  349             NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  350         aiolio_zone = uma_zcreate("AIOLIO", sizeof(struct aio_liojob), NULL,
  351             NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  352         aiod_timeout = AIOD_TIMEOUT_DEFAULT;
  353         aiod_lifetime = AIOD_LIFETIME_DEFAULT;
  354         jobrefid = 1;
  355         async_io_version = _POSIX_VERSION;
  356         p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, AIO_LISTIO_MAX);
  357         p31b_setcfg(CTL_P1003_1B_AIO_MAX, MAX_AIO_QUEUE);
  358         p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, 0);
  359 }
  360 
  361 /*
  362  * Callback for unload of AIO when used as a module.
  363  */
  364 static int
  365 aio_unload(void)
  366 {
  367         int error;
  368 
  369         /*
  370          * XXX: no unloads by default, it's too dangerous.
  371          * perhaps we could do it if locked out callers and then
  372          * did an aio_proc_rundown() on each process.
  373          */
  374         if (!unloadable)
  375                 return (EOPNOTSUPP);
  376 
  377         error = kqueue_del_filteropts(EVFILT_AIO);
  378         if (error)
  379                 return error;
  380 
  381         async_io_version = 0;
  382         aio_swake = NULL;
  383         EVENTHANDLER_DEREGISTER(process_exit, exit_tag);
  384         EVENTHANDLER_DEREGISTER(process_exec, exec_tag);
  385         p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, -1);
  386         p31b_setcfg(CTL_P1003_1B_AIO_MAX, -1);
  387         p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, -1);
  388         return (0);
  389 }
  390 
  391 /*
  392  * Init the per-process aioinfo structure.  The aioinfo limits are set
  393  * per-process for user limit (resource) management.
  394  */
  395 static void
  396 aio_init_aioinfo(struct proc *p)
  397 {
  398         struct kaioinfo *ki;
  399 
  400         ki = uma_zalloc(kaio_zone, M_WAITOK);
  401         ki->kaio_flags = 0;
  402         ki->kaio_maxactive_count = max_aio_per_proc;
  403         ki->kaio_active_count = 0;
  404         ki->kaio_qallowed_count = max_aio_queue_per_proc;
  405         ki->kaio_queue_count = 0;
  406         ki->kaio_ballowed_count = max_buf_aio;
  407         ki->kaio_buffer_count = 0;
  408         ki->kaio_buffer_finished_count = 0;
  409         TAILQ_INIT(&ki->kaio_jobdone);
  410         TAILQ_INIT(&ki->kaio_jobqueue);
  411         TAILQ_INIT(&ki->kaio_bufdone);
  412         TAILQ_INIT(&ki->kaio_bufqueue);
  413         TAILQ_INIT(&ki->kaio_liojoblist);
  414         TAILQ_INIT(&ki->kaio_sockqueue);
  415         PROC_LOCK(p);
  416         if (p->p_aioinfo == NULL) {
  417                 p->p_aioinfo = ki;
  418                 PROC_UNLOCK(p);
  419         } else {
  420                 PROC_UNLOCK(p);
  421                 uma_zfree(kaio_zone, ki);
  422         }
  423 
  424         while (num_aio_procs < target_aio_procs)
  425                 aio_newproc();
  426 }
  427 
  428 /*
  429  * Free a job entry.  Wait for completion if it is currently active, but don't
  430  * delay forever.  If we delay, we return a flag that says that we have to
  431  * restart the queue scan.
  432  */
  433 static int
  434 aio_free_entry(struct aiocblist *aiocbe)
  435 {
  436         struct kaioinfo *ki;
  437         struct aio_liojob *lj;
  438         struct proc *p;
  439         int error;
  440         int s;
  441 
  442         if (aiocbe->jobstate == JOBST_NULL)
  443                 panic("aio_free_entry: freeing already free job");
  444 
  445         p = aiocbe->userproc;
  446         ki = p->p_aioinfo;
  447         lj = aiocbe->lio;
  448         if (ki == NULL)
  449                 panic("aio_free_entry: missing p->p_aioinfo");
  450 
  451         while (aiocbe->jobstate == JOBST_JOBRUNNING) {
  452                 aiocbe->jobflags |= AIOCBLIST_RUNDOWN;
  453                 tsleep(aiocbe, PRIBIO, "jobwai", 0);
  454         }
  455         if (aiocbe->bp == NULL) {
  456                 if (ki->kaio_queue_count <= 0)
  457                         panic("aio_free_entry: process queue size <= 0");
  458                 if (num_queue_count <= 0)
  459                         panic("aio_free_entry: system wide queue size <= 0");
  460 
  461                 if (lj) {
  462                         lj->lioj_queue_count--;
  463                         if (aiocbe->jobflags & AIOCBLIST_DONE)
  464                                 lj->lioj_queue_finished_count--;
  465                 }
  466                 ki->kaio_queue_count--;
  467                 if (aiocbe->jobflags & AIOCBLIST_DONE)
  468                         ki->kaio_queue_finished_count--;
  469                 num_queue_count--;
  470         } else {
  471                 if (lj) {
  472                         lj->lioj_buffer_count--;
  473                         if (aiocbe->jobflags & AIOCBLIST_DONE)
  474                                 lj->lioj_buffer_finished_count--;
  475                 }
  476                 if (aiocbe->jobflags & AIOCBLIST_DONE)
  477                         ki->kaio_buffer_finished_count--;
  478                 ki->kaio_buffer_count--;
  479                 num_buf_aio--;
  480         }
  481 
  482         /* aiocbe is going away, we need to destroy any knotes */
  483         /* XXXKSE Note the thread here is used to eventually find the
  484          * owning process again, but it is also used to do a fo_close
  485          * and that requires the thread. (but does it require the
  486          * OWNING thread? (or maybe the running thread?)
  487          * There is a semantic problem here...
  488          */
  489         knlist_delete(&aiocbe->klist, FIRST_THREAD_IN_PROC(p), 0); /* XXXKSE */
  490 
  491         if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags & KAIO_RUNDOWN)
  492             && ((ki->kaio_buffer_count == 0) && (ki->kaio_queue_count == 0)))) {
  493                 ki->kaio_flags &= ~KAIO_WAKEUP;
  494                 wakeup(p);
  495         }
  496 
  497         if (aiocbe->jobstate == JOBST_JOBQBUF) {
  498                 if ((error = aio_fphysio(aiocbe)) != 0)
  499                         return (error);
  500                 if (aiocbe->jobstate != JOBST_JOBBFINISHED)
  501                         panic("aio_free_entry: invalid physio finish-up state");
  502                 s = splbio();
  503                 TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist);
  504                 splx(s);
  505         } else if (aiocbe->jobstate == JOBST_JOBQGLOBAL) {
  506                 s = splnet();
  507                 TAILQ_REMOVE(&aio_jobs, aiocbe, list);
  508                 TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
  509                 splx(s);
  510         } else if (aiocbe->jobstate == JOBST_JOBFINISHED)
  511                 TAILQ_REMOVE(&ki->kaio_jobdone, aiocbe, plist);
  512         else if (aiocbe->jobstate == JOBST_JOBBFINISHED) {
  513                 s = splbio();
  514                 TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist);
  515                 splx(s);
  516                 if (aiocbe->bp) {
  517                         vunmapbuf(aiocbe->bp);
  518                         relpbuf(aiocbe->bp, NULL);
  519                         aiocbe->bp = NULL;
  520                 }
  521         }
  522         if (lj && (lj->lioj_buffer_count == 0) && (lj->lioj_queue_count == 0)) {
  523                 TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
  524                 uma_zfree(aiolio_zone, lj);
  525         }
  526         aiocbe->jobstate = JOBST_NULL;
  527         fdrop(aiocbe->fd_file, curthread);
  528         crfree(aiocbe->cred);
  529         uma_zfree(aiocb_zone, aiocbe);
  530         return (0);
  531 }
  532 
  533 /*
  534  * Rundown the jobs for a given process.
  535  */
  536 static void
  537 aio_proc_rundown(void *arg, struct proc *p)
  538 {
  539         int s;
  540         struct kaioinfo *ki;
  541         struct aio_liojob *lj, *ljn;
  542         struct aiocblist *aiocbe, *aiocbn;
  543         struct file *fp;
  544         struct socket *so;
  545 
  546         ki = p->p_aioinfo;
  547         if (ki == NULL)
  548                 return;
  549 
  550         mtx_lock(&Giant);
  551         ki->kaio_flags |= LIOJ_SIGNAL_POSTED;
  552         while ((ki->kaio_active_count > 0) || (ki->kaio_buffer_count >
  553             ki->kaio_buffer_finished_count)) {
  554                 ki->kaio_flags |= KAIO_RUNDOWN;
  555                 if (tsleep(p, PRIBIO, "kaiowt", aiod_timeout))
  556                         break;
  557         }
  558 
  559         /*
  560          * Move any aio ops that are waiting on socket I/O to the normal job
  561          * queues so they are cleaned up with any others.
  562          */
  563         s = splnet();
  564         for (aiocbe = TAILQ_FIRST(&ki->kaio_sockqueue); aiocbe; aiocbe =
  565             aiocbn) {
  566                 aiocbn = TAILQ_NEXT(aiocbe, plist);
  567                 fp = aiocbe->fd_file;
  568                 if (fp != NULL) {
  569                         so = fp->f_data;
  570                         TAILQ_REMOVE(&so->so_aiojobq, aiocbe, list);
  571                         if (TAILQ_EMPTY(&so->so_aiojobq)) {
  572                                 SOCKBUF_LOCK(&so->so_snd);
  573                                 so->so_snd.sb_flags &= ~SB_AIO;
  574                                 SOCKBUF_UNLOCK(&so->so_snd);
  575                                 SOCKBUF_LOCK(&so->so_rcv);
  576                                 so->so_rcv.sb_flags &= ~SB_AIO;
  577                                 SOCKBUF_UNLOCK(&so->so_rcv);
  578                         }
  579                 }
  580                 TAILQ_REMOVE(&ki->kaio_sockqueue, aiocbe, plist);
  581                 TAILQ_INSERT_HEAD(&aio_jobs, aiocbe, list);
  582                 TAILQ_INSERT_HEAD(&ki->kaio_jobqueue, aiocbe, plist);
  583         }
  584         splx(s);
  585 
  586 restart1:
  587         for (aiocbe = TAILQ_FIRST(&ki->kaio_jobdone); aiocbe; aiocbe = aiocbn) {
  588                 aiocbn = TAILQ_NEXT(aiocbe, plist);
  589                 if (aio_free_entry(aiocbe))
  590                         goto restart1;
  591         }
  592 
  593 restart2:
  594         for (aiocbe = TAILQ_FIRST(&ki->kaio_jobqueue); aiocbe; aiocbe =
  595             aiocbn) {
  596                 aiocbn = TAILQ_NEXT(aiocbe, plist);
  597                 if (aio_free_entry(aiocbe))
  598                         goto restart2;
  599         }
  600 
  601 /*
  602  * Note the use of lots of splbio here, trying to avoid splbio for long chains
  603  * of I/O.  Probably unnecessary.
  604  */
  605 restart3:
  606         s = splbio();
  607         while (TAILQ_FIRST(&ki->kaio_bufqueue)) {
  608                 ki->kaio_flags |= KAIO_WAKEUP;
  609                 tsleep(p, PRIBIO, "aioprn", 0);
  610                 splx(s);
  611                 goto restart3;
  612         }
  613         splx(s);
  614 
  615 restart4:
  616         s = splbio();
  617         for (aiocbe = TAILQ_FIRST(&ki->kaio_bufdone); aiocbe; aiocbe = aiocbn) {
  618                 aiocbn = TAILQ_NEXT(aiocbe, plist);
  619                 if (aio_free_entry(aiocbe)) {
  620                         splx(s);
  621                         goto restart4;
  622                 }
  623         }
  624         splx(s);
  625 
  626         /*
  627          * If we've slept, jobs might have moved from one queue to another.
  628          * Retry rundown if we didn't manage to empty the queues.
  629          */
  630         if (TAILQ_FIRST(&ki->kaio_jobdone) != NULL ||
  631             TAILQ_FIRST(&ki->kaio_jobqueue) != NULL ||
  632             TAILQ_FIRST(&ki->kaio_bufqueue) != NULL ||
  633             TAILQ_FIRST(&ki->kaio_bufdone) != NULL)
  634                 goto restart1;
  635 
  636         for (lj = TAILQ_FIRST(&ki->kaio_liojoblist); lj; lj = ljn) {
  637                 ljn = TAILQ_NEXT(lj, lioj_list);
  638                 if ((lj->lioj_buffer_count == 0) && (lj->lioj_queue_count ==
  639                     0)) {
  640                         TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
  641                         uma_zfree(aiolio_zone, lj);
  642                 } else {
  643 #ifdef DIAGNOSTIC
  644                         printf("LIO job not cleaned up: B:%d, BF:%d, Q:%d, "
  645                             "QF:%d\n", lj->lioj_buffer_count,
  646                             lj->lioj_buffer_finished_count,
  647                             lj->lioj_queue_count,
  648                             lj->lioj_queue_finished_count);
  649 #endif
  650                 }
  651         }
  652 
  653         uma_zfree(kaio_zone, ki);
  654         p->p_aioinfo = NULL;
  655         mtx_unlock(&Giant);
  656 }
  657 
  658 /*
  659  * Select a job to run (called by an AIO daemon).
  660  */
  661 static struct aiocblist *
  662 aio_selectjob(struct aiothreadlist *aiop)
  663 {
  664         int s;
  665         struct aiocblist *aiocbe;
  666         struct kaioinfo *ki;
  667         struct proc *userp;
  668 
  669         s = splnet();
  670         for (aiocbe = TAILQ_FIRST(&aio_jobs); aiocbe; aiocbe =
  671             TAILQ_NEXT(aiocbe, list)) {
  672                 userp = aiocbe->userproc;
  673                 ki = userp->p_aioinfo;
  674 
  675                 if (ki->kaio_active_count < ki->kaio_maxactive_count) {
  676                         TAILQ_REMOVE(&aio_jobs, aiocbe, list);
  677                         splx(s);
  678                         return (aiocbe);
  679                 }
  680         }
  681         splx(s);
  682 
  683         return (NULL);
  684 }
  685 
  686 /*
  687  * The AIO processing activity.  This is the code that does the I/O request for
  688  * the non-physio version of the operations.  The normal vn operations are used,
  689  * and this code should work in all instances for every type of file, including
  690  * pipes, sockets, fifos, and regular files.
  691  */
  692 static void
  693 aio_process(struct aiocblist *aiocbe)
  694 {
  695         struct ucred *td_savedcred;
  696         struct thread *td;
  697         struct proc *mycp;
  698         struct aiocb *cb;
  699         struct file *fp;
  700         struct uio auio;
  701         struct iovec aiov;
  702         int cnt;
  703         int error;
  704         int oublock_st, oublock_end;
  705         int inblock_st, inblock_end;
  706 
  707         td = curthread;
  708         td_savedcred = td->td_ucred;
  709         td->td_ucred = aiocbe->cred;
  710         mycp = td->td_proc;
  711         cb = &aiocbe->uaiocb;
  712         fp = aiocbe->fd_file;
  713 
  714         aiov.iov_base = (void *)(uintptr_t)cb->aio_buf;
  715         aiov.iov_len = cb->aio_nbytes;
  716 
  717         auio.uio_iov = &aiov;
  718         auio.uio_iovcnt = 1;
  719         auio.uio_offset = cb->aio_offset;
  720         auio.uio_resid = cb->aio_nbytes;
  721         cnt = cb->aio_nbytes;
  722         auio.uio_segflg = UIO_USERSPACE;
  723         auio.uio_td = td;
  724 
  725         inblock_st = mycp->p_stats->p_ru.ru_inblock;
  726         oublock_st = mycp->p_stats->p_ru.ru_oublock;
  727         /*
  728          * _aio_aqueue() acquires a reference to the file that is
  729          * released in aio_free_entry().
  730          */
  731         if (cb->aio_lio_opcode == LIO_READ) {
  732                 auio.uio_rw = UIO_READ;
  733                 error = fo_read(fp, &auio, fp->f_cred, FOF_OFFSET, td);
  734         } else {
  735                 auio.uio_rw = UIO_WRITE;
  736                 error = fo_write(fp, &auio, fp->f_cred, FOF_OFFSET, td);
  737         }
  738         inblock_end = mycp->p_stats->p_ru.ru_inblock;
  739         oublock_end = mycp->p_stats->p_ru.ru_oublock;
  740 
  741         aiocbe->inputcharge = inblock_end - inblock_st;
  742         aiocbe->outputcharge = oublock_end - oublock_st;
  743 
  744         if ((error) && (auio.uio_resid != cnt)) {
  745                 if (error == ERESTART || error == EINTR || error == EWOULDBLOCK)
  746                         error = 0;
  747                 if ((error == EPIPE) && (cb->aio_lio_opcode == LIO_WRITE)) {
  748                         PROC_LOCK(aiocbe->userproc);
  749                         psignal(aiocbe->userproc, SIGPIPE);
  750                         PROC_UNLOCK(aiocbe->userproc);
  751                 }
  752         }
  753 
  754         cnt -= auio.uio_resid;
  755         cb->_aiocb_private.error = error;
  756         cb->_aiocb_private.status = cnt;
  757         td->td_ucred = td_savedcred;
  758 }
  759 
  760 /*
  761  * The AIO daemon, most of the actual work is done in aio_process,
  762  * but the setup (and address space mgmt) is done in this routine.
  763  */
  764 static void
  765 aio_daemon(void *uproc)
  766 {
  767         int s;
  768         struct aio_liojob *lj;
  769         struct aiocb *cb;
  770         struct aiocblist *aiocbe;
  771         struct aiothreadlist *aiop;
  772         struct kaioinfo *ki;
  773         struct proc *curcp, *mycp, *userp;
  774         struct vmspace *myvm, *tmpvm;
  775         struct thread *td = curthread;
  776         struct pgrp *newpgrp;
  777         struct session *newsess;
  778 
  779         /*
  780          * Local copies of curproc (cp) and vmspace (myvm)
  781          */
  782         mycp = td->td_proc;
  783         myvm = mycp->p_vmspace;
  784 
  785         KASSERT(mycp->p_textvp == NULL, ("kthread has a textvp"));
  786 
  787         /*
  788          * Allocate and ready the aio control info.  There is one aiop structure
  789          * per daemon.
  790          */
  791         aiop = uma_zalloc(aiop_zone, M_WAITOK);
  792         aiop->aiothread = td;
  793         aiop->aiothreadflags |= AIOP_FREE;
  794 
  795         /*
  796          * Place thread (lightweight process) onto the AIO free thread list.
  797          */
  798         mtx_lock(&aio_freeproc_mtx);
  799         TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
  800         mtx_unlock(&aio_freeproc_mtx);
  801 
  802         /* The daemon resides in its own pgrp. */
  803         MALLOC(newpgrp, struct pgrp *, sizeof(struct pgrp), M_PGRP,
  804                 M_WAITOK | M_ZERO);
  805         MALLOC(newsess, struct session *, sizeof(struct session), M_SESSION,
  806                 M_WAITOK | M_ZERO);
  807 
  808         sx_xlock(&proctree_lock);
  809         enterpgrp(mycp, mycp->p_pid, newpgrp, newsess);
  810         sx_xunlock(&proctree_lock);
  811         mtx_lock(&Giant);
  812 
  813         /*
  814          * Wakeup parent process.  (Parent sleeps to keep from blasting away
  815          * and creating too many daemons.)
  816          */
  817         wakeup(mycp);
  818 
  819         for (;;) {
  820                 /*
  821                  * curcp is the current daemon process context.
  822                  * userp is the current user process context.
  823                  */
  824                 curcp = mycp;
  825 
  826                 /*
  827                  * Take daemon off of free queue
  828                  */
  829                 mtx_lock(&aio_freeproc_mtx);
  830                 if (aiop->aiothreadflags & AIOP_FREE) {
  831                         TAILQ_REMOVE(&aio_freeproc, aiop, list);
  832                         aiop->aiothreadflags &= ~AIOP_FREE;
  833                 }
  834                 mtx_unlock(&aio_freeproc_mtx);
  835 
  836                 /*
  837                  * Check for jobs.
  838                  */
  839                 while ((aiocbe = aio_selectjob(aiop)) != NULL) {
  840                         cb = &aiocbe->uaiocb;
  841                         userp = aiocbe->userproc;
  842 
  843                         aiocbe->jobstate = JOBST_JOBRUNNING;
  844 
  845                         /*
  846                          * Connect to process address space for user program.
  847                          */
  848                         if (userp != curcp) {
  849                                 /*
  850                                  * Save the current address space that we are
  851                                  * connected to.
  852                                  */
  853                                 tmpvm = mycp->p_vmspace;
  854 
  855                                 /*
  856                                  * Point to the new user address space, and
  857                                  * refer to it.
  858                                  */
  859                                 mycp->p_vmspace = userp->p_vmspace;
  860                                 atomic_add_int(&mycp->p_vmspace->vm_refcnt, 1);
  861 
  862                                 /* Activate the new mapping. */
  863                                 pmap_activate(FIRST_THREAD_IN_PROC(mycp));
  864 
  865                                 /*
  866                                  * If the old address space wasn't the daemons
  867                                  * own address space, then we need to remove the
  868                                  * daemon's reference from the other process
  869                                  * that it was acting on behalf of.
  870                                  */
  871                                 if (tmpvm != myvm) {
  872                                         vmspace_free(tmpvm);
  873                                 }
  874                                 curcp = userp;
  875                         }
  876 
  877                         ki = userp->p_aioinfo;
  878                         lj = aiocbe->lio;
  879 
  880                         /* Account for currently active jobs. */
  881                         ki->kaio_active_count++;
  882 
  883                         /* Do the I/O function. */
  884                         aio_process(aiocbe);
  885 
  886                         /* Decrement the active job count. */
  887                         ki->kaio_active_count--;
  888 
  889                         /*
  890                          * Increment the completion count for wakeup/signal
  891                          * comparisons.
  892                          */
  893                         aiocbe->jobflags |= AIOCBLIST_DONE;
  894                         ki->kaio_queue_finished_count++;
  895                         if (lj)
  896                                 lj->lioj_queue_finished_count++;
  897                         if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags
  898                             & KAIO_RUNDOWN) && (ki->kaio_active_count == 0))) {
  899                                 ki->kaio_flags &= ~KAIO_WAKEUP;
  900                                 wakeup(userp);
  901                         }
  902 
  903                         s = splbio();
  904                         if (lj && (lj->lioj_flags &
  905                             (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) == LIOJ_SIGNAL) {
  906                                 if ((lj->lioj_queue_finished_count ==
  907                                     lj->lioj_queue_count) &&
  908                                     (lj->lioj_buffer_finished_count ==
  909                                     lj->lioj_buffer_count)) {
  910                                         PROC_LOCK(userp);
  911                                         psignal(userp,
  912                                             lj->lioj_signal.sigev_signo);
  913                                         PROC_UNLOCK(userp);
  914                                         lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
  915                                 }
  916                         }
  917                         splx(s);
  918 
  919                         aiocbe->jobstate = JOBST_JOBFINISHED;
  920 
  921                         s = splnet();
  922                         TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
  923                         TAILQ_INSERT_TAIL(&ki->kaio_jobdone, aiocbe, plist);
  924                         splx(s);
  925                         KNOTE_UNLOCKED(&aiocbe->klist, 0);
  926 
  927                         if (aiocbe->jobflags & AIOCBLIST_RUNDOWN) {
  928                                 wakeup(aiocbe);
  929                                 aiocbe->jobflags &= ~AIOCBLIST_RUNDOWN;
  930                         }
  931 
  932                         if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) {
  933                                 PROC_LOCK(userp);
  934                                 psignal(userp, cb->aio_sigevent.sigev_signo);
  935                                 PROC_UNLOCK(userp);
  936                         }
  937                 }
  938 
  939                 /*
  940                  * Disconnect from user address space.
  941                  */
  942                 if (curcp != mycp) {
  943                         /* Get the user address space to disconnect from. */
  944                         tmpvm = mycp->p_vmspace;
  945 
  946                         /* Get original address space for daemon. */
  947                         mycp->p_vmspace = myvm;
  948 
  949                         /* Activate the daemon's address space. */
  950                         pmap_activate(FIRST_THREAD_IN_PROC(mycp));
  951 #ifdef DIAGNOSTIC
  952                         if (tmpvm == myvm) {
  953                                 printf("AIOD: vmspace problem -- %d\n",
  954                                     mycp->p_pid);
  955                         }
  956 #endif
  957                         /* Remove our vmspace reference. */
  958                         vmspace_free(tmpvm);
  959 
  960                         curcp = mycp;
  961                 }
  962 
  963                 mtx_lock(&aio_freeproc_mtx);
  964                 TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
  965                 aiop->aiothreadflags |= AIOP_FREE;
  966 
  967                 /*
  968                  * If daemon is inactive for a long time, allow it to exit,
  969                  * thereby freeing resources.
  970                  */
  971                 if (msleep(aiop->aiothread, &aio_freeproc_mtx, PDROP | PRIBIO,
  972                     "aiordy", aiod_lifetime)) {
  973                         s = splnet();
  974                         if (TAILQ_EMPTY(&aio_jobs)) {
  975                                 mtx_lock(&aio_freeproc_mtx);
  976                                 if ((aiop->aiothreadflags & AIOP_FREE) &&
  977                                     (num_aio_procs > target_aio_procs)) {
  978                                         TAILQ_REMOVE(&aio_freeproc, aiop, list);
  979                                         mtx_unlock(&aio_freeproc_mtx);
  980                                         splx(s);
  981                                         uma_zfree(aiop_zone, aiop);
  982                                         num_aio_procs--;
  983 #ifdef DIAGNOSTIC
  984                                         if (mycp->p_vmspace->vm_refcnt <= 1) {
  985                                                 printf("AIOD: bad vm refcnt for"
  986                                                     " exiting daemon: %d\n",
  987                                                     mycp->p_vmspace->vm_refcnt);
  988                                         }
  989 #endif
  990                                         kthread_exit(0);
  991                                 }
  992                                 mtx_unlock(&aio_freeproc_mtx);
  993                         }
  994                         splx(s);
  995                 }
  996         }
  997 }
  998 
  999 /*
 1000  * Create a new AIO daemon.  This is mostly a kernel-thread fork routine.  The
 1001  * AIO daemon modifies its environment itself.
 1002  */
 1003 static int
 1004 aio_newproc(void)
 1005 {
 1006         int error;
 1007         struct proc *p;
 1008 
 1009         error = kthread_create(aio_daemon, curproc, &p, RFNOWAIT, 0, "aiod%d",
 1010             num_aio_procs);
 1011         if (error)
 1012                 return (error);
 1013 
 1014         /*
 1015          * Wait until daemon is started, but continue on just in case to
 1016          * handle error conditions.
 1017          */
 1018         error = tsleep(p, PZERO, "aiosta", aiod_timeout);
 1019 
 1020         num_aio_procs++;
 1021 
 1022         return (error);
 1023 }
 1024 
 1025 /*
 1026  * Try the high-performance, low-overhead physio method for eligible
 1027  * VCHR devices.  This method doesn't use an aio helper thread, and
 1028  * thus has very low overhead.
 1029  *
 1030  * Assumes that the caller, _aio_aqueue(), has incremented the file
 1031  * structure's reference count, preventing its deallocation for the
 1032  * duration of this call.
 1033  */
 1034 static int
 1035 aio_qphysio(struct proc *p, struct aiocblist *aiocbe)
 1036 {
 1037         int error;
 1038         struct aiocb *cb;
 1039         struct file *fp;
 1040         struct buf *bp;
 1041         struct vnode *vp;
 1042         struct kaioinfo *ki;
 1043         struct aio_liojob *lj;
 1044         int s;
 1045         int notify;
 1046 
 1047         cb = &aiocbe->uaiocb;
 1048         fp = aiocbe->fd_file;
 1049 
 1050         if (fp->f_type != DTYPE_VNODE)
 1051                 return (-1);
 1052 
 1053         vp = fp->f_vnode;
 1054 
 1055         /*
 1056          * If its not a disk, we don't want to return a positive error.
 1057          * It causes the aio code to not fall through to try the thread
 1058          * way when you're talking to a regular file.
 1059          */
 1060         if (!vn_isdisk(vp, &error)) {
 1061                 if (error == ENOTBLK)
 1062                         return (-1);
 1063                 else
 1064                         return (error);
 1065         }
 1066 
 1067         if (vp->v_bufobj.bo_bsize == 0)
 1068                 return (-1);
 1069 
 1070         if (cb->aio_nbytes % vp->v_bufobj.bo_bsize)
 1071                 return (-1);
 1072 
 1073         if (cb->aio_nbytes >
 1074             MAXPHYS - (((vm_offset_t) cb->aio_buf) & PAGE_MASK))
 1075                 return (-1);
 1076 
 1077         ki = p->p_aioinfo;
 1078         if (ki->kaio_buffer_count >= ki->kaio_ballowed_count)
 1079                 return (-1);
 1080 
 1081         ki->kaio_buffer_count++;
 1082 
 1083         lj = aiocbe->lio;
 1084         if (lj)
 1085                 lj->lioj_buffer_count++;
 1086 
 1087         /* Create and build a buffer header for a transfer. */
 1088         bp = (struct buf *)getpbuf(NULL);
 1089         BUF_KERNPROC(bp);
 1090 
 1091         /*
 1092          * Get a copy of the kva from the physical buffer.
 1093          */
 1094         error = 0;
 1095 
 1096         bp->b_bcount = cb->aio_nbytes;
 1097         bp->b_bufsize = cb->aio_nbytes;
 1098         bp->b_iodone = aio_physwakeup;
 1099         bp->b_saveaddr = bp->b_data;
 1100         bp->b_data = (void *)(uintptr_t)cb->aio_buf;
 1101         bp->b_offset = cb->aio_offset;
 1102         bp->b_iooffset = cb->aio_offset;
 1103         bp->b_blkno = btodb(cb->aio_offset);
 1104         bp->b_iocmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ;
 1105 
 1106         /*
 1107          * Bring buffer into kernel space.
 1108          */
 1109         if (vmapbuf(bp) < 0) {
 1110                 error = EFAULT;
 1111                 goto doerror;
 1112         }
 1113 
 1114         s = splbio();
 1115         aiocbe->bp = bp;
 1116         bp->b_caller1 = (void *)aiocbe;
 1117         TAILQ_INSERT_TAIL(&ki->kaio_bufqueue, aiocbe, plist);
 1118         aiocbe->jobstate = JOBST_JOBQBUF;
 1119         cb->_aiocb_private.status = cb->aio_nbytes;
 1120         num_buf_aio++;
 1121         bp->b_error = 0;
 1122 
 1123         splx(s);
 1124 
 1125         /* Perform transfer. */
 1126         dev_strategy(vp->v_rdev, bp);
 1127 
 1128         notify = 0;
 1129         s = splbio();
 1130 
 1131         /*
 1132          * If we had an error invoking the request, or an error in processing
 1133          * the request before we have returned, we process it as an error in
 1134          * transfer.  Note that such an I/O error is not indicated immediately,
 1135          * but is returned using the aio_error mechanism.  In this case,
 1136          * aio_suspend will return immediately.
 1137          */
 1138         if (bp->b_error || (bp->b_ioflags & BIO_ERROR)) {
 1139                 struct aiocb *job = aiocbe->uuaiocb;
 1140 
 1141                 aiocbe->uaiocb._aiocb_private.status = 0;
 1142                 suword(&job->_aiocb_private.status, 0);
 1143                 aiocbe->uaiocb._aiocb_private.error = bp->b_error;
 1144                 suword(&job->_aiocb_private.error, bp->b_error);
 1145 
 1146                 ki->kaio_buffer_finished_count++;
 1147 
 1148                 if (aiocbe->jobstate != JOBST_JOBBFINISHED) {
 1149                         aiocbe->jobstate = JOBST_JOBBFINISHED;
 1150                         aiocbe->jobflags |= AIOCBLIST_DONE;
 1151                         TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist);
 1152                         TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist);
 1153                         notify = 1;
 1154                 }
 1155         }
 1156         splx(s);
 1157         if (notify)
 1158                 KNOTE_UNLOCKED(&aiocbe->klist, 0);
 1159         return (0);
 1160 
 1161 doerror:
 1162         ki->kaio_buffer_count--;
 1163         if (lj)
 1164                 lj->lioj_buffer_count--;
 1165         aiocbe->bp = NULL;
 1166         relpbuf(bp, NULL);
 1167         return (error);
 1168 }
 1169 
 1170 /*
 1171  * This waits/tests physio completion.
 1172  */
 1173 static int
 1174 aio_fphysio(struct aiocblist *iocb)
 1175 {
 1176         int s;
 1177         struct buf *bp;
 1178         int error;
 1179 
 1180         bp = iocb->bp;
 1181 
 1182         s = splbio();
 1183         while ((bp->b_flags & B_DONE) == 0) {
 1184                 if (tsleep(bp, PRIBIO, "physstr", aiod_timeout)) {
 1185                         if ((bp->b_flags & B_DONE) == 0) {
 1186                                 splx(s);
 1187                                 return (EINPROGRESS);
 1188                         } else
 1189                                 break;
 1190                 }
 1191         }
 1192         splx(s);
 1193 
 1194         /* Release mapping into kernel space. */
 1195         vunmapbuf(bp);
 1196         iocb->bp = 0;
 1197 
 1198         error = 0;
 1199 
 1200         /* Check for an error. */
 1201         if (bp->b_ioflags & BIO_ERROR)
 1202                 error = bp->b_error;
 1203 
 1204         relpbuf(bp, NULL);
 1205         return (error);
 1206 }
 1207 
 1208 /*
 1209  * Wake up aio requests that may be serviceable now.
 1210  */
 1211 static void
 1212 aio_swake_cb(struct socket *so, struct sockbuf *sb)
 1213 {
 1214         struct aiocblist *cb,*cbn;
 1215         struct proc *p;
 1216         struct kaioinfo *ki = NULL;
 1217         int opcode, wakecount = 0;
 1218         struct aiothreadlist *aiop;
 1219 
 1220         if (sb == &so->so_snd) {
 1221                 opcode = LIO_WRITE;
 1222                 SOCKBUF_LOCK(&so->so_snd);
 1223                 so->so_snd.sb_flags &= ~SB_AIO;
 1224                 SOCKBUF_UNLOCK(&so->so_snd);
 1225         } else {
 1226                 opcode = LIO_READ;
 1227                 SOCKBUF_LOCK(&so->so_rcv);
 1228                 so->so_rcv.sb_flags &= ~SB_AIO;
 1229                 SOCKBUF_UNLOCK(&so->so_rcv);
 1230         }
 1231 
 1232         for (cb = TAILQ_FIRST(&so->so_aiojobq); cb; cb = cbn) {
 1233                 cbn = TAILQ_NEXT(cb, list);
 1234                 if (opcode == cb->uaiocb.aio_lio_opcode) {
 1235                         p = cb->userproc;
 1236                         ki = p->p_aioinfo;
 1237                         TAILQ_REMOVE(&so->so_aiojobq, cb, list);
 1238                         TAILQ_REMOVE(&ki->kaio_sockqueue, cb, plist);
 1239                         TAILQ_INSERT_TAIL(&aio_jobs, cb, list);
 1240                         TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, cb, plist);
 1241                         wakecount++;
 1242                         if (cb->jobstate != JOBST_JOBQGLOBAL)
 1243                                 panic("invalid queue value");
 1244                 }
 1245         }
 1246 
 1247         while (wakecount--) {
 1248                 mtx_lock(&aio_freeproc_mtx);
 1249                 if ((aiop = TAILQ_FIRST(&aio_freeproc)) != 0) {
 1250                         TAILQ_REMOVE(&aio_freeproc, aiop, list);
 1251                         aiop->aiothreadflags &= ~AIOP_FREE;
 1252                         wakeup(aiop->aiothread);
 1253                 }
 1254                 mtx_unlock(&aio_freeproc_mtx);
 1255         }
 1256 }
 1257 
 1258 /*
 1259  * Queue a new AIO request.  Choosing either the threaded or direct physio VCHR
 1260  * technique is done in this code.
 1261  */
 1262 static int
 1263 _aio_aqueue(struct thread *td, struct aiocb *job, struct aio_liojob *lj, int type)
 1264 {
 1265         struct proc *p = td->td_proc;
 1266         struct filedesc *fdp;
 1267         struct file *fp;
 1268         unsigned int fd;
 1269         struct socket *so;
 1270         int s;
 1271         int error;
 1272         int opcode;
 1273         struct aiocblist *aiocbe;
 1274         struct aiothreadlist *aiop;
 1275         struct kaioinfo *ki;
 1276         struct kevent kev;
 1277         struct kqueue *kq;
 1278         struct file *kq_fp;
 1279         struct sockbuf *sb;
 1280 
 1281         aiocbe = uma_zalloc(aiocb_zone, M_WAITOK);
 1282         aiocbe->inputcharge = 0;
 1283         aiocbe->outputcharge = 0;
 1284         /* XXX - need a lock */
 1285         knlist_init(&aiocbe->klist, NULL, NULL, NULL, NULL);
 1286 
 1287         suword(&job->_aiocb_private.status, -1);
 1288         suword(&job->_aiocb_private.error, 0);
 1289         suword(&job->_aiocb_private.kernelinfo, -1);
 1290 
 1291         error = copyin(job, &aiocbe->uaiocb, sizeof(aiocbe->uaiocb));
 1292         if (error) {
 1293                 suword(&job->_aiocb_private.error, error);
 1294                 uma_zfree(aiocb_zone, aiocbe);
 1295                 return (error);
 1296         }
 1297         if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL &&
 1298                 !_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) {
 1299                 uma_zfree(aiocb_zone, aiocbe);
 1300                 return (EINVAL);
 1301         }
 1302 
 1303         /* Save userspace address of the job info. */
 1304         aiocbe->uuaiocb = job;
 1305 
 1306         /* Get the opcode. */
 1307         if (type != LIO_NOP)
 1308                 aiocbe->uaiocb.aio_lio_opcode = type;
 1309         opcode = aiocbe->uaiocb.aio_lio_opcode;
 1310 
 1311         /* Get the fd info for process. */
 1312         fdp = p->p_fd;
 1313 
 1314         /*
 1315          * Range check file descriptor.
 1316          */
 1317         FILEDESC_LOCK(fdp);
 1318         fd = aiocbe->uaiocb.aio_fildes;
 1319         if (fd >= fdp->fd_nfiles) {
 1320                 FILEDESC_UNLOCK(fdp);
 1321                 uma_zfree(aiocb_zone, aiocbe);
 1322                 if (type == 0)
 1323                         suword(&job->_aiocb_private.error, EBADF);
 1324                 return (EBADF);
 1325         }
 1326 
 1327         fp = aiocbe->fd_file = fdp->fd_ofiles[fd];
 1328         if ((fp == NULL) ||
 1329             ((opcode == LIO_WRITE) && ((fp->f_flag & FWRITE) == 0)) ||
 1330             ((opcode == LIO_READ) && ((fp->f_flag & FREAD) == 0))) {
 1331                 FILEDESC_UNLOCK(fdp);
 1332                 uma_zfree(aiocb_zone, aiocbe);
 1333                 if (type == 0)
 1334                         suword(&job->_aiocb_private.error, EBADF);
 1335                 return (EBADF);
 1336         }
 1337         fhold(fp);
 1338         FILEDESC_UNLOCK(fdp);
 1339 
 1340         if (aiocbe->uaiocb.aio_offset == -1LL) {
 1341                 error = EINVAL;
 1342                 goto aqueue_fail;
 1343         }
 1344         error = suword(&job->_aiocb_private.kernelinfo, jobrefid);
 1345         if (error) {
 1346                 error = EINVAL;
 1347                 goto aqueue_fail;
 1348         }
 1349         aiocbe->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jobrefid;
 1350         if (jobrefid == LONG_MAX)
 1351                 jobrefid = 1;
 1352         else
 1353                 jobrefid++;
 1354 
 1355         if (opcode == LIO_NOP) {
 1356                 fdrop(fp, td);
 1357                 uma_zfree(aiocb_zone, aiocbe);
 1358                 if (type == 0) {
 1359                         suword(&job->_aiocb_private.error, 0);
 1360                         suword(&job->_aiocb_private.status, 0);
 1361                         suword(&job->_aiocb_private.kernelinfo, 0);
 1362                 }
 1363                 return (0);
 1364         }
 1365         if ((opcode != LIO_READ) && (opcode != LIO_WRITE)) {
 1366                 if (type == 0)
 1367                         suword(&job->_aiocb_private.status, 0);
 1368                 error = EINVAL;
 1369                 goto aqueue_fail;
 1370         }
 1371 
 1372         if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_KEVENT) {
 1373                 kev.ident = aiocbe->uaiocb.aio_sigevent.sigev_notify_kqueue;
 1374                 kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sigval_ptr;
 1375         } else
 1376                 goto no_kqueue;
 1377         if ((u_int)kev.ident >= fdp->fd_nfiles ||
 1378             (kq_fp = fdp->fd_ofiles[kev.ident]) == NULL ||
 1379             (kq_fp->f_type != DTYPE_KQUEUE)) {
 1380                 error = EBADF;
 1381                 goto aqueue_fail;
 1382         }
 1383         kq = kq_fp->f_data;
 1384         kev.ident = (uintptr_t)aiocbe->uuaiocb;
 1385         kev.filter = EVFILT_AIO;
 1386         kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
 1387         kev.data = (intptr_t)aiocbe;
 1388         error = kqueue_register(kq, &kev, td, 1);
 1389 aqueue_fail:
 1390         if (error) {
 1391                 fdrop(fp, td);
 1392                 uma_zfree(aiocb_zone, aiocbe);
 1393                 if (type == 0)
 1394                         suword(&job->_aiocb_private.error, error);
 1395                 goto done;
 1396         }
 1397 no_kqueue:
 1398 
 1399         suword(&job->_aiocb_private.error, EINPROGRESS);
 1400         aiocbe->uaiocb._aiocb_private.error = EINPROGRESS;
 1401         aiocbe->userproc = p;
 1402         aiocbe->cred = crhold(td->td_ucred);
 1403         aiocbe->jobflags = 0;
 1404         aiocbe->lio = lj;
 1405         ki = p->p_aioinfo;
 1406 
 1407         if (fp->f_type == DTYPE_SOCKET) {
 1408                 /*
 1409                  * Alternate queueing for socket ops: Reach down into the
 1410                  * descriptor to get the socket data.  Then check to see if the
 1411                  * socket is ready to be read or written (based on the requested
 1412                  * operation).
 1413                  *
 1414                  * If it is not ready for io, then queue the aiocbe on the
 1415                  * socket, and set the flags so we get a call when sbnotify()
 1416                  * happens.
 1417                  *
 1418                  * Note if opcode is neither LIO_WRITE nor LIO_READ we lock
 1419                  * and unlock the snd sockbuf for no reason.
 1420                  */
 1421                 so = fp->f_data;
 1422                 sb = (opcode == LIO_READ) ? &so->so_rcv : &so->so_snd;
 1423                 SOCKBUF_LOCK(sb);
 1424                 s = splnet();
 1425                 if (((opcode == LIO_READ) && (!soreadable(so))) || ((opcode ==
 1426                     LIO_WRITE) && (!sowriteable(so)))) {
 1427                         TAILQ_INSERT_TAIL(&so->so_aiojobq, aiocbe, list);
 1428                         TAILQ_INSERT_TAIL(&ki->kaio_sockqueue, aiocbe, plist);
 1429                         sb->sb_flags |= SB_AIO;
 1430                         aiocbe->jobstate = JOBST_JOBQGLOBAL; /* XXX */
 1431                         ki->kaio_queue_count++;
 1432                         num_queue_count++;
 1433                         SOCKBUF_UNLOCK(sb);
 1434                         splx(s);
 1435                         error = 0;
 1436                         goto done;
 1437                 }
 1438                 SOCKBUF_UNLOCK(sb);
 1439                 splx(s);
 1440         }
 1441 
 1442         if ((error = aio_qphysio(p, aiocbe)) == 0)
 1443                 goto done;
 1444         if (error > 0) {
 1445                 suword(&job->_aiocb_private.status, 0);
 1446                 aiocbe->uaiocb._aiocb_private.error = error;
 1447                 suword(&job->_aiocb_private.error, error);
 1448                 goto done;
 1449         }
 1450 
 1451         /* No buffer for daemon I/O. */
 1452         aiocbe->bp = NULL;
 1453 
 1454         ki->kaio_queue_count++;
 1455         if (lj)
 1456                 lj->lioj_queue_count++;
 1457         s = splnet();
 1458         TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist);
 1459         TAILQ_INSERT_TAIL(&aio_jobs, aiocbe, list);
 1460         splx(s);
 1461         aiocbe->jobstate = JOBST_JOBQGLOBAL;
 1462 
 1463         num_queue_count++;
 1464         error = 0;
 1465 
 1466         /*
 1467          * If we don't have a free AIO process, and we are below our quota, then
 1468          * start one.  Otherwise, depend on the subsequent I/O completions to
 1469          * pick-up this job.  If we don't sucessfully create the new process
 1470          * (thread) due to resource issues, we return an error for now (EAGAIN),
 1471          * which is likely not the correct thing to do.
 1472          */
 1473         mtx_lock(&aio_freeproc_mtx);
 1474 retryproc:
 1475         if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
 1476                 TAILQ_REMOVE(&aio_freeproc, aiop, list);
 1477                 aiop->aiothreadflags &= ~AIOP_FREE;
 1478                 wakeup(aiop->aiothread);
 1479         } else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) &&
 1480             ((ki->kaio_active_count + num_aio_resv_start) <
 1481             ki->kaio_maxactive_count)) {
 1482                 num_aio_resv_start++;
 1483                 mtx_unlock(&aio_freeproc_mtx);
 1484                 if ((error = aio_newproc()) == 0) {
 1485                         mtx_lock(&aio_freeproc_mtx);
 1486                         num_aio_resv_start--;
 1487                         goto retryproc;
 1488                 }
 1489                 mtx_lock(&aio_freeproc_mtx);
 1490                 num_aio_resv_start--;
 1491         }
 1492         mtx_unlock(&aio_freeproc_mtx);
 1493 done:
 1494         return (error);
 1495 }
 1496 
 1497 /*
 1498  * This routine queues an AIO request, checking for quotas.
 1499  */
 1500 static int
 1501 aio_aqueue(struct thread *td, struct aiocb *job, int type)
 1502 {
 1503         struct proc *p = td->td_proc;
 1504         struct kaioinfo *ki;
 1505 
 1506         if (p->p_aioinfo == NULL)
 1507                 aio_init_aioinfo(p);
 1508 
 1509         if (num_queue_count >= max_queue_count)
 1510                 return (EAGAIN);
 1511 
 1512         ki = p->p_aioinfo;
 1513         if (ki->kaio_queue_count >= ki->kaio_qallowed_count)
 1514                 return (EAGAIN);
 1515 
 1516         return _aio_aqueue(td, job, NULL, type);
 1517 }
 1518 
 1519 /*
 1520  * Support the aio_return system call, as a side-effect, kernel resources are
 1521  * released.
 1522  */
 1523 int
 1524 aio_return(struct thread *td, struct aio_return_args *uap)
 1525 {
 1526         struct proc *p = td->td_proc;
 1527         int s;
 1528         long jobref;
 1529         struct aiocblist *cb, *ncb;
 1530         struct aiocb *ujob;
 1531         struct kaioinfo *ki;
 1532 
 1533         ujob = uap->aiocbp;
 1534         jobref = fuword(&ujob->_aiocb_private.kernelinfo);
 1535         if (jobref == -1 || jobref == 0)
 1536                 return (EINVAL);
 1537 
 1538         ki = p->p_aioinfo;
 1539         if (ki == NULL)
 1540                 return (EINVAL);
 1541         PROC_LOCK(p);
 1542         TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
 1543                 if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo) ==
 1544                     jobref) {
 1545                         if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
 1546                                 p->p_stats->p_ru.ru_oublock +=
 1547                                     cb->outputcharge;
 1548                                 cb->outputcharge = 0;
 1549                         } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
 1550                                 p->p_stats->p_ru.ru_inblock += cb->inputcharge;
 1551                                 cb->inputcharge = 0;
 1552                         }
 1553                         goto done;
 1554                 }
 1555         }
 1556         s = splbio();
 1557         for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = ncb) {
 1558                 ncb = TAILQ_NEXT(cb, plist);
 1559                 if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo)
 1560                     == jobref) {
 1561                         break;
 1562                 }
 1563         }
 1564         splx(s);
 1565  done:
 1566         PROC_UNLOCK(p);
 1567         if (cb != NULL) {
 1568                 if (ujob == cb->uuaiocb) {
 1569                         td->td_retval[0] =
 1570                             cb->uaiocb._aiocb_private.status;
 1571                 } else
 1572                         td->td_retval[0] = EFAULT;
 1573                 aio_free_entry(cb);
 1574                 return (0);
 1575         }
 1576         return (EINVAL);
 1577 }
 1578 
 1579 /*
 1580  * Allow a process to wakeup when any of the I/O requests are completed.
 1581  */
 1582 int
 1583 aio_suspend(struct thread *td, struct aio_suspend_args *uap)
 1584 {
 1585         struct proc *p = td->td_proc;
 1586         struct timeval atv;
 1587         struct timespec ts;
 1588         struct aiocb *const *cbptr, *cbp;
 1589         struct kaioinfo *ki;
 1590         struct aiocblist *cb;
 1591         int i;
 1592         int njoblist;
 1593         int error, s, timo;
 1594         long *ijoblist;
 1595         struct aiocb **ujoblist;
 1596 
 1597         if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
 1598                 return (EINVAL);
 1599 
 1600         timo = 0;
 1601         if (uap->timeout) {
 1602                 /* Get timespec struct. */
 1603                 if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0)
 1604                         return (error);
 1605 
 1606                 if (ts.tv_nsec < 0 || ts.tv_nsec >= 1000000000)
 1607                         return (EINVAL);
 1608 
 1609                 TIMESPEC_TO_TIMEVAL(&atv, &ts);
 1610                 if (itimerfix(&atv))
 1611                         return (EINVAL);
 1612                 timo = tvtohz(&atv);
 1613         }
 1614 
 1615         ki = p->p_aioinfo;
 1616         if (ki == NULL)
 1617                 return (EAGAIN);
 1618 
 1619         njoblist = 0;
 1620         ijoblist = uma_zalloc(aiol_zone, M_WAITOK);
 1621         ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
 1622         cbptr = uap->aiocbp;
 1623 
 1624         for (i = 0; i < uap->nent; i++) {
 1625                 cbp = (struct aiocb *)(intptr_t)fuword(&cbptr[i]);
 1626                 if (cbp == 0)
 1627                         continue;
 1628                 ujoblist[njoblist] = cbp;
 1629                 ijoblist[njoblist] = fuword(&cbp->_aiocb_private.kernelinfo);
 1630                 njoblist++;
 1631         }
 1632 
 1633         if (njoblist == 0) {
 1634                 uma_zfree(aiol_zone, ijoblist);
 1635                 uma_zfree(aiol_zone, ujoblist);
 1636                 return (0);
 1637         }
 1638 
 1639         error = 0;
 1640         for (;;) {
 1641                 PROC_LOCK(p);
 1642                 TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
 1643                         for (i = 0; i < njoblist; i++) {
 1644                                 if (((intptr_t)
 1645                                     cb->uaiocb._aiocb_private.kernelinfo) ==
 1646                                     ijoblist[i]) {
 1647                                         PROC_UNLOCK(p);
 1648                                         if (ujoblist[i] != cb->uuaiocb)
 1649                                                 error = EINVAL;
 1650                                         uma_zfree(aiol_zone, ijoblist);
 1651                                         uma_zfree(aiol_zone, ujoblist);
 1652                                         return (error);
 1653                                 }
 1654                         }
 1655                 }
 1656 
 1657                 s = splbio();
 1658                 for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb =
 1659                     TAILQ_NEXT(cb, plist)) {
 1660                         for (i = 0; i < njoblist; i++) {
 1661                                 if (((intptr_t)
 1662                                     cb->uaiocb._aiocb_private.kernelinfo) ==
 1663                                     ijoblist[i]) {
 1664                                         PROC_UNLOCK(p);
 1665                                         splx(s);
 1666                                         if (ujoblist[i] != cb->uuaiocb)
 1667                                                 error = EINVAL;
 1668                                         uma_zfree(aiol_zone, ijoblist);
 1669                                         uma_zfree(aiol_zone, ujoblist);
 1670                                         return (error);
 1671                                 }
 1672                         }
 1673                 }
 1674 
 1675                 ki->kaio_flags |= KAIO_WAKEUP;
 1676                 error = msleep(p, &p->p_mtx, PDROP | PRIBIO | PCATCH, "aiospn",
 1677                     timo);
 1678                 splx(s);
 1679 
 1680                 if (error == ERESTART || error == EINTR) {
 1681                         uma_zfree(aiol_zone, ijoblist);
 1682                         uma_zfree(aiol_zone, ujoblist);
 1683                         return (EINTR);
 1684                 } else if (error == EWOULDBLOCK) {
 1685                         uma_zfree(aiol_zone, ijoblist);
 1686                         uma_zfree(aiol_zone, ujoblist);
 1687                         return (EAGAIN);
 1688                 }
 1689         }
 1690 
 1691 /* NOTREACHED */
 1692         return (EINVAL);
 1693 }
 1694 
 1695 /*
 1696  * aio_cancel cancels any non-physio aio operations not currently in
 1697  * progress.
 1698  */
 1699 int
 1700 aio_cancel(struct thread *td, struct aio_cancel_args *uap)
 1701 {
 1702         struct proc *p = td->td_proc;
 1703         struct kaioinfo *ki;
 1704         struct aiocblist *cbe, *cbn;
 1705         struct file *fp;
 1706         struct filedesc *fdp;
 1707         struct socket *so;
 1708         struct proc *po;
 1709         int s,error;
 1710         int cancelled=0;
 1711         int notcancelled=0;
 1712         struct vnode *vp;
 1713 
 1714         fdp = p->p_fd;
 1715         if ((u_int)uap->fd >= fdp->fd_nfiles ||
 1716             (fp = fdp->fd_ofiles[uap->fd]) == NULL)
 1717                 return (EBADF);
 1718 
 1719         if (fp->f_type == DTYPE_VNODE) {
 1720                 vp = fp->f_vnode;
 1721 
 1722                 if (vn_isdisk(vp,&error)) {
 1723                         td->td_retval[0] = AIO_NOTCANCELED;
 1724                         return (0);
 1725                 }
 1726         } else if (fp->f_type == DTYPE_SOCKET) {
 1727                 so = fp->f_data;
 1728 
 1729                 s = splnet();
 1730 
 1731                 for (cbe = TAILQ_FIRST(&so->so_aiojobq); cbe; cbe = cbn) {
 1732                         cbn = TAILQ_NEXT(cbe, list);
 1733                         if ((uap->aiocbp == NULL) ||
 1734                                 (uap->aiocbp == cbe->uuaiocb) ) {
 1735                                 po = cbe->userproc;
 1736                                 ki = po->p_aioinfo;
 1737                                 TAILQ_REMOVE(&so->so_aiojobq, cbe, list);
 1738                                 TAILQ_REMOVE(&ki->kaio_sockqueue, cbe, plist);
 1739                                 TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe, plist);
 1740                                 if (ki->kaio_flags & KAIO_WAKEUP) {
 1741                                         wakeup(po);
 1742                                 }
 1743                                 cbe->jobstate = JOBST_JOBFINISHED;
 1744                                 cbe->uaiocb._aiocb_private.status=-1;
 1745                                 cbe->uaiocb._aiocb_private.error=ECANCELED;
 1746                                 cancelled++;
 1747 /* XXX cancelled, knote? */
 1748                                 if (cbe->uaiocb.aio_sigevent.sigev_notify ==
 1749                                     SIGEV_SIGNAL) {
 1750                                         PROC_LOCK(cbe->userproc);
 1751                                         psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo);
 1752                                         PROC_UNLOCK(cbe->userproc);
 1753                                 }
 1754                                 if (uap->aiocbp)
 1755                                         break;
 1756                         }
 1757                 }
 1758                 splx(s);
 1759 
 1760                 if ((cancelled) && (uap->aiocbp)) {
 1761                         td->td_retval[0] = AIO_CANCELED;
 1762                         return (0);
 1763                 }
 1764         }
 1765         ki=p->p_aioinfo;
 1766         if (ki == NULL)
 1767                 goto done;
 1768         s = splnet();
 1769 
 1770         for (cbe = TAILQ_FIRST(&ki->kaio_jobqueue); cbe; cbe = cbn) {
 1771                 cbn = TAILQ_NEXT(cbe, plist);
 1772 
 1773                 if ((uap->fd == cbe->uaiocb.aio_fildes) &&
 1774                     ((uap->aiocbp == NULL ) ||
 1775                      (uap->aiocbp == cbe->uuaiocb))) {
 1776 
 1777                         if (cbe->jobstate == JOBST_JOBQGLOBAL) {
 1778                                 TAILQ_REMOVE(&aio_jobs, cbe, list);
 1779                                 TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist);
 1780                                 TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe,
 1781                                     plist);
 1782                                 cancelled++;
 1783                                 ki->kaio_queue_finished_count++;
 1784                                 cbe->jobstate = JOBST_JOBFINISHED;
 1785                                 cbe->uaiocb._aiocb_private.status = -1;
 1786                                 cbe->uaiocb._aiocb_private.error = ECANCELED;
 1787 /* XXX cancelled, knote? */
 1788                                 if (cbe->uaiocb.aio_sigevent.sigev_notify ==
 1789                                     SIGEV_SIGNAL) {
 1790                                         PROC_LOCK(cbe->userproc);
 1791                                         psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo);
 1792                                         PROC_UNLOCK(cbe->userproc);
 1793                                 }
 1794                         } else {
 1795                                 notcancelled++;
 1796                         }
 1797                 }
 1798         }
 1799         splx(s);
 1800 done:
 1801         if (notcancelled) {
 1802                 td->td_retval[0] = AIO_NOTCANCELED;
 1803                 return (0);
 1804         }
 1805         if (cancelled) {
 1806                 td->td_retval[0] = AIO_CANCELED;
 1807                 return (0);
 1808         }
 1809         td->td_retval[0] = AIO_ALLDONE;
 1810 
 1811         return (0);
 1812 }
 1813 
 1814 /*
 1815  * aio_error is implemented in the kernel level for compatibility purposes only.
 1816  * For a user mode async implementation, it would be best to do it in a userland
 1817  * subroutine.
 1818  */
 1819 int
 1820 aio_error(struct thread *td, struct aio_error_args *uap)
 1821 {
 1822         struct proc *p = td->td_proc;
 1823         int s;
 1824         struct aiocblist *cb;
 1825         struct kaioinfo *ki;
 1826         long jobref;
 1827 
 1828         ki = p->p_aioinfo;
 1829         if (ki == NULL)
 1830                 return (EINVAL);
 1831 
 1832         jobref = fuword(&uap->aiocbp->_aiocb_private.kernelinfo);
 1833         if ((jobref == -1) || (jobref == 0))
 1834                 return (EINVAL);
 1835 
 1836         PROC_LOCK(p);
 1837         TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
 1838                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1839                     jobref) {
 1840                         PROC_UNLOCK(p);
 1841                         td->td_retval[0] = cb->uaiocb._aiocb_private.error;
 1842                         return (0);
 1843                 }
 1844         }
 1845 
 1846         s = splnet();
 1847 
 1848         for (cb = TAILQ_FIRST(&ki->kaio_jobqueue); cb; cb = TAILQ_NEXT(cb,
 1849             plist)) {
 1850                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1851                     jobref) {
 1852                         PROC_UNLOCK(p);
 1853                         td->td_retval[0] = EINPROGRESS;
 1854                         splx(s);
 1855                         return (0);
 1856                 }
 1857         }
 1858 
 1859         for (cb = TAILQ_FIRST(&ki->kaio_sockqueue); cb; cb = TAILQ_NEXT(cb,
 1860             plist)) {
 1861                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1862                     jobref) {
 1863                         PROC_UNLOCK(p);
 1864                         td->td_retval[0] = EINPROGRESS;
 1865                         splx(s);
 1866                         return (0);
 1867                 }
 1868         }
 1869         splx(s);
 1870 
 1871         s = splbio();
 1872         for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = TAILQ_NEXT(cb,
 1873             plist)) {
 1874                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1875                     jobref) {
 1876                         PROC_UNLOCK(p);
 1877                         td->td_retval[0] = cb->uaiocb._aiocb_private.error;
 1878                         splx(s);
 1879                         return (0);
 1880                 }
 1881         }
 1882 
 1883         for (cb = TAILQ_FIRST(&ki->kaio_bufqueue); cb; cb = TAILQ_NEXT(cb,
 1884             plist)) {
 1885                 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
 1886                     jobref) {
 1887                         PROC_UNLOCK(p);
 1888                         td->td_retval[0] = EINPROGRESS;
 1889                         splx(s);
 1890                         return (0);
 1891                 }
 1892         }
 1893         splx(s);
 1894         PROC_UNLOCK(p);
 1895 
 1896 #if (0)
 1897         /*
 1898          * Hack for lio.
 1899          */
 1900         status = fuword(&uap->aiocbp->_aiocb_private.status);
 1901         if (status == -1)
 1902                 return fuword(&uap->aiocbp->_aiocb_private.error);
 1903 #endif
 1904         return (EINVAL);
 1905 }
 1906 
 1907 /* syscall - asynchronous read from a file (REALTIME) */
 1908 int
 1909 aio_read(struct thread *td, struct aio_read_args *uap)
 1910 {
 1911 
 1912         return aio_aqueue(td, uap->aiocbp, LIO_READ);
 1913 }
 1914 
 1915 /* syscall - asynchronous write to a file (REALTIME) */
 1916 int
 1917 aio_write(struct thread *td, struct aio_write_args *uap)
 1918 {
 1919 
 1920         return aio_aqueue(td, uap->aiocbp, LIO_WRITE);
 1921 }
 1922 
 1923 /* syscall - list directed I/O (REALTIME) */
 1924 int
 1925 lio_listio(struct thread *td, struct lio_listio_args *uap)
 1926 {
 1927         struct proc *p = td->td_proc;
 1928         int nent, nentqueued;
 1929         struct aiocb *iocb, * const *cbptr;
 1930         struct aiocblist *cb;
 1931         struct kaioinfo *ki;
 1932         struct aio_liojob *lj;
 1933         int error, runningcode;
 1934         int nerror;
 1935         int i;
 1936         int s;
 1937 
 1938         if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
 1939                 return (EINVAL);
 1940 
 1941         nent = uap->nent;
 1942         if (nent < 0 || nent > AIO_LISTIO_MAX)
 1943                 return (EINVAL);
 1944 
 1945         if (p->p_aioinfo == NULL)
 1946                 aio_init_aioinfo(p);
 1947 
 1948         if ((nent + num_queue_count) > max_queue_count)
 1949                 return (EAGAIN);
 1950 
 1951         ki = p->p_aioinfo;
 1952         if ((nent + ki->kaio_queue_count) > ki->kaio_qallowed_count)
 1953                 return (EAGAIN);
 1954 
 1955         lj = uma_zalloc(aiolio_zone, M_WAITOK);
 1956         if (!lj)
 1957                 return (EAGAIN);
 1958 
 1959         lj->lioj_flags = 0;
 1960         lj->lioj_buffer_count = 0;
 1961         lj->lioj_buffer_finished_count = 0;
 1962         lj->lioj_queue_count = 0;
 1963         lj->lioj_queue_finished_count = 0;
 1964 
 1965         /*
 1966          * Setup signal.
 1967          */
 1968         if (uap->sig && (uap->mode == LIO_NOWAIT)) {
 1969                 error = copyin(uap->sig, &lj->lioj_signal,
 1970                     sizeof(lj->lioj_signal));
 1971                 if (error) {
 1972                         uma_zfree(aiolio_zone, lj);
 1973                         return (error);
 1974                 }
 1975                 if (!_SIG_VALID(lj->lioj_signal.sigev_signo)) {
 1976                         uma_zfree(aiolio_zone, lj);
 1977                         return (EINVAL);
 1978                 }
 1979                 lj->lioj_flags |= LIOJ_SIGNAL;
 1980         }
 1981         TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list);
 1982         /*
 1983          * Get pointers to the list of I/O requests.
 1984          */
 1985         nerror = 0;
 1986         nentqueued = 0;
 1987         cbptr = uap->acb_list;
 1988         for (i = 0; i < uap->nent; i++) {
 1989                 iocb = (struct aiocb *)(intptr_t)fuword(&cbptr[i]);
 1990                 if (((intptr_t)iocb != -1) && ((intptr_t)iocb != 0)) {
 1991                         error = _aio_aqueue(td, iocb, lj, 0);
 1992                         if (error == 0)
 1993                                 nentqueued++;
 1994                         else
 1995                                 nerror++;
 1996                 }
 1997         }
 1998 
 1999         /*
 2000          * If we haven't queued any, then just return error.
 2001          */
 2002         if (nentqueued == 0)
 2003                 return (0);
 2004 
 2005         /*
 2006          * Calculate the appropriate error return.
 2007          */
 2008         runningcode = 0;
 2009         if (nerror)
 2010                 runningcode = EIO;
 2011 
 2012         if (uap->mode == LIO_WAIT) {
 2013                 int command, found;
 2014                 long jobref;
 2015 
 2016                 for (;;) {
 2017                         found = 0;
 2018                         for (i = 0; i < uap->nent; i++) {
 2019                                 /*
 2020                                  * Fetch address of the control buf pointer in
 2021                                  * user space.
 2022                                  */
 2023                                 iocb = (struct aiocb *)
 2024                                     (intptr_t)fuword(&cbptr[i]);
 2025                                 if (((intptr_t)iocb == -1) || ((intptr_t)iocb
 2026                                     == 0))
 2027                                         continue;
 2028 
 2029                                 /*
 2030                                  * Fetch the associated command from user space.
 2031                                  */
 2032                                 command = fuword(&iocb->aio_lio_opcode);
 2033                                 if (command == LIO_NOP) {
 2034                                         found++;
 2035                                         continue;
 2036                                 }
 2037 
 2038                                 jobref =
 2039                                     fuword(&iocb->_aiocb_private.kernelinfo);
 2040 
 2041                                 TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
 2042                                         if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo)
 2043                                             == jobref) {
 2044                                                 if (cb->uaiocb.aio_lio_opcode
 2045                                                     == LIO_WRITE) {
 2046                                                         p->p_stats->p_ru.ru_oublock
 2047                                                             +=
 2048                                                             cb->outputcharge;
 2049                                                         cb->outputcharge = 0;
 2050                                                 } else if (cb->uaiocb.aio_lio_opcode
 2051                                                     == LIO_READ) {
 2052                                                         p->p_stats->p_ru.ru_inblock
 2053                                                             += cb->inputcharge;
 2054                                                         cb->inputcharge = 0;
 2055                                                 }
 2056                                                 found++;
 2057                                                 break;
 2058                                         }
 2059                                 }
 2060 
 2061                                 s = splbio();
 2062                                 TAILQ_FOREACH(cb, &ki->kaio_bufdone, plist) {
 2063                                         if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo)
 2064                                             == jobref) {
 2065                                                 found++;
 2066                                                 break;
 2067                                         }
 2068                                 }
 2069                                 splx(s);
 2070                         }
 2071 
 2072                         /*
 2073                          * If all I/Os have been disposed of, then we can
 2074                          * return.
 2075                          */
 2076                         if (found == nentqueued)
 2077                                 return (runningcode);
 2078 
 2079                         ki->kaio_flags |= KAIO_WAKEUP;
 2080                         error = tsleep(p, PRIBIO | PCATCH, "aiospn", 0);
 2081 
 2082                         if (error == EINTR)
 2083                                 return (EINTR);
 2084                         else if (error == EWOULDBLOCK)
 2085                                 return (EAGAIN);
 2086                 }
 2087         }
 2088 
 2089         return (runningcode);
 2090 }
 2091 
 2092 /*
 2093  * Interrupt handler for physio, performs the necessary process wakeups, and
 2094  * signals.
 2095  */
 2096 static void
 2097 aio_physwakeup(struct buf *bp)
 2098 {
 2099         struct aiocblist *aiocbe;
 2100         struct proc *p;
 2101         struct kaioinfo *ki;
 2102         struct aio_liojob *lj;
 2103 
 2104         mtx_lock(&Giant);
 2105         bp->b_flags |= B_DONE;
 2106         wakeup(bp);
 2107 
 2108         aiocbe = (struct aiocblist *)bp->b_caller1;
 2109         if (aiocbe) {
 2110                 p = aiocbe->userproc;
 2111 
 2112                 aiocbe->jobstate = JOBST_JOBBFINISHED;
 2113                 aiocbe->uaiocb._aiocb_private.status -= bp->b_resid;
 2114                 aiocbe->uaiocb._aiocb_private.error = 0;
 2115                 aiocbe->jobflags |= AIOCBLIST_DONE;
 2116 
 2117                 if (bp->b_ioflags & BIO_ERROR)
 2118                         aiocbe->uaiocb._aiocb_private.error = bp->b_error;
 2119 
 2120                 lj = aiocbe->lio;
 2121                 if (lj) {
 2122                         lj->lioj_buffer_finished_count++;
 2123 
 2124                         /*
 2125                          * wakeup/signal if all of the interrupt jobs are done.
 2126                          */
 2127                         if (lj->lioj_buffer_finished_count ==
 2128                             lj->lioj_buffer_count &&
 2129                             lj->lioj_queue_finished_count ==
 2130                             lj->lioj_queue_count) {
 2131                                 /*
 2132                                  * Post a signal if it is called for.
 2133                                  */
 2134                                 if ((lj->lioj_flags &
 2135                                     (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) ==
 2136                                     LIOJ_SIGNAL) {
 2137                                         PROC_LOCK(p);
 2138                                         psignal(p, lj->lioj_signal.sigev_signo);
 2139                                         PROC_UNLOCK(p);
 2140                                         lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
 2141                                 }
 2142                         }
 2143                 }
 2144 
 2145                 ki = p->p_aioinfo;
 2146                 if (ki) {
 2147                         ki->kaio_buffer_finished_count++;
 2148                         TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist);
 2149                         TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist);
 2150 
 2151                         KNOTE_UNLOCKED(&aiocbe->klist, 0);
 2152                         /* Do the wakeup. */
 2153                         if (ki->kaio_flags & (KAIO_RUNDOWN|KAIO_WAKEUP)) {
 2154                                 ki->kaio_flags &= ~KAIO_WAKEUP;
 2155                                 wakeup(p);
 2156                         }
 2157                 }
 2158 
 2159                 if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL) {
 2160                         PROC_LOCK(p);
 2161                         psignal(p, aiocbe->uaiocb.aio_sigevent.sigev_signo);
 2162                         PROC_UNLOCK(p);
 2163                 }
 2164         }
 2165         mtx_unlock(&Giant);
 2166 }
 2167 
 2168 /* syscall - wait for the next completion of an aio request */
 2169 int
 2170 aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
 2171 {
 2172         struct proc *p = td->td_proc;
 2173         struct timeval atv;
 2174         struct timespec ts;
 2175         struct kaioinfo *ki;
 2176         struct aiocblist *cb = NULL;
 2177         int error, s, timo;
 2178 
 2179         suword(uap->aiocbp, (int)NULL);
 2180 
 2181         timo = 0;
 2182         if (uap->timeout) {
 2183                 /* Get timespec struct. */
 2184                 error = copyin(uap->timeout, &ts, sizeof(ts));
 2185                 if (error)
 2186                         return (error);
 2187 
 2188                 if ((ts.tv_nsec < 0) || (ts.tv_nsec >= 1000000000))
 2189                         return (EINVAL);
 2190 
 2191                 TIMESPEC_TO_TIMEVAL(&atv, &ts);
 2192                 if (itimerfix(&atv))
 2193                         return (EINVAL);
 2194                 timo = tvtohz(&atv);
 2195         }
 2196 
 2197         ki = p->p_aioinfo;
 2198         if (ki == NULL)
 2199                 return (EAGAIN);
 2200 
 2201         for (;;) {
 2202                 PROC_LOCK(p);
 2203                 if ((cb = TAILQ_FIRST(&ki->kaio_jobdone)) != 0) {
 2204                         PROC_UNLOCK(p);
 2205                         suword(uap->aiocbp, (uintptr_t)cb->uuaiocb);
 2206                         td->td_retval[0] = cb->uaiocb._aiocb_private.status;
 2207                         if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
 2208                                 p->p_stats->p_ru.ru_oublock +=
 2209                                     cb->outputcharge;
 2210                                 cb->outputcharge = 0;
 2211                         } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
 2212                                 p->p_stats->p_ru.ru_inblock += cb->inputcharge;
 2213                                 cb->inputcharge = 0;
 2214                         }
 2215                         error = cb->uaiocb._aiocb_private.error;
 2216                         aio_free_entry(cb);
 2217                         return (error);
 2218                 }
 2219 
 2220                 s = splbio();
 2221                 if ((cb = TAILQ_FIRST(&ki->kaio_bufdone)) != 0 ) {
 2222                         PROC_UNLOCK(p);
 2223                         splx(s);
 2224                         suword(uap->aiocbp, (uintptr_t)cb->uuaiocb);
 2225                         error = cb->uaiocb._aiocb_private.error;
 2226                         td->td_retval[0] = cb->uaiocb._aiocb_private.status;
 2227                         aio_free_entry(cb);
 2228                         return (error);
 2229                 }
 2230 
 2231                 ki->kaio_flags |= KAIO_WAKEUP;
 2232                 error = msleep(p, &p->p_mtx, PDROP | PRIBIO | PCATCH, "aiowc",
 2233                     timo);
 2234                 splx(s);
 2235 
 2236                 if (error == ERESTART)
 2237                         return (EINTR);
 2238                 else if (error < 0)
 2239                         return (error);
 2240                 else if (error == EINTR)
 2241                         return (EINTR);
 2242                 else if (error == EWOULDBLOCK)
 2243                         return (EAGAIN);
 2244         }
 2245 }
 2246 
 2247 /* kqueue attach function */
 2248 static int
 2249 filt_aioattach(struct knote *kn)
 2250 {
 2251         struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
 2252 
 2253         /*
 2254          * The aiocbe pointer must be validated before using it, so
 2255          * registration is restricted to the kernel; the user cannot
 2256          * set EV_FLAG1.
 2257          */
 2258         if ((kn->kn_flags & EV_FLAG1) == 0)
 2259                 return (EPERM);
 2260         kn->kn_flags &= ~EV_FLAG1;
 2261 
 2262         knlist_add(&aiocbe->klist, kn, 0);
 2263 
 2264         return (0);
 2265 }
 2266 
 2267 /* kqueue detach function */
 2268 static void
 2269 filt_aiodetach(struct knote *kn)
 2270 {
 2271         struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
 2272 
 2273         knlist_remove(&aiocbe->klist, kn, 0);
 2274 }
 2275 
 2276 /* kqueue filter function */
 2277 /*ARGSUSED*/
 2278 static int
 2279 filt_aio(struct knote *kn, long hint)
 2280 {
 2281         struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
 2282 
 2283         kn->kn_data = aiocbe->uaiocb._aiocb_private.error;
 2284         if (aiocbe->jobstate != JOBST_JOBFINISHED &&
 2285             aiocbe->jobstate != JOBST_JOBBFINISHED)
 2286                 return (0);
 2287         kn->kn_flags |= EV_EOF;
 2288         return (1);
 2289 }

Cache object: ffb82fadfbf4491985577e6a01373016


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.