The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/vfs_aio.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1997 John S. Dyson.  All rights reserved.
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. John S. Dyson's name may not be used to endorse or promote products
   10  *    derived from this software without specific prior written permission.
   11  *
   12  * DISCLAIMER:  This code isn't warranted to do anything useful.  Anything
   13  * bad that happens because of using this software isn't the responsibility
   14  * of the author.  This software is distributed AS-IS.
   15  */
   16 
   17 /*
   18  * This file contains support for the POSIX 1003.1B AIO/LIO facility.
   19  */
   20 
   21 #include <sys/cdefs.h>
   22 __FBSDID("$FreeBSD: releng/8.2/sys/kern/vfs_aio.c 206352 2010-04-07 14:50:58Z kib $");
   23 
   24 #include "opt_compat.h"
   25 
   26 #include <sys/param.h>
   27 #include <sys/systm.h>
   28 #include <sys/malloc.h>
   29 #include <sys/bio.h>
   30 #include <sys/buf.h>
   31 #include <sys/eventhandler.h>
   32 #include <sys/sysproto.h>
   33 #include <sys/filedesc.h>
   34 #include <sys/kernel.h>
   35 #include <sys/module.h>
   36 #include <sys/kthread.h>
   37 #include <sys/fcntl.h>
   38 #include <sys/file.h>
   39 #include <sys/limits.h>
   40 #include <sys/lock.h>
   41 #include <sys/mutex.h>
   42 #include <sys/unistd.h>
   43 #include <sys/posix4.h>
   44 #include <sys/proc.h>
   45 #include <sys/resourcevar.h>
   46 #include <sys/signalvar.h>
   47 #include <sys/protosw.h>
   48 #include <sys/sema.h>
   49 #include <sys/socket.h>
   50 #include <sys/socketvar.h>
   51 #include <sys/syscall.h>
   52 #include <sys/sysent.h>
   53 #include <sys/sysctl.h>
   54 #include <sys/sx.h>
   55 #include <sys/taskqueue.h>
   56 #include <sys/vnode.h>
   57 #include <sys/conf.h>
   58 #include <sys/event.h>
   59 #include <sys/mount.h>
   60 
   61 #include <machine/atomic.h>
   62 
   63 #include <vm/vm.h>
   64 #include <vm/vm_extern.h>
   65 #include <vm/pmap.h>
   66 #include <vm/vm_map.h>
   67 #include <vm/vm_object.h>
   68 #include <vm/uma.h>
   69 #include <sys/aio.h>
   70 
   71 #include "opt_vfs_aio.h"
   72 
   73 /*
   74  * Counter for allocating reference ids to new jobs.  Wrapped to 1 on
   75  * overflow. (XXX will be removed soon.)
   76  */
   77 static u_long jobrefid;
   78 
   79 /*
   80  * Counter for aio_fsync.
   81  */
   82 static uint64_t jobseqno;
   83 
   84 #define JOBST_NULL              0
   85 #define JOBST_JOBQSOCK          1
   86 #define JOBST_JOBQGLOBAL        2
   87 #define JOBST_JOBRUNNING        3
   88 #define JOBST_JOBFINISHED       4
   89 #define JOBST_JOBQBUF           5
   90 #define JOBST_JOBQSYNC          6
   91 
   92 #ifndef MAX_AIO_PER_PROC
   93 #define MAX_AIO_PER_PROC        32
   94 #endif
   95 
   96 #ifndef MAX_AIO_QUEUE_PER_PROC
   97 #define MAX_AIO_QUEUE_PER_PROC  256 /* Bigger than AIO_LISTIO_MAX */
   98 #endif
   99 
  100 #ifndef MAX_AIO_PROCS
  101 #define MAX_AIO_PROCS           32
  102 #endif
  103 
  104 #ifndef MAX_AIO_QUEUE
  105 #define MAX_AIO_QUEUE           1024 /* Bigger than AIO_LISTIO_MAX */
  106 #endif
  107 
  108 #ifndef TARGET_AIO_PROCS
  109 #define TARGET_AIO_PROCS        4
  110 #endif
  111 
  112 #ifndef MAX_BUF_AIO
  113 #define MAX_BUF_AIO             16
  114 #endif
  115 
  116 #ifndef AIOD_TIMEOUT_DEFAULT
  117 #define AIOD_TIMEOUT_DEFAULT    (10 * hz)
  118 #endif
  119 
  120 #ifndef AIOD_LIFETIME_DEFAULT
  121 #define AIOD_LIFETIME_DEFAULT   (30 * hz)
  122 #endif
  123 
  124 FEATURE(aio, "Asynchronous I/O");
  125 
  126 static MALLOC_DEFINE(M_LIO, "lio", "listio aio control block list");
  127 
  128 static SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, "Async IO management");
  129 
  130 static int max_aio_procs = MAX_AIO_PROCS;
  131 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_procs,
  132         CTLFLAG_RW, &max_aio_procs, 0,
  133         "Maximum number of kernel threads to use for handling async IO ");
  134 
  135 static int num_aio_procs = 0;
  136 SYSCTL_INT(_vfs_aio, OID_AUTO, num_aio_procs,
  137         CTLFLAG_RD, &num_aio_procs, 0,
  138         "Number of presently active kernel threads for async IO");
  139 
  140 /*
  141  * The code will adjust the actual number of AIO processes towards this
  142  * number when it gets a chance.
  143  */
  144 static int target_aio_procs = TARGET_AIO_PROCS;
  145 SYSCTL_INT(_vfs_aio, OID_AUTO, target_aio_procs, CTLFLAG_RW, &target_aio_procs,
  146         0, "Preferred number of ready kernel threads for async IO");
  147 
  148 static int max_queue_count = MAX_AIO_QUEUE;
  149 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue, CTLFLAG_RW, &max_queue_count, 0,
  150     "Maximum number of aio requests to queue, globally");
  151 
  152 static int num_queue_count = 0;
  153 SYSCTL_INT(_vfs_aio, OID_AUTO, num_queue_count, CTLFLAG_RD, &num_queue_count, 0,
  154     "Number of queued aio requests");
  155 
  156 static int num_buf_aio = 0;
  157 SYSCTL_INT(_vfs_aio, OID_AUTO, num_buf_aio, CTLFLAG_RD, &num_buf_aio, 0,
  158     "Number of aio requests presently handled by the buf subsystem");
  159 
  160 /* Number of async I/O thread in the process of being started */
  161 /* XXX This should be local to aio_aqueue() */
  162 static int num_aio_resv_start = 0;
  163 
  164 static int aiod_timeout;
  165 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_timeout, CTLFLAG_RW, &aiod_timeout, 0,
  166     "Timeout value for synchronous aio operations");
  167 
  168 static int aiod_lifetime;
  169 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime, CTLFLAG_RW, &aiod_lifetime, 0,
  170     "Maximum lifetime for idle aiod");
  171 
  172 static int unloadable = 0;
  173 SYSCTL_INT(_vfs_aio, OID_AUTO, unloadable, CTLFLAG_RW, &unloadable, 0,
  174     "Allow unload of aio (not recommended)");
  175 
  176 
  177 static int max_aio_per_proc = MAX_AIO_PER_PROC;
  178 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_per_proc, CTLFLAG_RW, &max_aio_per_proc,
  179     0, "Maximum active aio requests per process (stored in the process)");
  180 
  181 static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC;
  182 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW,
  183     &max_aio_queue_per_proc, 0,
  184     "Maximum queued aio requests per process (stored in the process)");
  185 
  186 static int max_buf_aio = MAX_BUF_AIO;
  187 SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW, &max_buf_aio, 0,
  188     "Maximum buf aio requests per process (stored in the process)");
  189 
  190 typedef struct oaiocb {
  191         int     aio_fildes;             /* File descriptor */
  192         off_t   aio_offset;             /* File offset for I/O */
  193         volatile void *aio_buf;         /* I/O buffer in process space */
  194         size_t  aio_nbytes;             /* Number of bytes for I/O */
  195         struct  osigevent aio_sigevent; /* Signal to deliver */
  196         int     aio_lio_opcode;         /* LIO opcode */
  197         int     aio_reqprio;            /* Request priority -- ignored */
  198         struct  __aiocb_private _aiocb_private;
  199 } oaiocb_t;
  200 
  201 /*
  202  * Below is a key of locks used to protect each member of struct aiocblist
  203  * aioliojob and kaioinfo and any backends.
  204  *
  205  * * - need not protected
  206  * a - locked by kaioinfo lock
  207  * b - locked by backend lock, the backend lock can be null in some cases,
  208  *     for example, BIO belongs to this type, in this case, proc lock is
  209  *     reused.
  210  * c - locked by aio_job_mtx, the lock for the generic file I/O backend.
  211  */
  212 
  213 /*
  214  * Current, there is only two backends: BIO and generic file I/O.
  215  * socket I/O is served by generic file I/O, this is not a good idea, since
  216  * disk file I/O and any other types without O_NONBLOCK flag can block daemon
  217  * threads, if there is no thread to serve socket I/O, the socket I/O will be
  218  * delayed too long or starved, we should create some threads dedicated to
  219  * sockets to do non-blocking I/O, same for pipe and fifo, for these I/O
  220  * systems we really need non-blocking interface, fiddling O_NONBLOCK in file
  221  * structure is not safe because there is race between userland and aio
  222  * daemons.
  223  */
  224 
  225 struct aiocblist {
  226         TAILQ_ENTRY(aiocblist) list;    /* (b) internal list of for backend */
  227         TAILQ_ENTRY(aiocblist) plist;   /* (a) list of jobs for each backend */
  228         TAILQ_ENTRY(aiocblist) allist;  /* (a) list of all jobs in proc */
  229         int     jobflags;               /* (a) job flags */
  230         int     jobstate;               /* (b) job state */
  231         int     inputcharge;            /* (*) input blockes */
  232         int     outputcharge;           /* (*) output blockes */
  233         struct  buf *bp;                /* (*) private to BIO backend,
  234                                          * buffer pointer
  235                                          */
  236         struct  proc *userproc;         /* (*) user process */
  237         struct  ucred *cred;            /* (*) active credential when created */
  238         struct  file *fd_file;          /* (*) pointer to file structure */
  239         struct  aioliojob *lio;         /* (*) optional lio job */
  240         struct  aiocb *uuaiocb;         /* (*) pointer in userspace of aiocb */
  241         struct  knlist klist;           /* (a) list of knotes */
  242         struct  aiocb uaiocb;           /* (*) kernel I/O control block */
  243         ksiginfo_t ksi;                 /* (a) realtime signal info */
  244         struct  task biotask;           /* (*) private to BIO backend */
  245         uint64_t seqno;                 /* (*) job number */
  246         int     pending;                /* (a) number of pending I/O, aio_fsync only */
  247 };
  248 
  249 /* jobflags */
  250 #define AIOCBLIST_DONE          0x01
  251 #define AIOCBLIST_BUFDONE       0x02
  252 #define AIOCBLIST_RUNDOWN       0x04
  253 #define AIOCBLIST_CHECKSYNC     0x08
  254 
  255 /*
  256  * AIO process info
  257  */
  258 #define AIOP_FREE       0x1                     /* proc on free queue */
  259 
  260 struct aiothreadlist {
  261         int aiothreadflags;                     /* (c) AIO proc flags */
  262         TAILQ_ENTRY(aiothreadlist) list;        /* (c) list of processes */
  263         struct thread *aiothread;               /* (*) the AIO thread */
  264 };
  265 
  266 /*
  267  * data-structure for lio signal management
  268  */
  269 struct aioliojob {
  270         int     lioj_flags;                     /* (a) listio flags */
  271         int     lioj_count;                     /* (a) listio flags */
  272         int     lioj_finished_count;            /* (a) listio flags */
  273         struct  sigevent lioj_signal;           /* (a) signal on all I/O done */
  274         TAILQ_ENTRY(aioliojob) lioj_list;       /* (a) lio list */
  275         struct  knlist klist;                   /* (a) list of knotes */
  276         ksiginfo_t lioj_ksi;                    /* (a) Realtime signal info */
  277 };
  278 
  279 #define LIOJ_SIGNAL             0x1     /* signal on all done (lio) */
  280 #define LIOJ_SIGNAL_POSTED      0x2     /* signal has been posted */
  281 #define LIOJ_KEVENT_POSTED      0x4     /* kevent triggered */
  282 
  283 /*
  284  * per process aio data structure
  285  */
  286 struct kaioinfo {
  287         struct mtx      kaio_mtx;       /* the lock to protect this struct */
  288         int     kaio_flags;             /* (a) per process kaio flags */
  289         int     kaio_maxactive_count;   /* (*) maximum number of AIOs */
  290         int     kaio_active_count;      /* (c) number of currently used AIOs */
  291         int     kaio_qallowed_count;    /* (*) maxiumu size of AIO queue */
  292         int     kaio_count;             /* (a) size of AIO queue */
  293         int     kaio_ballowed_count;    /* (*) maximum number of buffers */
  294         int     kaio_buffer_count;      /* (a) number of physio buffers */
  295         TAILQ_HEAD(,aiocblist) kaio_all;        /* (a) all AIOs in the process */
  296         TAILQ_HEAD(,aiocblist) kaio_done;       /* (a) done queue for process */
  297         TAILQ_HEAD(,aioliojob) kaio_liojoblist; /* (a) list of lio jobs */
  298         TAILQ_HEAD(,aiocblist) kaio_jobqueue;   /* (a) job queue for process */
  299         TAILQ_HEAD(,aiocblist) kaio_bufqueue;   /* (a) buffer job queue for process */
  300         TAILQ_HEAD(,aiocblist) kaio_sockqueue;  /* (a) queue for aios waiting on sockets,
  301                                                  *  NOT USED YET.
  302                                                  */
  303         TAILQ_HEAD(,aiocblist) kaio_syncqueue;  /* (a) queue for aio_fsync */
  304         struct  task    kaio_task;      /* (*) task to kick aio threads */
  305 };
  306 
  307 #define AIO_LOCK(ki)            mtx_lock(&(ki)->kaio_mtx)
  308 #define AIO_UNLOCK(ki)          mtx_unlock(&(ki)->kaio_mtx)
  309 #define AIO_LOCK_ASSERT(ki, f)  mtx_assert(&(ki)->kaio_mtx, (f))
  310 #define AIO_MTX(ki)             (&(ki)->kaio_mtx)
  311 
  312 #define KAIO_RUNDOWN    0x1     /* process is being run down */
  313 #define KAIO_WAKEUP     0x2     /* wakeup process when there is a significant event */
  314 
  315 /*
  316  * Operations used to interact with userland aio control blocks.
  317  * Different ABIs provide their own operations.
  318  */
  319 struct aiocb_ops {
  320         int     (*copyin)(struct aiocb *ujob, struct aiocb *kjob);
  321         long    (*fetch_status)(struct aiocb *ujob);
  322         long    (*fetch_error)(struct aiocb *ujob);
  323         int     (*store_status)(struct aiocb *ujob, long status);
  324         int     (*store_error)(struct aiocb *ujob, long error);
  325         int     (*store_kernelinfo)(struct aiocb *ujob, long jobref);
  326         int     (*store_aiocb)(struct aiocb **ujobp, struct aiocb *ujob);
  327 };
  328 
  329 static TAILQ_HEAD(,aiothreadlist) aio_freeproc;         /* (c) Idle daemons */
  330 static struct sema aio_newproc_sem;
  331 static struct mtx aio_job_mtx;
  332 static struct mtx aio_sock_mtx;
  333 static TAILQ_HEAD(,aiocblist) aio_jobs;                 /* (c) Async job list */
  334 static struct unrhdr *aiod_unr;
  335 
  336 void            aio_init_aioinfo(struct proc *p);
  337 static int      aio_onceonly(void);
  338 static int      aio_free_entry(struct aiocblist *aiocbe);
  339 static void     aio_process(struct aiocblist *aiocbe);
  340 static int      aio_newproc(int *);
  341 int             aio_aqueue(struct thread *td, struct aiocb *job,
  342                         struct aioliojob *lio, int type, struct aiocb_ops *ops);
  343 static void     aio_physwakeup(struct buf *bp);
  344 static void     aio_proc_rundown(void *arg, struct proc *p);
  345 static void     aio_proc_rundown_exec(void *arg, struct proc *p, struct image_params *imgp);
  346 static int      aio_qphysio(struct proc *p, struct aiocblist *iocb);
  347 static void     biohelper(void *, int);
  348 static void     aio_daemon(void *param);
  349 static void     aio_swake_cb(struct socket *, struct sockbuf *);
  350 static int      aio_unload(void);
  351 static void     aio_bio_done_notify(struct proc *userp, struct aiocblist *aiocbe, int type);
  352 #define DONE_BUF        1
  353 #define DONE_QUEUE      2
  354 static int      aio_kick(struct proc *userp);
  355 static void     aio_kick_nowait(struct proc *userp);
  356 static void     aio_kick_helper(void *context, int pending);
  357 static int      filt_aioattach(struct knote *kn);
  358 static void     filt_aiodetach(struct knote *kn);
  359 static int      filt_aio(struct knote *kn, long hint);
  360 static int      filt_lioattach(struct knote *kn);
  361 static void     filt_liodetach(struct knote *kn);
  362 static int      filt_lio(struct knote *kn, long hint);
  363 
  364 /*
  365  * Zones for:
  366  *      kaio    Per process async io info
  367  *      aiop    async io thread data
  368  *      aiocb   async io jobs
  369  *      aiol    list io job pointer - internal to aio_suspend XXX
  370  *      aiolio  list io jobs
  371  */
  372 static uma_zone_t kaio_zone, aiop_zone, aiocb_zone, aiol_zone, aiolio_zone;
  373 
  374 /* kqueue filters for aio */
  375 static struct filterops aio_filtops =
  376         { 0, filt_aioattach, filt_aiodetach, filt_aio };
  377 static struct filterops lio_filtops =
  378         { 0, filt_lioattach, filt_liodetach, filt_lio };
  379 
  380 static eventhandler_tag exit_tag, exec_tag;
  381 
  382 TASKQUEUE_DEFINE_THREAD(aiod_bio);
  383 
  384 /*
  385  * Main operations function for use as a kernel module.
  386  */
  387 static int
  388 aio_modload(struct module *module, int cmd, void *arg)
  389 {
  390         int error = 0;
  391 
  392         switch (cmd) {
  393         case MOD_LOAD:
  394                 aio_onceonly();
  395                 break;
  396         case MOD_UNLOAD:
  397                 error = aio_unload();
  398                 break;
  399         case MOD_SHUTDOWN:
  400                 break;
  401         default:
  402                 error = EINVAL;
  403                 break;
  404         }
  405         return (error);
  406 }
  407 
  408 static moduledata_t aio_mod = {
  409         "aio",
  410         &aio_modload,
  411         NULL
  412 };
  413 
  414 static struct syscall_helper_data aio_syscalls[] = {
  415         SYSCALL_INIT_HELPER(aio_cancel),
  416         SYSCALL_INIT_HELPER(aio_error),
  417         SYSCALL_INIT_HELPER(aio_fsync),
  418         SYSCALL_INIT_HELPER(aio_read),
  419         SYSCALL_INIT_HELPER(aio_return),
  420         SYSCALL_INIT_HELPER(aio_suspend),
  421         SYSCALL_INIT_HELPER(aio_waitcomplete),
  422         SYSCALL_INIT_HELPER(aio_write),
  423         SYSCALL_INIT_HELPER(lio_listio),
  424         SYSCALL_INIT_HELPER(oaio_read),
  425         SYSCALL_INIT_HELPER(oaio_write),
  426         SYSCALL_INIT_HELPER(olio_listio),
  427         SYSCALL_INIT_LAST
  428 };
  429 
  430 #ifdef COMPAT_FREEBSD32
  431 #include <sys/mount.h>
  432 #include <sys/socket.h>
  433 #include <compat/freebsd32/freebsd32.h>
  434 #include <compat/freebsd32/freebsd32_proto.h>
  435 #include <compat/freebsd32/freebsd32_signal.h>
  436 #include <compat/freebsd32/freebsd32_syscall.h>
  437 #include <compat/freebsd32/freebsd32_util.h>
  438 
  439 static struct syscall_helper_data aio32_syscalls[] = {
  440         SYSCALL32_INIT_HELPER(freebsd32_aio_return),
  441         SYSCALL32_INIT_HELPER(freebsd32_aio_suspend),
  442         SYSCALL32_INIT_HELPER(freebsd32_aio_cancel),
  443         SYSCALL32_INIT_HELPER(freebsd32_aio_error),
  444         SYSCALL32_INIT_HELPER(freebsd32_aio_fsync),
  445         SYSCALL32_INIT_HELPER(freebsd32_aio_read),
  446         SYSCALL32_INIT_HELPER(freebsd32_aio_write),
  447         SYSCALL32_INIT_HELPER(freebsd32_aio_waitcomplete),
  448         SYSCALL32_INIT_HELPER(freebsd32_lio_listio),
  449         SYSCALL32_INIT_HELPER(freebsd32_oaio_read),
  450         SYSCALL32_INIT_HELPER(freebsd32_oaio_write),
  451         SYSCALL32_INIT_HELPER(freebsd32_olio_listio),
  452         SYSCALL_INIT_LAST
  453 };
  454 #endif
  455 
  456 DECLARE_MODULE(aio, aio_mod,
  457         SI_SUB_VFS, SI_ORDER_ANY);
  458 MODULE_VERSION(aio, 1);
  459 
  460 /*
  461  * Startup initialization
  462  */
  463 static int
  464 aio_onceonly(void)
  465 {
  466         int error;
  467 
  468         /* XXX: should probably just use so->callback */
  469         aio_swake = &aio_swake_cb;
  470         exit_tag = EVENTHANDLER_REGISTER(process_exit, aio_proc_rundown, NULL,
  471             EVENTHANDLER_PRI_ANY);
  472         exec_tag = EVENTHANDLER_REGISTER(process_exec, aio_proc_rundown_exec, NULL,
  473             EVENTHANDLER_PRI_ANY);
  474         kqueue_add_filteropts(EVFILT_AIO, &aio_filtops);
  475         kqueue_add_filteropts(EVFILT_LIO, &lio_filtops);
  476         TAILQ_INIT(&aio_freeproc);
  477         sema_init(&aio_newproc_sem, 0, "aio_new_proc");
  478         mtx_init(&aio_job_mtx, "aio_job", NULL, MTX_DEF);
  479         mtx_init(&aio_sock_mtx, "aio_sock", NULL, MTX_DEF);
  480         TAILQ_INIT(&aio_jobs);
  481         aiod_unr = new_unrhdr(1, INT_MAX, NULL);
  482         kaio_zone = uma_zcreate("AIO", sizeof(struct kaioinfo), NULL, NULL,
  483             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  484         aiop_zone = uma_zcreate("AIOP", sizeof(struct aiothreadlist), NULL,
  485             NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  486         aiocb_zone = uma_zcreate("AIOCB", sizeof(struct aiocblist), NULL, NULL,
  487             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  488         aiol_zone = uma_zcreate("AIOL", AIO_LISTIO_MAX*sizeof(intptr_t) , NULL,
  489             NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  490         aiolio_zone = uma_zcreate("AIOLIO", sizeof(struct aioliojob), NULL,
  491             NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  492         aiod_timeout = AIOD_TIMEOUT_DEFAULT;
  493         aiod_lifetime = AIOD_LIFETIME_DEFAULT;
  494         jobrefid = 1;
  495         async_io_version = _POSIX_VERSION;
  496         p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, AIO_LISTIO_MAX);
  497         p31b_setcfg(CTL_P1003_1B_AIO_MAX, MAX_AIO_QUEUE);
  498         p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, 0);
  499 
  500         error = syscall_helper_register(aio_syscalls);
  501         if (error)
  502                 return (error);
  503 #ifdef COMPAT_FREEBSD32
  504         error = syscall32_helper_register(aio32_syscalls);
  505         if (error)
  506                 return (error);
  507 #endif
  508         return (0);
  509 }
  510 
  511 /*
  512  * Callback for unload of AIO when used as a module.
  513  */
  514 static int
  515 aio_unload(void)
  516 {
  517         int error;
  518 
  519         /*
  520          * XXX: no unloads by default, it's too dangerous.
  521          * perhaps we could do it if locked out callers and then
  522          * did an aio_proc_rundown() on each process.
  523          *
  524          * jhb: aio_proc_rundown() needs to run on curproc though,
  525          * so I don't think that would fly.
  526          */
  527         if (!unloadable)
  528                 return (EOPNOTSUPP);
  529 
  530 #ifdef COMPAT_FREEBSD32
  531         syscall32_helper_unregister(aio32_syscalls);
  532 #endif
  533         syscall_helper_unregister(aio_syscalls);
  534 
  535         error = kqueue_del_filteropts(EVFILT_AIO);
  536         if (error)
  537                 return error;
  538         error = kqueue_del_filteropts(EVFILT_LIO);
  539         if (error)
  540                 return error;
  541         async_io_version = 0;
  542         aio_swake = NULL;
  543         taskqueue_free(taskqueue_aiod_bio);
  544         delete_unrhdr(aiod_unr);
  545         uma_zdestroy(kaio_zone);
  546         uma_zdestroy(aiop_zone);
  547         uma_zdestroy(aiocb_zone);
  548         uma_zdestroy(aiol_zone);
  549         uma_zdestroy(aiolio_zone);
  550         EVENTHANDLER_DEREGISTER(process_exit, exit_tag);
  551         EVENTHANDLER_DEREGISTER(process_exec, exec_tag);
  552         mtx_destroy(&aio_job_mtx);
  553         mtx_destroy(&aio_sock_mtx);
  554         sema_destroy(&aio_newproc_sem);
  555         p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, -1);
  556         p31b_setcfg(CTL_P1003_1B_AIO_MAX, -1);
  557         p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, -1);
  558         return (0);
  559 }
  560 
  561 /*
  562  * Init the per-process aioinfo structure.  The aioinfo limits are set
  563  * per-process for user limit (resource) management.
  564  */
  565 void
  566 aio_init_aioinfo(struct proc *p)
  567 {
  568         struct kaioinfo *ki;
  569 
  570         ki = uma_zalloc(kaio_zone, M_WAITOK);
  571         mtx_init(&ki->kaio_mtx, "aiomtx", NULL, MTX_DEF);
  572         ki->kaio_flags = 0;
  573         ki->kaio_maxactive_count = max_aio_per_proc;
  574         ki->kaio_active_count = 0;
  575         ki->kaio_qallowed_count = max_aio_queue_per_proc;
  576         ki->kaio_count = 0;
  577         ki->kaio_ballowed_count = max_buf_aio;
  578         ki->kaio_buffer_count = 0;
  579         TAILQ_INIT(&ki->kaio_all);
  580         TAILQ_INIT(&ki->kaio_done);
  581         TAILQ_INIT(&ki->kaio_jobqueue);
  582         TAILQ_INIT(&ki->kaio_bufqueue);
  583         TAILQ_INIT(&ki->kaio_liojoblist);
  584         TAILQ_INIT(&ki->kaio_sockqueue);
  585         TAILQ_INIT(&ki->kaio_syncqueue);
  586         TASK_INIT(&ki->kaio_task, 0, aio_kick_helper, p);
  587         PROC_LOCK(p);
  588         if (p->p_aioinfo == NULL) {
  589                 p->p_aioinfo = ki;
  590                 PROC_UNLOCK(p);
  591         } else {
  592                 PROC_UNLOCK(p);
  593                 mtx_destroy(&ki->kaio_mtx);
  594                 uma_zfree(kaio_zone, ki);
  595         }
  596 
  597         while (num_aio_procs < MIN(target_aio_procs, max_aio_procs))
  598                 aio_newproc(NULL);
  599 }
  600 
  601 static int
  602 aio_sendsig(struct proc *p, struct sigevent *sigev, ksiginfo_t *ksi)
  603 {
  604         int ret = 0;
  605 
  606         PROC_LOCK(p);
  607         if (!KSI_ONQ(ksi)) {
  608                 ksi->ksi_code = SI_ASYNCIO;
  609                 ksi->ksi_flags |= KSI_EXT | KSI_INS;
  610                 ret = psignal_event(p, sigev, ksi);
  611         }
  612         PROC_UNLOCK(p);
  613         return (ret);
  614 }
  615 
  616 /*
  617  * Free a job entry.  Wait for completion if it is currently active, but don't
  618  * delay forever.  If we delay, we return a flag that says that we have to
  619  * restart the queue scan.
  620  */
  621 static int
  622 aio_free_entry(struct aiocblist *aiocbe)
  623 {
  624         struct kaioinfo *ki;
  625         struct aioliojob *lj;
  626         struct proc *p;
  627 
  628         p = aiocbe->userproc;
  629         MPASS(curproc == p);
  630         ki = p->p_aioinfo;
  631         MPASS(ki != NULL);
  632 
  633         AIO_LOCK_ASSERT(ki, MA_OWNED);
  634         MPASS(aiocbe->jobstate == JOBST_JOBFINISHED);
  635 
  636         atomic_subtract_int(&num_queue_count, 1);
  637 
  638         ki->kaio_count--;
  639         MPASS(ki->kaio_count >= 0);
  640 
  641         TAILQ_REMOVE(&ki->kaio_done, aiocbe, plist);
  642         TAILQ_REMOVE(&ki->kaio_all, aiocbe, allist);
  643 
  644         lj = aiocbe->lio;
  645         if (lj) {
  646                 lj->lioj_count--;
  647                 lj->lioj_finished_count--;
  648 
  649                 if (lj->lioj_count == 0) {
  650                         TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
  651                         /* lio is going away, we need to destroy any knotes */
  652                         knlist_delete(&lj->klist, curthread, 1);
  653                         PROC_LOCK(p);
  654                         sigqueue_take(&lj->lioj_ksi);
  655                         PROC_UNLOCK(p);
  656                         uma_zfree(aiolio_zone, lj);
  657                 }
  658         }
  659 
  660         /* aiocbe is going away, we need to destroy any knotes */
  661         knlist_delete(&aiocbe->klist, curthread, 1);
  662         PROC_LOCK(p);
  663         sigqueue_take(&aiocbe->ksi);
  664         PROC_UNLOCK(p);
  665 
  666         MPASS(aiocbe->bp == NULL);
  667         aiocbe->jobstate = JOBST_NULL;
  668         AIO_UNLOCK(ki);
  669 
  670         /*
  671          * The thread argument here is used to find the owning process
  672          * and is also passed to fo_close() which may pass it to various
  673          * places such as devsw close() routines.  Because of that, we
  674          * need a thread pointer from the process owning the job that is
  675          * persistent and won't disappear out from under us or move to
  676          * another process.
  677          *
  678          * Currently, all the callers of this function call it to remove
  679          * an aiocblist from the current process' job list either via a
  680          * syscall or due to the current process calling exit() or
  681          * execve().  Thus, we know that p == curproc.  We also know that
  682          * curthread can't exit since we are curthread.
  683          *
  684          * Therefore, we use curthread as the thread to pass to
  685          * knlist_delete().  This does mean that it is possible for the
  686          * thread pointer at close time to differ from the thread pointer
  687          * at open time, but this is already true of file descriptors in
  688          * a multithreaded process.
  689          */
  690         fdrop(aiocbe->fd_file, curthread);
  691         crfree(aiocbe->cred);
  692         uma_zfree(aiocb_zone, aiocbe);
  693         AIO_LOCK(ki);
  694 
  695         return (0);
  696 }
  697 
  698 static void
  699 aio_proc_rundown_exec(void *arg, struct proc *p, struct image_params *imgp __unused)
  700 {
  701         aio_proc_rundown(arg, p);
  702 }
  703 
  704 /*
  705  * Rundown the jobs for a given process.
  706  */
  707 static void
  708 aio_proc_rundown(void *arg, struct proc *p)
  709 {
  710         struct kaioinfo *ki;
  711         struct aioliojob *lj;
  712         struct aiocblist *cbe, *cbn;
  713         struct file *fp;
  714         struct socket *so;
  715         int remove;
  716 
  717         KASSERT(curthread->td_proc == p,
  718             ("%s: called on non-curproc", __func__));
  719         ki = p->p_aioinfo;
  720         if (ki == NULL)
  721                 return;
  722 
  723         AIO_LOCK(ki);
  724         ki->kaio_flags |= KAIO_RUNDOWN;
  725 
  726 restart:
  727 
  728         /*
  729          * Try to cancel all pending requests. This code simulates
  730          * aio_cancel on all pending I/O requests.
  731          */
  732         TAILQ_FOREACH_SAFE(cbe, &ki->kaio_jobqueue, plist, cbn) {
  733                 remove = 0;
  734                 mtx_lock(&aio_job_mtx);
  735                 if (cbe->jobstate == JOBST_JOBQGLOBAL) {
  736                         TAILQ_REMOVE(&aio_jobs, cbe, list);
  737                         remove = 1;
  738                 } else if (cbe->jobstate == JOBST_JOBQSOCK) {
  739                         fp = cbe->fd_file;
  740                         MPASS(fp->f_type == DTYPE_SOCKET);
  741                         so = fp->f_data;
  742                         TAILQ_REMOVE(&so->so_aiojobq, cbe, list);
  743                         remove = 1;
  744                 } else if (cbe->jobstate == JOBST_JOBQSYNC) {
  745                         TAILQ_REMOVE(&ki->kaio_syncqueue, cbe, list);
  746                         remove = 1;
  747                 }
  748                 mtx_unlock(&aio_job_mtx);
  749 
  750                 if (remove) {
  751                         cbe->jobstate = JOBST_JOBFINISHED;
  752                         cbe->uaiocb._aiocb_private.status = -1;
  753                         cbe->uaiocb._aiocb_private.error = ECANCELED;
  754                         TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist);
  755                         aio_bio_done_notify(p, cbe, DONE_QUEUE);
  756                 }
  757         }
  758 
  759         /* Wait for all running I/O to be finished */
  760         if (TAILQ_FIRST(&ki->kaio_bufqueue) ||
  761             TAILQ_FIRST(&ki->kaio_jobqueue)) {
  762                 ki->kaio_flags |= KAIO_WAKEUP;
  763                 msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO, "aioprn", hz);
  764                 goto restart;
  765         }
  766 
  767         /* Free all completed I/O requests. */
  768         while ((cbe = TAILQ_FIRST(&ki->kaio_done)) != NULL)
  769                 aio_free_entry(cbe);
  770 
  771         while ((lj = TAILQ_FIRST(&ki->kaio_liojoblist)) != NULL) {
  772                 if (lj->lioj_count == 0) {
  773                         TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
  774                         knlist_delete(&lj->klist, curthread, 1);
  775                         PROC_LOCK(p);
  776                         sigqueue_take(&lj->lioj_ksi);
  777                         PROC_UNLOCK(p);
  778                         uma_zfree(aiolio_zone, lj);
  779                 } else {
  780                         panic("LIO job not cleaned up: C:%d, FC:%d\n",
  781                             lj->lioj_count, lj->lioj_finished_count);
  782                 }
  783         }
  784         AIO_UNLOCK(ki);
  785         taskqueue_drain(taskqueue_aiod_bio, &ki->kaio_task);
  786         mtx_destroy(&ki->kaio_mtx);
  787         uma_zfree(kaio_zone, ki);
  788         p->p_aioinfo = NULL;
  789 }
  790 
  791 /*
  792  * Select a job to run (called by an AIO daemon).
  793  */
  794 static struct aiocblist *
  795 aio_selectjob(struct aiothreadlist *aiop)
  796 {
  797         struct aiocblist *aiocbe;
  798         struct kaioinfo *ki;
  799         struct proc *userp;
  800 
  801         mtx_assert(&aio_job_mtx, MA_OWNED);
  802         TAILQ_FOREACH(aiocbe, &aio_jobs, list) {
  803                 userp = aiocbe->userproc;
  804                 ki = userp->p_aioinfo;
  805 
  806                 if (ki->kaio_active_count < ki->kaio_maxactive_count) {
  807                         TAILQ_REMOVE(&aio_jobs, aiocbe, list);
  808                         /* Account for currently active jobs. */
  809                         ki->kaio_active_count++;
  810                         aiocbe->jobstate = JOBST_JOBRUNNING;
  811                         break;
  812                 }
  813         }
  814         return (aiocbe);
  815 }
  816 
  817 /*
  818  *  Move all data to a permanent storage device, this code
  819  *  simulates fsync syscall.
  820  */
  821 static int
  822 aio_fsync_vnode(struct thread *td, struct vnode *vp)
  823 {
  824         struct mount *mp;
  825         int vfslocked;
  826         int error;
  827 
  828         vfslocked = VFS_LOCK_GIANT(vp->v_mount);
  829         if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
  830                 goto drop;
  831         vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
  832         if (vp->v_object != NULL) {
  833                 VM_OBJECT_LOCK(vp->v_object);
  834                 vm_object_page_clean(vp->v_object, 0, 0, 0);
  835                 VM_OBJECT_UNLOCK(vp->v_object);
  836         }
  837         error = VOP_FSYNC(vp, MNT_WAIT, td);
  838 
  839         VOP_UNLOCK(vp, 0);
  840         vn_finished_write(mp);
  841 drop:
  842         VFS_UNLOCK_GIANT(vfslocked);
  843         return (error);
  844 }
  845 
  846 /*
  847  * The AIO processing activity.  This is the code that does the I/O request for
  848  * the non-physio version of the operations.  The normal vn operations are used,
  849  * and this code should work in all instances for every type of file, including
  850  * pipes, sockets, fifos, and regular files.
  851  *
  852  * XXX I don't think it works well for socket, pipe, and fifo.
  853  */
  854 static void
  855 aio_process(struct aiocblist *aiocbe)
  856 {
  857         struct ucred *td_savedcred;
  858         struct thread *td;
  859         struct aiocb *cb;
  860         struct file *fp;
  861         struct socket *so;
  862         struct uio auio;
  863         struct iovec aiov;
  864         int cnt;
  865         int error;
  866         int oublock_st, oublock_end;
  867         int inblock_st, inblock_end;
  868 
  869         td = curthread;
  870         td_savedcred = td->td_ucred;
  871         td->td_ucred = aiocbe->cred;
  872         cb = &aiocbe->uaiocb;
  873         fp = aiocbe->fd_file;
  874 
  875         if (cb->aio_lio_opcode == LIO_SYNC) {
  876                 error = 0;
  877                 cnt = 0;
  878                 if (fp->f_vnode != NULL)
  879                         error = aio_fsync_vnode(td, fp->f_vnode);
  880                 cb->_aiocb_private.error = error;
  881                 cb->_aiocb_private.status = 0;
  882                 td->td_ucred = td_savedcred;
  883                 return;
  884         }
  885 
  886         aiov.iov_base = (void *)(uintptr_t)cb->aio_buf;
  887         aiov.iov_len = cb->aio_nbytes;
  888 
  889         auio.uio_iov = &aiov;
  890         auio.uio_iovcnt = 1;
  891         auio.uio_offset = cb->aio_offset;
  892         auio.uio_resid = cb->aio_nbytes;
  893         cnt = cb->aio_nbytes;
  894         auio.uio_segflg = UIO_USERSPACE;
  895         auio.uio_td = td;
  896 
  897         inblock_st = td->td_ru.ru_inblock;
  898         oublock_st = td->td_ru.ru_oublock;
  899         /*
  900          * aio_aqueue() acquires a reference to the file that is
  901          * released in aio_free_entry().
  902          */
  903         if (cb->aio_lio_opcode == LIO_READ) {
  904                 auio.uio_rw = UIO_READ;
  905                 if (auio.uio_resid == 0)
  906                         error = 0;
  907                 else
  908                         error = fo_read(fp, &auio, fp->f_cred, FOF_OFFSET, td);
  909         } else {
  910                 if (fp->f_type == DTYPE_VNODE)
  911                         bwillwrite();
  912                 auio.uio_rw = UIO_WRITE;
  913                 error = fo_write(fp, &auio, fp->f_cred, FOF_OFFSET, td);
  914         }
  915         inblock_end = td->td_ru.ru_inblock;
  916         oublock_end = td->td_ru.ru_oublock;
  917 
  918         aiocbe->inputcharge = inblock_end - inblock_st;
  919         aiocbe->outputcharge = oublock_end - oublock_st;
  920 
  921         if ((error) && (auio.uio_resid != cnt)) {
  922                 if (error == ERESTART || error == EINTR || error == EWOULDBLOCK)
  923                         error = 0;
  924                 if ((error == EPIPE) && (cb->aio_lio_opcode == LIO_WRITE)) {
  925                         int sigpipe = 1;
  926                         if (fp->f_type == DTYPE_SOCKET) {
  927                                 so = fp->f_data;
  928                                 if (so->so_options & SO_NOSIGPIPE)
  929                                         sigpipe = 0;
  930                         }
  931                         if (sigpipe) {
  932                                 PROC_LOCK(aiocbe->userproc);
  933                                 psignal(aiocbe->userproc, SIGPIPE);
  934                                 PROC_UNLOCK(aiocbe->userproc);
  935                         }
  936                 }
  937         }
  938 
  939         cnt -= auio.uio_resid;
  940         cb->_aiocb_private.error = error;
  941         cb->_aiocb_private.status = cnt;
  942         td->td_ucred = td_savedcred;
  943 }
  944 
  945 static void
  946 aio_bio_done_notify(struct proc *userp, struct aiocblist *aiocbe, int type)
  947 {
  948         struct aioliojob *lj;
  949         struct kaioinfo *ki;
  950         struct aiocblist *scb, *scbn;
  951         int lj_done;
  952 
  953         ki = userp->p_aioinfo;
  954         AIO_LOCK_ASSERT(ki, MA_OWNED);
  955         lj = aiocbe->lio;
  956         lj_done = 0;
  957         if (lj) {
  958                 lj->lioj_finished_count++;
  959                 if (lj->lioj_count == lj->lioj_finished_count)
  960                         lj_done = 1;
  961         }
  962         if (type == DONE_QUEUE) {
  963                 aiocbe->jobflags |= AIOCBLIST_DONE;
  964         } else {
  965                 aiocbe->jobflags |= AIOCBLIST_BUFDONE;
  966         }
  967         TAILQ_INSERT_TAIL(&ki->kaio_done, aiocbe, plist);
  968         aiocbe->jobstate = JOBST_JOBFINISHED;
  969 
  970         if (ki->kaio_flags & KAIO_RUNDOWN)
  971                 goto notification_done;
  972 
  973         if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL ||
  974             aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID)
  975                 aio_sendsig(userp, &aiocbe->uaiocb.aio_sigevent, &aiocbe->ksi);
  976 
  977         KNOTE_LOCKED(&aiocbe->klist, 1);
  978 
  979         if (lj_done) {
  980                 if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
  981                         lj->lioj_flags |= LIOJ_KEVENT_POSTED;
  982                         KNOTE_LOCKED(&lj->klist, 1);
  983                 }
  984                 if ((lj->lioj_flags & (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED))
  985                     == LIOJ_SIGNAL
  986                     && (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL ||
  987                         lj->lioj_signal.sigev_notify == SIGEV_THREAD_ID)) {
  988                         aio_sendsig(userp, &lj->lioj_signal, &lj->lioj_ksi);
  989                         lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
  990                 }
  991         }
  992 
  993 notification_done:
  994         if (aiocbe->jobflags & AIOCBLIST_CHECKSYNC) {
  995                 TAILQ_FOREACH_SAFE(scb, &ki->kaio_syncqueue, list, scbn) {
  996                         if (aiocbe->fd_file == scb->fd_file &&
  997                             aiocbe->seqno < scb->seqno) {
  998                                 if (--scb->pending == 0) {
  999                                         mtx_lock(&aio_job_mtx);
 1000                                         scb->jobstate = JOBST_JOBQGLOBAL;
 1001                                         TAILQ_REMOVE(&ki->kaio_syncqueue, scb, list);
 1002                                         TAILQ_INSERT_TAIL(&aio_jobs, scb, list);
 1003                                         aio_kick_nowait(userp);
 1004                                         mtx_unlock(&aio_job_mtx);
 1005                                 }
 1006                         }
 1007                 }
 1008         }
 1009         if (ki->kaio_flags & KAIO_WAKEUP) {
 1010                 ki->kaio_flags &= ~KAIO_WAKEUP;
 1011                 wakeup(&userp->p_aioinfo);
 1012         }
 1013 }
 1014 
 1015 /*
 1016  * The AIO daemon, most of the actual work is done in aio_process,
 1017  * but the setup (and address space mgmt) is done in this routine.
 1018  */
 1019 static void
 1020 aio_daemon(void *_id)
 1021 {
 1022         struct aiocblist *aiocbe;
 1023         struct aiothreadlist *aiop;
 1024         struct kaioinfo *ki;
 1025         struct proc *curcp, *mycp, *userp;
 1026         struct vmspace *myvm, *tmpvm;
 1027         struct thread *td = curthread;
 1028         int id = (intptr_t)_id;
 1029 
 1030         /*
 1031          * Local copies of curproc (cp) and vmspace (myvm)
 1032          */
 1033         mycp = td->td_proc;
 1034         myvm = mycp->p_vmspace;
 1035 
 1036         KASSERT(mycp->p_textvp == NULL, ("kthread has a textvp"));
 1037 
 1038         /*
 1039          * Allocate and ready the aio control info.  There is one aiop structure
 1040          * per daemon.
 1041          */
 1042         aiop = uma_zalloc(aiop_zone, M_WAITOK);
 1043         aiop->aiothread = td;
 1044         aiop->aiothreadflags = 0;
 1045 
 1046         /* The daemon resides in its own pgrp. */
 1047         setsid(td, NULL);
 1048 
 1049         /*
 1050          * Wakeup parent process.  (Parent sleeps to keep from blasting away
 1051          * and creating too many daemons.)
 1052          */
 1053         sema_post(&aio_newproc_sem);
 1054 
 1055         mtx_lock(&aio_job_mtx);
 1056         for (;;) {
 1057                 /*
 1058                  * curcp is the current daemon process context.
 1059                  * userp is the current user process context.
 1060                  */
 1061                 curcp = mycp;
 1062 
 1063                 /*
 1064                  * Take daemon off of free queue
 1065                  */
 1066                 if (aiop->aiothreadflags & AIOP_FREE) {
 1067                         TAILQ_REMOVE(&aio_freeproc, aiop, list);
 1068                         aiop->aiothreadflags &= ~AIOP_FREE;
 1069                 }
 1070 
 1071                 /*
 1072                  * Check for jobs.
 1073                  */
 1074                 while ((aiocbe = aio_selectjob(aiop)) != NULL) {
 1075                         mtx_unlock(&aio_job_mtx);
 1076                         userp = aiocbe->userproc;
 1077 
 1078                         /*
 1079                          * Connect to process address space for user program.
 1080                          */
 1081                         if (userp != curcp) {
 1082                                 /*
 1083                                  * Save the current address space that we are
 1084                                  * connected to.
 1085                                  */
 1086                                 tmpvm = mycp->p_vmspace;
 1087 
 1088                                 /*
 1089                                  * Point to the new user address space, and
 1090                                  * refer to it.
 1091                                  */
 1092                                 mycp->p_vmspace = userp->p_vmspace;
 1093                                 atomic_add_int(&mycp->p_vmspace->vm_refcnt, 1);
 1094 
 1095                                 /* Activate the new mapping. */
 1096                                 pmap_activate(FIRST_THREAD_IN_PROC(mycp));
 1097 
 1098                                 /*
 1099                                  * If the old address space wasn't the daemons
 1100                                  * own address space, then we need to remove the
 1101                                  * daemon's reference from the other process
 1102                                  * that it was acting on behalf of.
 1103                                  */
 1104                                 if (tmpvm != myvm) {
 1105                                         vmspace_free(tmpvm);
 1106                                 }
 1107                                 curcp = userp;
 1108                         }
 1109 
 1110                         ki = userp->p_aioinfo;
 1111 
 1112                         /* Do the I/O function. */
 1113                         aio_process(aiocbe);
 1114 
 1115                         mtx_lock(&aio_job_mtx);
 1116                         /* Decrement the active job count. */
 1117                         ki->kaio_active_count--;
 1118                         mtx_unlock(&aio_job_mtx);
 1119 
 1120                         AIO_LOCK(ki);
 1121                         TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
 1122                         aio_bio_done_notify(userp, aiocbe, DONE_QUEUE);
 1123                         AIO_UNLOCK(ki);
 1124 
 1125                         mtx_lock(&aio_job_mtx);
 1126                 }
 1127 
 1128                 /*
 1129                  * Disconnect from user address space.
 1130                  */
 1131                 if (curcp != mycp) {
 1132 
 1133                         mtx_unlock(&aio_job_mtx);
 1134 
 1135                         /* Get the user address space to disconnect from. */
 1136                         tmpvm = mycp->p_vmspace;
 1137 
 1138                         /* Get original address space for daemon. */
 1139                         mycp->p_vmspace = myvm;
 1140 
 1141                         /* Activate the daemon's address space. */
 1142                         pmap_activate(FIRST_THREAD_IN_PROC(mycp));
 1143 #ifdef DIAGNOSTIC
 1144                         if (tmpvm == myvm) {
 1145                                 printf("AIOD: vmspace problem -- %d\n",
 1146                                     mycp->p_pid);
 1147                         }
 1148 #endif
 1149                         /* Remove our vmspace reference. */
 1150                         vmspace_free(tmpvm);
 1151 
 1152                         curcp = mycp;
 1153 
 1154                         mtx_lock(&aio_job_mtx);
 1155                         /*
 1156                          * We have to restart to avoid race, we only sleep if
 1157                          * no job can be selected, that should be
 1158                          * curcp == mycp.
 1159                          */
 1160                         continue;
 1161                 }
 1162 
 1163                 mtx_assert(&aio_job_mtx, MA_OWNED);
 1164 
 1165                 TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
 1166                 aiop->aiothreadflags |= AIOP_FREE;
 1167 
 1168                 /*
 1169                  * If daemon is inactive for a long time, allow it to exit,
 1170                  * thereby freeing resources.
 1171                  */
 1172                 if (msleep(aiop->aiothread, &aio_job_mtx, PRIBIO, "aiordy",
 1173                     aiod_lifetime)) {
 1174                         if (TAILQ_EMPTY(&aio_jobs)) {
 1175                                 if ((aiop->aiothreadflags & AIOP_FREE) &&
 1176                                     (num_aio_procs > target_aio_procs)) {
 1177                                         TAILQ_REMOVE(&aio_freeproc, aiop, list);
 1178                                         num_aio_procs--;
 1179                                         mtx_unlock(&aio_job_mtx);
 1180                                         uma_zfree(aiop_zone, aiop);
 1181                                         free_unr(aiod_unr, id);
 1182 #ifdef DIAGNOSTIC
 1183                                         if (mycp->p_vmspace->vm_refcnt <= 1) {
 1184                                                 printf("AIOD: bad vm refcnt for"
 1185                                                     " exiting daemon: %d\n",
 1186                                                     mycp->p_vmspace->vm_refcnt);
 1187                                         }
 1188 #endif
 1189                                         kproc_exit(0);
 1190                                 }
 1191                         }
 1192                 }
 1193         }
 1194         mtx_unlock(&aio_job_mtx);
 1195         panic("shouldn't be here\n");
 1196 }
 1197 
 1198 /*
 1199  * Create a new AIO daemon. This is mostly a kernel-thread fork routine. The
 1200  * AIO daemon modifies its environment itself.
 1201  */
 1202 static int
 1203 aio_newproc(int *start)
 1204 {
 1205         int error;
 1206         struct proc *p;
 1207         int id;
 1208 
 1209         id = alloc_unr(aiod_unr);
 1210         error = kproc_create(aio_daemon, (void *)(intptr_t)id, &p,
 1211                 RFNOWAIT, 0, "aiod%d", id);
 1212         if (error == 0) {
 1213                 /*
 1214                  * Wait until daemon is started.
 1215                  */
 1216                 sema_wait(&aio_newproc_sem);
 1217                 mtx_lock(&aio_job_mtx);
 1218                 num_aio_procs++;
 1219                 if (start != NULL)
 1220                         (*start)--;
 1221                 mtx_unlock(&aio_job_mtx);
 1222         } else {
 1223                 free_unr(aiod_unr, id);
 1224         }
 1225         return (error);
 1226 }
 1227 
 1228 /*
 1229  * Try the high-performance, low-overhead physio method for eligible
 1230  * VCHR devices.  This method doesn't use an aio helper thread, and
 1231  * thus has very low overhead.
 1232  *
 1233  * Assumes that the caller, aio_aqueue(), has incremented the file
 1234  * structure's reference count, preventing its deallocation for the
 1235  * duration of this call.
 1236  */
 1237 static int
 1238 aio_qphysio(struct proc *p, struct aiocblist *aiocbe)
 1239 {
 1240         struct aiocb *cb;
 1241         struct file *fp;
 1242         struct buf *bp;
 1243         struct vnode *vp;
 1244         struct kaioinfo *ki;
 1245         struct aioliojob *lj;
 1246         int error;
 1247 
 1248         cb = &aiocbe->uaiocb;
 1249         fp = aiocbe->fd_file;
 1250 
 1251         if (fp->f_type != DTYPE_VNODE)
 1252                 return (-1);
 1253 
 1254         vp = fp->f_vnode;
 1255 
 1256         /*
 1257          * If its not a disk, we don't want to return a positive error.
 1258          * It causes the aio code to not fall through to try the thread
 1259          * way when you're talking to a regular file.
 1260          */
 1261         if (!vn_isdisk(vp, &error)) {
 1262                 if (error == ENOTBLK)
 1263                         return (-1);
 1264                 else
 1265                         return (error);
 1266         }
 1267 
 1268         if (vp->v_bufobj.bo_bsize == 0)
 1269                 return (-1);
 1270 
 1271         if (cb->aio_nbytes % vp->v_bufobj.bo_bsize)
 1272                 return (-1);
 1273 
 1274         if (cb->aio_nbytes > vp->v_rdev->si_iosize_max)
 1275                 return (-1);
 1276 
 1277         if (cb->aio_nbytes >
 1278             MAXPHYS - (((vm_offset_t) cb->aio_buf) & PAGE_MASK))
 1279                 return (-1);
 1280 
 1281         ki = p->p_aioinfo;
 1282         if (ki->kaio_buffer_count >= ki->kaio_ballowed_count)
 1283                 return (-1);
 1284 
 1285         /* Create and build a buffer header for a transfer. */
 1286         bp = (struct buf *)getpbuf(NULL);
 1287         BUF_KERNPROC(bp);
 1288 
 1289         AIO_LOCK(ki);
 1290         ki->kaio_count++;
 1291         ki->kaio_buffer_count++;
 1292         lj = aiocbe->lio;
 1293         if (lj)
 1294                 lj->lioj_count++;
 1295         AIO_UNLOCK(ki);
 1296 
 1297         /*
 1298          * Get a copy of the kva from the physical buffer.
 1299          */
 1300         error = 0;
 1301 
 1302         bp->b_bcount = cb->aio_nbytes;
 1303         bp->b_bufsize = cb->aio_nbytes;
 1304         bp->b_iodone = aio_physwakeup;
 1305         bp->b_saveaddr = bp->b_data;
 1306         bp->b_data = (void *)(uintptr_t)cb->aio_buf;
 1307         bp->b_offset = cb->aio_offset;
 1308         bp->b_iooffset = cb->aio_offset;
 1309         bp->b_blkno = btodb(cb->aio_offset);
 1310         bp->b_iocmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ;
 1311 
 1312         /*
 1313          * Bring buffer into kernel space.
 1314          */
 1315         if (vmapbuf(bp) < 0) {
 1316                 error = EFAULT;
 1317                 goto doerror;
 1318         }
 1319 
 1320         AIO_LOCK(ki);
 1321         aiocbe->bp = bp;
 1322         bp->b_caller1 = (void *)aiocbe;
 1323         TAILQ_INSERT_TAIL(&ki->kaio_bufqueue, aiocbe, plist);
 1324         TAILQ_INSERT_TAIL(&ki->kaio_all, aiocbe, allist);
 1325         aiocbe->jobstate = JOBST_JOBQBUF;
 1326         cb->_aiocb_private.status = cb->aio_nbytes;
 1327         AIO_UNLOCK(ki);
 1328 
 1329         atomic_add_int(&num_queue_count, 1);
 1330         atomic_add_int(&num_buf_aio, 1);
 1331 
 1332         bp->b_error = 0;
 1333 
 1334         TASK_INIT(&aiocbe->biotask, 0, biohelper, aiocbe);
 1335 
 1336         /* Perform transfer. */
 1337         dev_strategy(vp->v_rdev, bp);
 1338         return (0);
 1339 
 1340 doerror:
 1341         AIO_LOCK(ki);
 1342         ki->kaio_count--;
 1343         ki->kaio_buffer_count--;
 1344         if (lj)
 1345                 lj->lioj_count--;
 1346         aiocbe->bp = NULL;
 1347         AIO_UNLOCK(ki);
 1348         relpbuf(bp, NULL);
 1349         return (error);
 1350 }
 1351 
 1352 /*
 1353  * Wake up aio requests that may be serviceable now.
 1354  */
 1355 static void
 1356 aio_swake_cb(struct socket *so, struct sockbuf *sb)
 1357 {
 1358         struct aiocblist *cb, *cbn;
 1359         int opcode;
 1360 
 1361         SOCKBUF_LOCK_ASSERT(sb);
 1362         if (sb == &so->so_snd)
 1363                 opcode = LIO_WRITE;
 1364         else
 1365                 opcode = LIO_READ;
 1366 
 1367         sb->sb_flags &= ~SB_AIO;
 1368         mtx_lock(&aio_job_mtx);
 1369         TAILQ_FOREACH_SAFE(cb, &so->so_aiojobq, list, cbn) {
 1370                 if (opcode == cb->uaiocb.aio_lio_opcode) {
 1371                         if (cb->jobstate != JOBST_JOBQSOCK)
 1372                                 panic("invalid queue value");
 1373                         /* XXX
 1374                          * We don't have actual sockets backend yet,
 1375                          * so we simply move the requests to the generic
 1376                          * file I/O backend.
 1377                          */
 1378                         TAILQ_REMOVE(&so->so_aiojobq, cb, list);
 1379                         TAILQ_INSERT_TAIL(&aio_jobs, cb, list);
 1380                         aio_kick_nowait(cb->userproc);
 1381                 }
 1382         }
 1383         mtx_unlock(&aio_job_mtx);
 1384 }
 1385 
 1386 static int
 1387 convert_old_sigevent(struct osigevent *osig, struct sigevent *nsig)
 1388 {
 1389 
 1390         /*
 1391          * Only SIGEV_NONE, SIGEV_SIGNAL, and SIGEV_KEVENT are
 1392          * supported by AIO with the old sigevent structure.
 1393          */
 1394         nsig->sigev_notify = osig->sigev_notify;
 1395         switch (nsig->sigev_notify) {
 1396         case SIGEV_NONE:
 1397                 break;
 1398         case SIGEV_SIGNAL:
 1399                 nsig->sigev_signo = osig->__sigev_u.__sigev_signo;
 1400                 break;
 1401         case SIGEV_KEVENT:
 1402                 nsig->sigev_notify_kqueue =
 1403                     osig->__sigev_u.__sigev_notify_kqueue;
 1404                 nsig->sigev_value.sival_ptr = osig->sigev_value.sival_ptr;
 1405                 break;
 1406         default:
 1407                 return (EINVAL);
 1408         }
 1409         return (0);
 1410 }
 1411 
 1412 static int
 1413 aiocb_copyin_old_sigevent(struct aiocb *ujob, struct aiocb *kjob)
 1414 {
 1415         struct oaiocb *ojob;
 1416         int error;
 1417 
 1418         bzero(kjob, sizeof(struct aiocb));
 1419         error = copyin(ujob, kjob, sizeof(struct oaiocb));
 1420         if (error)
 1421                 return (error);
 1422         ojob = (struct oaiocb *)kjob;
 1423         return (convert_old_sigevent(&ojob->aio_sigevent, &kjob->aio_sigevent));
 1424 }
 1425 
 1426 static int
 1427 aiocb_copyin(struct aiocb *ujob, struct aiocb *kjob)
 1428 {
 1429 
 1430         return (copyin(ujob, kjob, sizeof(struct aiocb)));
 1431 }
 1432 
 1433 static long
 1434 aiocb_fetch_status(struct aiocb *ujob)
 1435 {
 1436 
 1437         return (fuword(&ujob->_aiocb_private.status));
 1438 }
 1439 
 1440 static long
 1441 aiocb_fetch_error(struct aiocb *ujob)
 1442 {
 1443 
 1444         return (fuword(&ujob->_aiocb_private.error));
 1445 }
 1446 
 1447 static int
 1448 aiocb_store_status(struct aiocb *ujob, long status)
 1449 {
 1450 
 1451         return (suword(&ujob->_aiocb_private.status, status));
 1452 }
 1453 
 1454 static int
 1455 aiocb_store_error(struct aiocb *ujob, long error)
 1456 {
 1457 
 1458         return (suword(&ujob->_aiocb_private.error, error));
 1459 }
 1460 
 1461 static int
 1462 aiocb_store_kernelinfo(struct aiocb *ujob, long jobref)
 1463 {
 1464 
 1465         return (suword(&ujob->_aiocb_private.kernelinfo, jobref));
 1466 }
 1467 
 1468 static int
 1469 aiocb_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob)
 1470 {
 1471 
 1472         return (suword(ujobp, (long)ujob));
 1473 }
 1474 
 1475 static struct aiocb_ops aiocb_ops = {
 1476         .copyin = aiocb_copyin,
 1477         .fetch_status = aiocb_fetch_status,
 1478         .fetch_error = aiocb_fetch_error,
 1479         .store_status = aiocb_store_status,
 1480         .store_error = aiocb_store_error,
 1481         .store_kernelinfo = aiocb_store_kernelinfo,
 1482         .store_aiocb = aiocb_store_aiocb,
 1483 };
 1484 
 1485 static struct aiocb_ops aiocb_ops_osigevent = {
 1486         .copyin = aiocb_copyin_old_sigevent,
 1487         .fetch_status = aiocb_fetch_status,
 1488         .fetch_error = aiocb_fetch_error,
 1489         .store_status = aiocb_store_status,
 1490         .store_error = aiocb_store_error,
 1491         .store_kernelinfo = aiocb_store_kernelinfo,
 1492         .store_aiocb = aiocb_store_aiocb,
 1493 };
 1494 
 1495 /*
 1496  * Queue a new AIO request.  Choosing either the threaded or direct physio VCHR
 1497  * technique is done in this code.
 1498  */
 1499 int
 1500 aio_aqueue(struct thread *td, struct aiocb *job, struct aioliojob *lj,
 1501         int type, struct aiocb_ops *ops)
 1502 {
 1503         struct proc *p = td->td_proc;
 1504         struct file *fp;
 1505         struct socket *so;
 1506         struct aiocblist *aiocbe, *cb;
 1507         struct kaioinfo *ki;
 1508         struct kevent kev;
 1509         struct sockbuf *sb;
 1510         int opcode;
 1511         int error;
 1512         int fd, kqfd;
 1513         int jid;
 1514 
 1515         if (p->p_aioinfo == NULL)
 1516                 aio_init_aioinfo(p);
 1517 
 1518         ki = p->p_aioinfo;
 1519 
 1520         ops->store_status(job, -1);
 1521         ops->store_error(job, 0);
 1522         ops->store_kernelinfo(job, -1);
 1523 
 1524         if (num_queue_count >= max_queue_count ||
 1525             ki->kaio_count >= ki->kaio_qallowed_count) {
 1526                 ops->store_error(job, EAGAIN);
 1527                 return (EAGAIN);
 1528         }
 1529 
 1530         aiocbe = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO);
 1531         aiocbe->inputcharge = 0;
 1532         aiocbe->outputcharge = 0;
 1533         knlist_init_mtx(&aiocbe->klist, AIO_MTX(ki));
 1534 
 1535         error = ops->copyin(job, &aiocbe->uaiocb);
 1536         if (error) {
 1537                 ops->store_error(job, error);
 1538                 uma_zfree(aiocb_zone, aiocbe);
 1539                 return (error);
 1540         }
 1541 
 1542         if (aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT &&
 1543             aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_SIGNAL &&
 1544             aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_THREAD_ID &&
 1545             aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_NONE) {
 1546                 ops->store_error(job, EINVAL);
 1547                 uma_zfree(aiocb_zone, aiocbe);
 1548                 return (EINVAL);
 1549         }
 1550 
 1551         if ((aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL ||
 1552              aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) &&
 1553                 !_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) {
 1554                 uma_zfree(aiocb_zone, aiocbe);
 1555                 return (EINVAL);
 1556         }
 1557 
 1558         ksiginfo_init(&aiocbe->ksi);
 1559 
 1560         /* Save userspace address of the job info. */
 1561         aiocbe->uuaiocb = job;
 1562 
 1563         /* Get the opcode. */
 1564         if (type != LIO_NOP)
 1565                 aiocbe->uaiocb.aio_lio_opcode = type;
 1566         opcode = aiocbe->uaiocb.aio_lio_opcode;
 1567 
 1568         /* Fetch the file object for the specified file descriptor. */
 1569         fd = aiocbe->uaiocb.aio_fildes;
 1570         switch (opcode) {
 1571         case LIO_WRITE:
 1572                 error = fget_write(td, fd, &fp);
 1573                 break;
 1574         case LIO_READ:
 1575                 error = fget_read(td, fd, &fp);
 1576                 break;
 1577         default:
 1578                 error = fget(td, fd, &fp);
 1579         }
 1580         if (error) {
 1581                 uma_zfree(aiocb_zone, aiocbe);
 1582                 ops->store_error(job, error);
 1583                 return (error);
 1584         }
 1585 
 1586         if (opcode == LIO_SYNC && fp->f_vnode == NULL) {
 1587                 error = EINVAL;
 1588                 goto aqueue_fail;
 1589         }
 1590 
 1591         if (opcode != LIO_SYNC && aiocbe->uaiocb.aio_offset == -1LL) {
 1592                 error = EINVAL;
 1593                 goto aqueue_fail;
 1594         }
 1595 
 1596         aiocbe->fd_file = fp;
 1597 
 1598         mtx_lock(&aio_job_mtx);
 1599         jid = jobrefid++;
 1600         aiocbe->seqno = jobseqno++;
 1601         mtx_unlock(&aio_job_mtx);
 1602         error = ops->store_kernelinfo(job, jid);
 1603         if (error) {
 1604                 error = EINVAL;
 1605                 goto aqueue_fail;
 1606         }
 1607         aiocbe->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jid;
 1608 
 1609         if (opcode == LIO_NOP) {
 1610                 fdrop(fp, td);
 1611                 uma_zfree(aiocb_zone, aiocbe);
 1612                 return (0);
 1613         }
 1614         if ((opcode != LIO_READ) && (opcode != LIO_WRITE) &&
 1615             (opcode != LIO_SYNC)) {
 1616                 error = EINVAL;
 1617                 goto aqueue_fail;
 1618         }
 1619 
 1620         if (aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT)
 1621                 goto no_kqueue;
 1622         kqfd = aiocbe->uaiocb.aio_sigevent.sigev_notify_kqueue;
 1623         kev.ident = (uintptr_t)aiocbe->uuaiocb;
 1624         kev.filter = EVFILT_AIO;
 1625         kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
 1626         kev.data = (intptr_t)aiocbe;
 1627         kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sival_ptr;
 1628         error = kqfd_register(kqfd, &kev, td, 1);
 1629 aqueue_fail:
 1630         if (error) {
 1631                 fdrop(fp, td);
 1632                 uma_zfree(aiocb_zone, aiocbe);
 1633                 ops->store_error(job, error);
 1634                 goto done;
 1635         }
 1636 no_kqueue:
 1637 
 1638         ops->store_error(job, EINPROGRESS);
 1639         aiocbe->uaiocb._aiocb_private.error = EINPROGRESS;
 1640         aiocbe->userproc = p;
 1641         aiocbe->cred = crhold(td->td_ucred);
 1642         aiocbe->jobflags = 0;
 1643         aiocbe->lio = lj;
 1644 
 1645         if (opcode == LIO_SYNC)
 1646                 goto queueit;
 1647 
 1648         if (fp->f_type == DTYPE_SOCKET) {
 1649                 /*
 1650                  * Alternate queueing for socket ops: Reach down into the
 1651                  * descriptor to get the socket data.  Then check to see if the
 1652                  * socket is ready to be read or written (based on the requested
 1653                  * operation).
 1654                  *
 1655                  * If it is not ready for io, then queue the aiocbe on the
 1656                  * socket, and set the flags so we get a call when sbnotify()
 1657                  * happens.
 1658                  *
 1659                  * Note if opcode is neither LIO_WRITE nor LIO_READ we lock
 1660                  * and unlock the snd sockbuf for no reason.
 1661                  */
 1662                 so = fp->f_data;
 1663                 sb = (opcode == LIO_READ) ? &so->so_rcv : &so->so_snd;
 1664                 SOCKBUF_LOCK(sb);
 1665                 if (((opcode == LIO_READ) && (!soreadable(so))) || ((opcode ==
 1666                     LIO_WRITE) && (!sowriteable(so)))) {
 1667                         sb->sb_flags |= SB_AIO;
 1668 
 1669                         mtx_lock(&aio_job_mtx);
 1670                         TAILQ_INSERT_TAIL(&so->so_aiojobq, aiocbe, list);
 1671                         mtx_unlock(&aio_job_mtx);
 1672 
 1673                         AIO_LOCK(ki);
 1674                         TAILQ_INSERT_TAIL(&ki->kaio_all, aiocbe, allist);
 1675                         TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist);
 1676                         aiocbe->jobstate = JOBST_JOBQSOCK;
 1677                         ki->kaio_count++;
 1678                         if (lj)
 1679                                 lj->lioj_count++;
 1680                         AIO_UNLOCK(ki);
 1681                         SOCKBUF_UNLOCK(sb);
 1682                         atomic_add_int(&num_queue_count, 1);
 1683                         error = 0;
 1684                         goto done;
 1685                 }
 1686                 SOCKBUF_UNLOCK(sb);
 1687         }
 1688 
 1689         if ((error = aio_qphysio(p, aiocbe)) == 0)
 1690                 goto done;
 1691 #if 0
 1692         if (error > 0) {
 1693                 aiocbe->uaiocb._aiocb_private.error = error;
 1694                 ops->store_error(job, error);
 1695                 goto done;
 1696         }
 1697 #endif
 1698 queueit:
 1699         /* No buffer for daemon I/O. */
 1700         aiocbe->bp = NULL;
 1701         atomic_add_int(&num_queue_count, 1);
 1702 
 1703         AIO_LOCK(ki);
 1704         ki->kaio_count++;
 1705         if (lj)
 1706                 lj->lioj_count++;
 1707         TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist);
 1708         TAILQ_INSERT_TAIL(&ki->kaio_all, aiocbe, allist);
 1709         if (opcode == LIO_SYNC) {
 1710                 TAILQ_FOREACH(cb, &ki->kaio_jobqueue, plist) {
 1711                         if (cb->fd_file == aiocbe->fd_file &&
 1712                             cb->uaiocb.aio_lio_opcode != LIO_SYNC &&
 1713                             cb->seqno < aiocbe->seqno) {
 1714                                 cb->jobflags |= AIOCBLIST_CHECKSYNC;
 1715                                 aiocbe->pending++;
 1716                         }
 1717                 }
 1718                 TAILQ_FOREACH(cb, &ki->kaio_bufqueue, plist) {
 1719                         if (cb->fd_file == aiocbe->fd_file &&
 1720                             cb->uaiocb.aio_lio_opcode != LIO_SYNC &&
 1721                             cb->seqno < aiocbe->seqno) {
 1722                                 cb->jobflags |= AIOCBLIST_CHECKSYNC;
 1723                                 aiocbe->pending++;
 1724                         }
 1725                 }
 1726                 if (aiocbe->pending != 0) {
 1727                         TAILQ_INSERT_TAIL(&ki->kaio_syncqueue, aiocbe, list);
 1728                         aiocbe->jobstate = JOBST_JOBQSYNC;
 1729                         AIO_UNLOCK(ki);
 1730                         goto done;
 1731                 }
 1732         }
 1733         mtx_lock(&aio_job_mtx);
 1734         TAILQ_INSERT_TAIL(&aio_jobs, aiocbe, list);
 1735         aiocbe->jobstate = JOBST_JOBQGLOBAL;
 1736         aio_kick_nowait(p);
 1737         mtx_unlock(&aio_job_mtx);
 1738         AIO_UNLOCK(ki);
 1739         error = 0;
 1740 done:
 1741         return (error);
 1742 }
 1743 
 1744 static void
 1745 aio_kick_nowait(struct proc *userp)
 1746 {
 1747         struct kaioinfo *ki = userp->p_aioinfo;
 1748         struct aiothreadlist *aiop;
 1749 
 1750         mtx_assert(&aio_job_mtx, MA_OWNED);
 1751         if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
 1752                 TAILQ_REMOVE(&aio_freeproc, aiop, list);
 1753                 aiop->aiothreadflags &= ~AIOP_FREE;
 1754                 wakeup(aiop->aiothread);
 1755         } else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) &&
 1756             ((ki->kaio_active_count + num_aio_resv_start) <
 1757             ki->kaio_maxactive_count)) {
 1758                 taskqueue_enqueue(taskqueue_aiod_bio, &ki->kaio_task);
 1759         }
 1760 }
 1761 
 1762 static int
 1763 aio_kick(struct proc *userp)
 1764 {
 1765         struct kaioinfo *ki = userp->p_aioinfo;
 1766         struct aiothreadlist *aiop;
 1767         int error, ret = 0;
 1768 
 1769         mtx_assert(&aio_job_mtx, MA_OWNED);
 1770 retryproc:
 1771         if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
 1772                 TAILQ_REMOVE(&aio_freeproc, aiop, list);
 1773                 aiop->aiothreadflags &= ~AIOP_FREE;
 1774                 wakeup(aiop->aiothread);
 1775         } else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) &&
 1776             ((ki->kaio_active_count + num_aio_resv_start) <
 1777             ki->kaio_maxactive_count)) {
 1778                 num_aio_resv_start++;
 1779                 mtx_unlock(&aio_job_mtx);
 1780                 error = aio_newproc(&num_aio_resv_start);
 1781                 mtx_lock(&aio_job_mtx);
 1782                 if (error) {
 1783                         num_aio_resv_start--;
 1784                         goto retryproc;
 1785                 }
 1786         } else {
 1787                 ret = -1;
 1788         }
 1789         return (ret);
 1790 }
 1791 
 1792 static void
 1793 aio_kick_helper(void *context, int pending)
 1794 {
 1795         struct proc *userp = context;
 1796 
 1797         mtx_lock(&aio_job_mtx);
 1798         while (--pending >= 0) {
 1799                 if (aio_kick(userp))
 1800                         break;
 1801         }
 1802         mtx_unlock(&aio_job_mtx);
 1803 }
 1804 
 1805 /*
 1806  * Support the aio_return system call, as a side-effect, kernel resources are
 1807  * released.
 1808  */
 1809 static int
 1810 kern_aio_return(struct thread *td, struct aiocb *uaiocb, struct aiocb_ops *ops)
 1811 {
 1812         struct proc *p = td->td_proc;
 1813         struct aiocblist *cb;
 1814         struct kaioinfo *ki;
 1815         int status, error;
 1816 
 1817         ki = p->p_aioinfo;
 1818         if (ki == NULL)
 1819                 return (EINVAL);
 1820         AIO_LOCK(ki);
 1821         TAILQ_FOREACH(cb, &ki->kaio_done, plist) {
 1822                 if (cb->uuaiocb == uaiocb)
 1823                         break;
 1824         }
 1825         if (cb != NULL) {
 1826                 MPASS(cb->jobstate == JOBST_JOBFINISHED);
 1827                 status = cb->uaiocb._aiocb_private.status;
 1828                 error = cb->uaiocb._aiocb_private.error;
 1829                 td->td_retval[0] = status;
 1830                 if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
 1831                         td->td_ru.ru_oublock += cb->outputcharge;
 1832                         cb->outputcharge = 0;
 1833                 } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
 1834                         td->td_ru.ru_inblock += cb->inputcharge;
 1835                         cb->inputcharge = 0;
 1836                 }
 1837                 aio_free_entry(cb);
 1838                 AIO_UNLOCK(ki);
 1839                 ops->store_error(uaiocb, error);
 1840                 ops->store_status(uaiocb, status);
 1841         } else {
 1842                 error = EINVAL;
 1843                 AIO_UNLOCK(ki);
 1844         }
 1845         return (error);
 1846 }
 1847 
 1848 int
 1849 aio_return(struct thread *td, struct aio_return_args *uap)
 1850 {
 1851 
 1852         return (kern_aio_return(td, uap->aiocbp, &aiocb_ops));
 1853 }
 1854 
 1855 /*
 1856  * Allow a process to wakeup when any of the I/O requests are completed.
 1857  */
 1858 static int
 1859 kern_aio_suspend(struct thread *td, int njoblist, struct aiocb **ujoblist,
 1860     struct timespec *ts)
 1861 {
 1862         struct proc *p = td->td_proc;
 1863         struct timeval atv;
 1864         struct kaioinfo *ki;
 1865         struct aiocblist *cb, *cbfirst;
 1866         int error, i, timo;
 1867 
 1868         timo = 0;
 1869         if (ts) {
 1870                 if (ts->tv_nsec < 0 || ts->tv_nsec >= 1000000000)
 1871                         return (EINVAL);
 1872 
 1873                 TIMESPEC_TO_TIMEVAL(&atv, ts);
 1874                 if (itimerfix(&atv))
 1875                         return (EINVAL);
 1876                 timo = tvtohz(&atv);
 1877         }
 1878 
 1879         ki = p->p_aioinfo;
 1880         if (ki == NULL)
 1881                 return (EAGAIN);
 1882 
 1883         if (njoblist == 0)
 1884                 return (0);
 1885 
 1886         AIO_LOCK(ki);
 1887         for (;;) {
 1888                 cbfirst = NULL;
 1889                 error = 0;
 1890                 TAILQ_FOREACH(cb, &ki->kaio_all, allist) {
 1891                         for (i = 0; i < njoblist; i++) {
 1892                                 if (cb->uuaiocb == ujoblist[i]) {
 1893                                         if (cbfirst == NULL)
 1894                                                 cbfirst = cb;
 1895                                         if (cb->jobstate == JOBST_JOBFINISHED)
 1896                                                 goto RETURN;
 1897                                 }
 1898                         }
 1899                 }
 1900                 /* All tasks were finished. */
 1901                 if (cbfirst == NULL)
 1902                         break;
 1903 
 1904                 ki->kaio_flags |= KAIO_WAKEUP;
 1905                 error = msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO | PCATCH,
 1906                     "aiospn", timo);
 1907                 if (error == ERESTART)
 1908                         error = EINTR;
 1909                 if (error)
 1910                         break;
 1911         }
 1912 RETURN:
 1913         AIO_UNLOCK(ki);
 1914         return (error);
 1915 }
 1916 
 1917 int
 1918 aio_suspend(struct thread *td, struct aio_suspend_args *uap)
 1919 {
 1920         struct timespec ts, *tsp;
 1921         struct aiocb **ujoblist;
 1922         int error;
 1923 
 1924         if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
 1925                 return (EINVAL);
 1926 
 1927         if (uap->timeout) {
 1928                 /* Get timespec struct. */
 1929                 if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0)
 1930                         return (error);
 1931                 tsp = &ts;
 1932         } else
 1933                 tsp = NULL;
 1934 
 1935         ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
 1936         error = copyin(uap->aiocbp, ujoblist, uap->nent * sizeof(ujoblist[0]));
 1937         if (error == 0)
 1938                 error = kern_aio_suspend(td, uap->nent, ujoblist, tsp);
 1939         uma_zfree(aiol_zone, ujoblist);
 1940         return (error);
 1941 }
 1942 
 1943 /*
 1944  * aio_cancel cancels any non-physio aio operations not currently in
 1945  * progress.
 1946  */
 1947 int
 1948 aio_cancel(struct thread *td, struct aio_cancel_args *uap)
 1949 {
 1950         struct proc *p = td->td_proc;
 1951         struct kaioinfo *ki;
 1952         struct aiocblist *cbe, *cbn;
 1953         struct file *fp;
 1954         struct socket *so;
 1955         int error;
 1956         int remove;
 1957         int cancelled = 0;
 1958         int notcancelled = 0;
 1959         struct vnode *vp;
 1960 
 1961         /* Lookup file object. */
 1962         error = fget(td, uap->fd, &fp);
 1963         if (error)
 1964                 return (error);
 1965 
 1966         ki = p->p_aioinfo;
 1967         if (ki == NULL)
 1968                 goto done;
 1969 
 1970         if (fp->f_type == DTYPE_VNODE) {
 1971                 vp = fp->f_vnode;
 1972                 if (vn_isdisk(vp, &error)) {
 1973                         fdrop(fp, td);
 1974                         td->td_retval[0] = AIO_NOTCANCELED;
 1975                         return (0);
 1976                 }
 1977         }
 1978 
 1979         AIO_LOCK(ki);
 1980         TAILQ_FOREACH_SAFE(cbe, &ki->kaio_jobqueue, plist, cbn) {
 1981                 if ((uap->fd == cbe->uaiocb.aio_fildes) &&
 1982                     ((uap->aiocbp == NULL) ||
 1983                      (uap->aiocbp == cbe->uuaiocb))) {
 1984                         remove = 0;
 1985 
 1986                         mtx_lock(&aio_job_mtx);
 1987                         if (cbe->jobstate == JOBST_JOBQGLOBAL) {
 1988                                 TAILQ_REMOVE(&aio_jobs, cbe, list);
 1989                                 remove = 1;
 1990                         } else if (cbe->jobstate == JOBST_JOBQSOCK) {
 1991                                 MPASS(fp->f_type == DTYPE_SOCKET);
 1992                                 so = fp->f_data;
 1993                                 TAILQ_REMOVE(&so->so_aiojobq, cbe, list);
 1994                                 remove = 1;
 1995                         } else if (cbe->jobstate == JOBST_JOBQSYNC) {
 1996                                 TAILQ_REMOVE(&ki->kaio_syncqueue, cbe, list);
 1997                                 remove = 1;
 1998                         }
 1999                         mtx_unlock(&aio_job_mtx);
 2000 
 2001                         if (remove) {
 2002                                 TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist);
 2003                                 cbe->uaiocb._aiocb_private.status = -1;
 2004                                 cbe->uaiocb._aiocb_private.error = ECANCELED;
 2005                                 aio_bio_done_notify(p, cbe, DONE_QUEUE);
 2006                                 cancelled++;
 2007                         } else {
 2008                                 notcancelled++;
 2009                         }
 2010                         if (uap->aiocbp != NULL)
 2011                                 break;
 2012                 }
 2013         }
 2014         AIO_UNLOCK(ki);
 2015 
 2016 done:
 2017         fdrop(fp, td);
 2018 
 2019         if (uap->aiocbp != NULL) {
 2020                 if (cancelled) {
 2021                         td->td_retval[0] = AIO_CANCELED;
 2022                         return (0);
 2023                 }
 2024         }
 2025 
 2026         if (notcancelled) {
 2027                 td->td_retval[0] = AIO_NOTCANCELED;
 2028                 return (0);
 2029         }
 2030 
 2031         if (cancelled) {
 2032                 td->td_retval[0] = AIO_CANCELED;
 2033                 return (0);
 2034         }
 2035 
 2036         td->td_retval[0] = AIO_ALLDONE;
 2037 
 2038         return (0);
 2039 }
 2040 
 2041 /*
 2042  * aio_error is implemented in the kernel level for compatibility purposes
 2043  * only.  For a user mode async implementation, it would be best to do it in
 2044  * a userland subroutine.
 2045  */
 2046 static int
 2047 kern_aio_error(struct thread *td, struct aiocb *aiocbp, struct aiocb_ops *ops)
 2048 {
 2049         struct proc *p = td->td_proc;
 2050         struct aiocblist *cb;
 2051         struct kaioinfo *ki;
 2052         int status;
 2053 
 2054         ki = p->p_aioinfo;
 2055         if (ki == NULL) {
 2056                 td->td_retval[0] = EINVAL;
 2057                 return (0);
 2058         }
 2059 
 2060         AIO_LOCK(ki);
 2061         TAILQ_FOREACH(cb, &ki->kaio_all, allist) {
 2062                 if (cb->uuaiocb == aiocbp) {
 2063                         if (cb->jobstate == JOBST_JOBFINISHED)
 2064                                 td->td_retval[0] =
 2065                                         cb->uaiocb._aiocb_private.error;
 2066                         else
 2067                                 td->td_retval[0] = EINPROGRESS;
 2068                         AIO_UNLOCK(ki);
 2069                         return (0);
 2070                 }
 2071         }
 2072         AIO_UNLOCK(ki);
 2073 
 2074         /*
 2075          * Hack for failure of aio_aqueue.
 2076          */
 2077         status = ops->fetch_status(aiocbp);
 2078         if (status == -1) {
 2079                 td->td_retval[0] = ops->fetch_error(aiocbp);
 2080                 return (0);
 2081         }
 2082 
 2083         td->td_retval[0] = EINVAL;
 2084         return (0);
 2085 }
 2086 
 2087 int
 2088 aio_error(struct thread *td, struct aio_error_args *uap)
 2089 {
 2090 
 2091         return (kern_aio_error(td, uap->aiocbp, &aiocb_ops));
 2092 }
 2093 
 2094 /* syscall - asynchronous read from a file (REALTIME) */
 2095 int
 2096 oaio_read(struct thread *td, struct oaio_read_args *uap)
 2097 {
 2098 
 2099         return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
 2100             &aiocb_ops_osigevent));
 2101 }
 2102 
 2103 int
 2104 aio_read(struct thread *td, struct aio_read_args *uap)
 2105 {
 2106 
 2107         return (aio_aqueue(td, uap->aiocbp, NULL, LIO_READ, &aiocb_ops));
 2108 }
 2109 
 2110 /* syscall - asynchronous write to a file (REALTIME) */
 2111 int
 2112 oaio_write(struct thread *td, struct oaio_write_args *uap)
 2113 {
 2114 
 2115         return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
 2116             &aiocb_ops_osigevent));
 2117 }
 2118 
 2119 int
 2120 aio_write(struct thread *td, struct aio_write_args *uap)
 2121 {
 2122 
 2123         return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, &aiocb_ops));
 2124 }
 2125 
 2126 static int
 2127 kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list,
 2128     struct aiocb **acb_list, int nent, struct sigevent *sig,
 2129     struct aiocb_ops *ops)
 2130 {
 2131         struct proc *p = td->td_proc;
 2132         struct aiocb *iocb;
 2133         struct kaioinfo *ki;
 2134         struct aioliojob *lj;
 2135         struct kevent kev;
 2136         int error;
 2137         int nerror;
 2138         int i;
 2139 
 2140         if ((mode != LIO_NOWAIT) && (mode != LIO_WAIT))
 2141                 return (EINVAL);
 2142 
 2143         if (nent < 0 || nent > AIO_LISTIO_MAX)
 2144                 return (EINVAL);
 2145 
 2146         if (p->p_aioinfo == NULL)
 2147                 aio_init_aioinfo(p);
 2148 
 2149         ki = p->p_aioinfo;
 2150 
 2151         lj = uma_zalloc(aiolio_zone, M_WAITOK);
 2152         lj->lioj_flags = 0;
 2153         lj->lioj_count = 0;
 2154         lj->lioj_finished_count = 0;
 2155         knlist_init_mtx(&lj->klist, AIO_MTX(ki));
 2156         ksiginfo_init(&lj->lioj_ksi);
 2157 
 2158         /*
 2159          * Setup signal.
 2160          */
 2161         if (sig && (mode == LIO_NOWAIT)) {
 2162                 bcopy(sig, &lj->lioj_signal, sizeof(lj->lioj_signal));
 2163                 if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
 2164                         /* Assume only new style KEVENT */
 2165                         kev.filter = EVFILT_LIO;
 2166                         kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
 2167                         kev.ident = (uintptr_t)uacb_list; /* something unique */
 2168                         kev.data = (intptr_t)lj;
 2169                         /* pass user defined sigval data */
 2170                         kev.udata = lj->lioj_signal.sigev_value.sival_ptr;
 2171                         error = kqfd_register(
 2172                             lj->lioj_signal.sigev_notify_kqueue, &kev, td, 1);
 2173                         if (error) {
 2174                                 uma_zfree(aiolio_zone, lj);
 2175                                 return (error);
 2176                         }
 2177                 } else if (lj->lioj_signal.sigev_notify == SIGEV_NONE) {
 2178                         ;
 2179                 } else if (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL ||
 2180                            lj->lioj_signal.sigev_notify == SIGEV_THREAD_ID) {
 2181                                 if (!_SIG_VALID(lj->lioj_signal.sigev_signo)) {
 2182                                         uma_zfree(aiolio_zone, lj);
 2183                                         return EINVAL;
 2184                                 }
 2185                                 lj->lioj_flags |= LIOJ_SIGNAL;
 2186                 } else {
 2187                         uma_zfree(aiolio_zone, lj);
 2188                         return EINVAL;
 2189                 }
 2190         }
 2191 
 2192         AIO_LOCK(ki);
 2193         TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list);
 2194         /*
 2195          * Add extra aiocb count to avoid the lio to be freed
 2196          * by other threads doing aio_waitcomplete or aio_return,
 2197          * and prevent event from being sent until we have queued
 2198          * all tasks.
 2199          */
 2200         lj->lioj_count = 1;
 2201         AIO_UNLOCK(ki);
 2202 
 2203         /*
 2204          * Get pointers to the list of I/O requests.
 2205          */
 2206         nerror = 0;
 2207         for (i = 0; i < nent; i++) {
 2208                 iocb = acb_list[i];
 2209                 if (iocb != NULL) {
 2210                         error = aio_aqueue(td, iocb, lj, LIO_NOP, ops);
 2211                         if (error != 0)
 2212                                 nerror++;
 2213                 }
 2214         }
 2215 
 2216         error = 0;
 2217         AIO_LOCK(ki);
 2218         if (mode == LIO_WAIT) {
 2219                 while (lj->lioj_count - 1 != lj->lioj_finished_count) {
 2220                         ki->kaio_flags |= KAIO_WAKEUP;
 2221                         error = msleep(&p->p_aioinfo, AIO_MTX(ki),
 2222                             PRIBIO | PCATCH, "aiospn", 0);
 2223                         if (error == ERESTART)
 2224                                 error = EINTR;
 2225                         if (error)
 2226                                 break;
 2227                 }
 2228         } else {
 2229                 if (lj->lioj_count - 1 == lj->lioj_finished_count) {
 2230                         if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
 2231                                 lj->lioj_flags |= LIOJ_KEVENT_POSTED;
 2232                                 KNOTE_LOCKED(&lj->klist, 1);
 2233                         }
 2234                         if ((lj->lioj_flags & (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED))
 2235                             == LIOJ_SIGNAL
 2236                             && (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL ||
 2237                             lj->lioj_signal.sigev_notify == SIGEV_THREAD_ID)) {
 2238                                 aio_sendsig(p, &lj->lioj_signal,
 2239                                             &lj->lioj_ksi);
 2240                                 lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
 2241                         }
 2242                 }
 2243         }
 2244         lj->lioj_count--;
 2245         if (lj->lioj_count == 0) {
 2246                 TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
 2247                 knlist_delete(&lj->klist, curthread, 1);
 2248                 PROC_LOCK(p);
 2249                 sigqueue_take(&lj->lioj_ksi);
 2250                 PROC_UNLOCK(p);
 2251                 AIO_UNLOCK(ki);
 2252                 uma_zfree(aiolio_zone, lj);
 2253         } else
 2254                 AIO_UNLOCK(ki);
 2255 
 2256         if (nerror)
 2257                 return (EIO);
 2258         return (error);
 2259 }
 2260 
 2261 /* syscall - list directed I/O (REALTIME) */
 2262 int
 2263 olio_listio(struct thread *td, struct olio_listio_args *uap)
 2264 {
 2265         struct aiocb **acb_list;
 2266         struct sigevent *sigp, sig;
 2267         struct osigevent osig;
 2268         int error, nent;
 2269 
 2270         if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
 2271                 return (EINVAL);
 2272 
 2273         nent = uap->nent;
 2274         if (nent < 0 || nent > AIO_LISTIO_MAX)
 2275                 return (EINVAL);
 2276 
 2277         if (uap->sig && (uap->mode == LIO_NOWAIT)) {
 2278                 error = copyin(uap->sig, &osig, sizeof(osig));
 2279                 if (error)
 2280                         return (error);
 2281                 error = convert_old_sigevent(&osig, &sig);
 2282                 if (error)
 2283                         return (error);
 2284                 sigp = &sig;
 2285         } else
 2286                 sigp = NULL;
 2287 
 2288         acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
 2289         error = copyin(uap->acb_list, acb_list, nent * sizeof(acb_list[0]));
 2290         if (error == 0)
 2291                 error = kern_lio_listio(td, uap->mode,
 2292                     (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
 2293                     &aiocb_ops_osigevent);
 2294         free(acb_list, M_LIO);
 2295         return (error);
 2296 }
 2297 
 2298 /* syscall - list directed I/O (REALTIME) */
 2299 int
 2300 lio_listio(struct thread *td, struct lio_listio_args *uap)
 2301 {
 2302         struct aiocb **acb_list;
 2303         struct sigevent *sigp, sig;
 2304         int error, nent;
 2305 
 2306         if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
 2307                 return (EINVAL);
 2308 
 2309         nent = uap->nent;
 2310         if (nent < 0 || nent > AIO_LISTIO_MAX)
 2311                 return (EINVAL);
 2312 
 2313         if (uap->sig && (uap->mode == LIO_NOWAIT)) {
 2314                 error = copyin(uap->sig, &sig, sizeof(sig));
 2315                 if (error)
 2316                         return (error);
 2317                 sigp = &sig;
 2318         } else
 2319                 sigp = NULL;
 2320 
 2321         acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
 2322         error = copyin(uap->acb_list, acb_list, nent * sizeof(acb_list[0]));
 2323         if (error == 0)
 2324                 error = kern_lio_listio(td, uap->mode, uap->acb_list, acb_list,
 2325                     nent, sigp, &aiocb_ops);
 2326         free(acb_list, M_LIO);
 2327         return (error);
 2328 }
 2329 
 2330 /*
 2331  * Called from interrupt thread for physio, we should return as fast
 2332  * as possible, so we schedule a biohelper task.
 2333  */
 2334 static void
 2335 aio_physwakeup(struct buf *bp)
 2336 {
 2337         struct aiocblist *aiocbe;
 2338 
 2339         aiocbe = (struct aiocblist *)bp->b_caller1;
 2340         taskqueue_enqueue(taskqueue_aiod_bio, &aiocbe->biotask);
 2341 }
 2342 
 2343 /*
 2344  * Task routine to perform heavy tasks, process wakeup, and signals.
 2345  */
 2346 static void
 2347 biohelper(void *context, int pending)
 2348 {
 2349         struct aiocblist *aiocbe = context;
 2350         struct buf *bp;
 2351         struct proc *userp;
 2352         struct kaioinfo *ki;
 2353         int nblks;
 2354 
 2355         bp = aiocbe->bp;
 2356         userp = aiocbe->userproc;
 2357         ki = userp->p_aioinfo;
 2358         AIO_LOCK(ki);
 2359         aiocbe->uaiocb._aiocb_private.status -= bp->b_resid;
 2360         aiocbe->uaiocb._aiocb_private.error = 0;
 2361         if (bp->b_ioflags & BIO_ERROR)
 2362                 aiocbe->uaiocb._aiocb_private.error = bp->b_error;
 2363         nblks = btodb(aiocbe->uaiocb.aio_nbytes);
 2364         if (aiocbe->uaiocb.aio_lio_opcode == LIO_WRITE)
 2365                 aiocbe->outputcharge += nblks;
 2366         else
 2367                 aiocbe->inputcharge += nblks;
 2368         aiocbe->bp = NULL;
 2369         TAILQ_REMOVE(&userp->p_aioinfo->kaio_bufqueue, aiocbe, plist);
 2370         ki->kaio_buffer_count--;
 2371         aio_bio_done_notify(userp, aiocbe, DONE_BUF);
 2372         AIO_UNLOCK(ki);
 2373 
 2374         /* Release mapping into kernel space. */
 2375         vunmapbuf(bp);
 2376         relpbuf(bp, NULL);
 2377         atomic_subtract_int(&num_buf_aio, 1);
 2378 }
 2379 
 2380 /* syscall - wait for the next completion of an aio request */
 2381 static int
 2382 kern_aio_waitcomplete(struct thread *td, struct aiocb **aiocbp,
 2383     struct timespec *ts, struct aiocb_ops *ops)
 2384 {
 2385         struct proc *p = td->td_proc;
 2386         struct timeval atv;
 2387         struct kaioinfo *ki;
 2388         struct aiocblist *cb;
 2389         struct aiocb *uuaiocb;
 2390         int error, status, timo;
 2391 
 2392         ops->store_aiocb(aiocbp, NULL);
 2393 
 2394         timo = 0;
 2395         if (ts) {
 2396                 if ((ts->tv_nsec < 0) || (ts->tv_nsec >= 1000000000))
 2397                         return (EINVAL);
 2398 
 2399                 TIMESPEC_TO_TIMEVAL(&atv, ts);
 2400                 if (itimerfix(&atv))
 2401                         return (EINVAL);
 2402                 timo = tvtohz(&atv);
 2403         }
 2404 
 2405         if (p->p_aioinfo == NULL)
 2406                 aio_init_aioinfo(p);
 2407         ki = p->p_aioinfo;
 2408 
 2409         error = 0;
 2410         cb = NULL;
 2411         AIO_LOCK(ki);
 2412         while ((cb = TAILQ_FIRST(&ki->kaio_done)) == NULL) {
 2413                 ki->kaio_flags |= KAIO_WAKEUP;
 2414                 error = msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO | PCATCH,
 2415                     "aiowc", timo);
 2416                 if (timo && error == ERESTART)
 2417                         error = EINTR;
 2418                 if (error)
 2419                         break;
 2420         }
 2421 
 2422         if (cb != NULL) {
 2423                 MPASS(cb->jobstate == JOBST_JOBFINISHED);
 2424                 uuaiocb = cb->uuaiocb;
 2425                 status = cb->uaiocb._aiocb_private.status;
 2426                 error = cb->uaiocb._aiocb_private.error;
 2427                 td->td_retval[0] = status;
 2428                 if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
 2429                         td->td_ru.ru_oublock += cb->outputcharge;
 2430                         cb->outputcharge = 0;
 2431                 } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
 2432                         td->td_ru.ru_inblock += cb->inputcharge;
 2433                         cb->inputcharge = 0;
 2434                 }
 2435                 aio_free_entry(cb);
 2436                 AIO_UNLOCK(ki);
 2437                 ops->store_aiocb(aiocbp, uuaiocb);
 2438                 ops->store_error(uuaiocb, error);
 2439                 ops->store_status(uuaiocb, status);
 2440         } else
 2441                 AIO_UNLOCK(ki);
 2442 
 2443         return (error);
 2444 }
 2445 
 2446 int
 2447 aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
 2448 {
 2449         struct timespec ts, *tsp;
 2450         int error;
 2451 
 2452         if (uap->timeout) {
 2453                 /* Get timespec struct. */
 2454                 error = copyin(uap->timeout, &ts, sizeof(ts));
 2455                 if (error)
 2456                         return (error);
 2457                 tsp = &ts;
 2458         } else
 2459                 tsp = NULL;
 2460 
 2461         return (kern_aio_waitcomplete(td, uap->aiocbp, tsp, &aiocb_ops));
 2462 }
 2463 
 2464 static int
 2465 kern_aio_fsync(struct thread *td, int op, struct aiocb *aiocbp,
 2466     struct aiocb_ops *ops)
 2467 {
 2468         struct proc *p = td->td_proc;
 2469         struct kaioinfo *ki;
 2470 
 2471         if (op != O_SYNC) /* XXX lack of O_DSYNC */
 2472                 return (EINVAL);
 2473         ki = p->p_aioinfo;
 2474         if (ki == NULL)
 2475                 aio_init_aioinfo(p);
 2476         return (aio_aqueue(td, aiocbp, NULL, LIO_SYNC, ops));
 2477 }
 2478 
 2479 int
 2480 aio_fsync(struct thread *td, struct aio_fsync_args *uap)
 2481 {
 2482 
 2483         return (kern_aio_fsync(td, uap->op, uap->aiocbp, &aiocb_ops));
 2484 }
 2485 
 2486 /* kqueue attach function */
 2487 static int
 2488 filt_aioattach(struct knote *kn)
 2489 {
 2490         struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
 2491 
 2492         /*
 2493          * The aiocbe pointer must be validated before using it, so
 2494          * registration is restricted to the kernel; the user cannot
 2495          * set EV_FLAG1.
 2496          */
 2497         if ((kn->kn_flags & EV_FLAG1) == 0)
 2498                 return (EPERM);
 2499         kn->kn_ptr.p_aio = aiocbe;
 2500         kn->kn_flags &= ~EV_FLAG1;
 2501 
 2502         knlist_add(&aiocbe->klist, kn, 0);
 2503 
 2504         return (0);
 2505 }
 2506 
 2507 /* kqueue detach function */
 2508 static void
 2509 filt_aiodetach(struct knote *kn)
 2510 {
 2511         struct aiocblist *aiocbe = kn->kn_ptr.p_aio;
 2512 
 2513         if (!knlist_empty(&aiocbe->klist))
 2514                 knlist_remove(&aiocbe->klist, kn, 0);
 2515 }
 2516 
 2517 /* kqueue filter function */
 2518 /*ARGSUSED*/
 2519 static int
 2520 filt_aio(struct knote *kn, long hint)
 2521 {
 2522         struct aiocblist *aiocbe = kn->kn_ptr.p_aio;
 2523 
 2524         kn->kn_data = aiocbe->uaiocb._aiocb_private.error;
 2525         if (aiocbe->jobstate != JOBST_JOBFINISHED)
 2526                 return (0);
 2527         kn->kn_flags |= EV_EOF;
 2528         return (1);
 2529 }
 2530 
 2531 /* kqueue attach function */
 2532 static int
 2533 filt_lioattach(struct knote *kn)
 2534 {
 2535         struct aioliojob * lj = (struct aioliojob *)kn->kn_sdata;
 2536 
 2537         /*
 2538          * The aioliojob pointer must be validated before using it, so
 2539          * registration is restricted to the kernel; the user cannot
 2540          * set EV_FLAG1.
 2541          */
 2542         if ((kn->kn_flags & EV_FLAG1) == 0)
 2543                 return (EPERM);
 2544         kn->kn_ptr.p_lio = lj;
 2545         kn->kn_flags &= ~EV_FLAG1;
 2546 
 2547         knlist_add(&lj->klist, kn, 0);
 2548 
 2549         return (0);
 2550 }
 2551 
 2552 /* kqueue detach function */
 2553 static void
 2554 filt_liodetach(struct knote *kn)
 2555 {
 2556         struct aioliojob * lj = kn->kn_ptr.p_lio;
 2557 
 2558         if (!knlist_empty(&lj->klist))
 2559                 knlist_remove(&lj->klist, kn, 0);
 2560 }
 2561 
 2562 /* kqueue filter function */
 2563 /*ARGSUSED*/
 2564 static int
 2565 filt_lio(struct knote *kn, long hint)
 2566 {
 2567         struct aioliojob * lj = kn->kn_ptr.p_lio;
 2568 
 2569         return (lj->lioj_flags & LIOJ_KEVENT_POSTED);
 2570 }
 2571 
 2572 #ifdef COMPAT_FREEBSD32
 2573 
 2574 struct __aiocb_private32 {
 2575         int32_t status;
 2576         int32_t error;
 2577         uint32_t kernelinfo;
 2578 };
 2579 
 2580 typedef struct oaiocb32 {
 2581         int     aio_fildes;             /* File descriptor */
 2582         uint64_t aio_offset __packed;   /* File offset for I/O */
 2583         uint32_t aio_buf;               /* I/O buffer in process space */
 2584         uint32_t aio_nbytes;            /* Number of bytes for I/O */
 2585         struct  osigevent32 aio_sigevent; /* Signal to deliver */
 2586         int     aio_lio_opcode;         /* LIO opcode */
 2587         int     aio_reqprio;            /* Request priority -- ignored */
 2588         struct  __aiocb_private32 _aiocb_private;
 2589 } oaiocb32_t;
 2590 
 2591 typedef struct aiocb32 {
 2592         int32_t aio_fildes;             /* File descriptor */
 2593         uint64_t aio_offset __packed;   /* File offset for I/O */
 2594         uint32_t aio_buf;               /* I/O buffer in process space */
 2595         uint32_t aio_nbytes;            /* Number of bytes for I/O */
 2596         int     __spare__[2];
 2597         uint32_t __spare2__;
 2598         int     aio_lio_opcode;         /* LIO opcode */
 2599         int     aio_reqprio;            /* Request priority -- ignored */
 2600         struct __aiocb_private32 _aiocb_private;
 2601         struct sigevent32 aio_sigevent; /* Signal to deliver */
 2602 } aiocb32_t;
 2603 
 2604 static int
 2605 convert_old_sigevent32(struct osigevent32 *osig, struct sigevent *nsig)
 2606 {
 2607 
 2608         /*
 2609          * Only SIGEV_NONE, SIGEV_SIGNAL, and SIGEV_KEVENT are
 2610          * supported by AIO with the old sigevent structure.
 2611          */
 2612         CP(*osig, *nsig, sigev_notify);
 2613         switch (nsig->sigev_notify) {
 2614         case SIGEV_NONE:
 2615                 break;
 2616         case SIGEV_SIGNAL:
 2617                 nsig->sigev_signo = osig->__sigev_u.__sigev_signo;
 2618                 break;
 2619         case SIGEV_KEVENT:
 2620                 nsig->sigev_notify_kqueue =
 2621                     osig->__sigev_u.__sigev_notify_kqueue;
 2622                 PTRIN_CP(*osig, *nsig, sigev_value.sival_ptr);
 2623                 break;
 2624         default:
 2625                 return (EINVAL);
 2626         }
 2627         return (0);
 2628 }
 2629 
 2630 static int
 2631 aiocb32_copyin_old_sigevent(struct aiocb *ujob, struct aiocb *kjob)
 2632 {
 2633         struct oaiocb32 job32;
 2634         int error;
 2635 
 2636         bzero(kjob, sizeof(struct aiocb));
 2637         error = copyin(ujob, &job32, sizeof(job32));
 2638         if (error)
 2639                 return (error);
 2640 
 2641         CP(job32, *kjob, aio_fildes);
 2642         CP(job32, *kjob, aio_offset);
 2643         PTRIN_CP(job32, *kjob, aio_buf);
 2644         CP(job32, *kjob, aio_nbytes);
 2645         CP(job32, *kjob, aio_lio_opcode);
 2646         CP(job32, *kjob, aio_reqprio);
 2647         CP(job32, *kjob, _aiocb_private.status);
 2648         CP(job32, *kjob, _aiocb_private.error);
 2649         PTRIN_CP(job32, *kjob, _aiocb_private.kernelinfo);
 2650         return (convert_old_sigevent32(&job32.aio_sigevent,
 2651             &kjob->aio_sigevent));
 2652 }
 2653 
 2654 static int
 2655 convert_sigevent32(struct sigevent32 *sig32, struct sigevent *sig)
 2656 {
 2657 
 2658         CP(*sig32, *sig, sigev_notify);
 2659         switch (sig->sigev_notify) {
 2660         case SIGEV_NONE:
 2661                 break;
 2662         case SIGEV_THREAD_ID:
 2663                 CP(*sig32, *sig, sigev_notify_thread_id);
 2664                 /* FALLTHROUGH */
 2665         case SIGEV_SIGNAL:
 2666                 CP(*sig32, *sig, sigev_signo);
 2667                 break;
 2668         case SIGEV_KEVENT:
 2669                 CP(*sig32, *sig, sigev_notify_kqueue);
 2670                 PTRIN_CP(*sig32, *sig, sigev_value.sival_ptr);
 2671                 break;
 2672         default:
 2673                 return (EINVAL);
 2674         }
 2675         return (0);
 2676 }
 2677 
 2678 static int
 2679 aiocb32_copyin(struct aiocb *ujob, struct aiocb *kjob)
 2680 {
 2681         struct aiocb32 job32;
 2682         int error;
 2683 
 2684         error = copyin(ujob, &job32, sizeof(job32));
 2685         if (error)
 2686                 return (error);
 2687         CP(job32, *kjob, aio_fildes);
 2688         CP(job32, *kjob, aio_offset);
 2689         PTRIN_CP(job32, *kjob, aio_buf);
 2690         CP(job32, *kjob, aio_nbytes);
 2691         CP(job32, *kjob, aio_lio_opcode);
 2692         CP(job32, *kjob, aio_reqprio);
 2693         CP(job32, *kjob, _aiocb_private.status);
 2694         CP(job32, *kjob, _aiocb_private.error);
 2695         PTRIN_CP(job32, *kjob, _aiocb_private.kernelinfo);
 2696         return (convert_sigevent32(&job32.aio_sigevent, &kjob->aio_sigevent));
 2697 }
 2698 
 2699 static long
 2700 aiocb32_fetch_status(struct aiocb *ujob)
 2701 {
 2702         struct aiocb32 *ujob32;
 2703 
 2704         ujob32 = (struct aiocb32 *)ujob;
 2705         return (fuword32(&ujob32->_aiocb_private.status));
 2706 }
 2707 
 2708 static long
 2709 aiocb32_fetch_error(struct aiocb *ujob)
 2710 {
 2711         struct aiocb32 *ujob32;
 2712 
 2713         ujob32 = (struct aiocb32 *)ujob;
 2714         return (fuword32(&ujob32->_aiocb_private.error));
 2715 }
 2716 
 2717 static int
 2718 aiocb32_store_status(struct aiocb *ujob, long status)
 2719 {
 2720         struct aiocb32 *ujob32;
 2721 
 2722         ujob32 = (struct aiocb32 *)ujob;
 2723         return (suword32(&ujob32->_aiocb_private.status, status));
 2724 }
 2725 
 2726 static int
 2727 aiocb32_store_error(struct aiocb *ujob, long error)
 2728 {
 2729         struct aiocb32 *ujob32;
 2730 
 2731         ujob32 = (struct aiocb32 *)ujob;
 2732         return (suword32(&ujob32->_aiocb_private.error, error));
 2733 }
 2734 
 2735 static int
 2736 aiocb32_store_kernelinfo(struct aiocb *ujob, long jobref)
 2737 {
 2738         struct aiocb32 *ujob32;
 2739 
 2740         ujob32 = (struct aiocb32 *)ujob;
 2741         return (suword32(&ujob32->_aiocb_private.kernelinfo, jobref));
 2742 }
 2743 
 2744 static int
 2745 aiocb32_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob)
 2746 {
 2747 
 2748         return (suword32(ujobp, (long)ujob));
 2749 }
 2750 
 2751 static struct aiocb_ops aiocb32_ops = {
 2752         .copyin = aiocb32_copyin,
 2753         .fetch_status = aiocb32_fetch_status,
 2754         .fetch_error = aiocb32_fetch_error,
 2755         .store_status = aiocb32_store_status,
 2756         .store_error = aiocb32_store_error,
 2757         .store_kernelinfo = aiocb32_store_kernelinfo,
 2758         .store_aiocb = aiocb32_store_aiocb,
 2759 };
 2760 
 2761 static struct aiocb_ops aiocb32_ops_osigevent = {
 2762         .copyin = aiocb32_copyin_old_sigevent,
 2763         .fetch_status = aiocb32_fetch_status,
 2764         .fetch_error = aiocb32_fetch_error,
 2765         .store_status = aiocb32_store_status,
 2766         .store_error = aiocb32_store_error,
 2767         .store_kernelinfo = aiocb32_store_kernelinfo,
 2768         .store_aiocb = aiocb32_store_aiocb,
 2769 };
 2770 
 2771 int
 2772 freebsd32_aio_return(struct thread *td, struct freebsd32_aio_return_args *uap)
 2773 {
 2774 
 2775         return (kern_aio_return(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops));
 2776 }
 2777 
 2778 int
 2779 freebsd32_aio_suspend(struct thread *td, struct freebsd32_aio_suspend_args *uap)
 2780 {
 2781         struct timespec32 ts32;
 2782         struct timespec ts, *tsp;
 2783         struct aiocb **ujoblist;
 2784         uint32_t *ujoblist32;
 2785         int error, i;
 2786 
 2787         if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
 2788                 return (EINVAL);
 2789 
 2790         if (uap->timeout) {
 2791                 /* Get timespec struct. */
 2792                 if ((error = copyin(uap->timeout, &ts32, sizeof(ts32))) != 0)
 2793                         return (error);
 2794                 CP(ts32, ts, tv_sec);
 2795                 CP(ts32, ts, tv_nsec);
 2796                 tsp = &ts;
 2797         } else
 2798                 tsp = NULL;
 2799 
 2800         ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
 2801         ujoblist32 = (uint32_t *)ujoblist;
 2802         error = copyin(uap->aiocbp, ujoblist32, uap->nent *
 2803             sizeof(ujoblist32[0]));
 2804         if (error == 0) {
 2805                 for (i = uap->nent; i > 0; i--)
 2806                         ujoblist[i] = PTRIN(ujoblist32[i]);
 2807 
 2808                 error = kern_aio_suspend(td, uap->nent, ujoblist, tsp);
 2809         }
 2810         uma_zfree(aiol_zone, ujoblist);
 2811         return (error);
 2812 }
 2813 
 2814 int
 2815 freebsd32_aio_cancel(struct thread *td, struct freebsd32_aio_cancel_args *uap)
 2816 {
 2817 
 2818         return (aio_cancel(td, (struct aio_cancel_args *)uap));
 2819 }
 2820 
 2821 int
 2822 freebsd32_aio_error(struct thread *td, struct freebsd32_aio_error_args *uap)
 2823 {
 2824 
 2825         return (kern_aio_error(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops));
 2826 }
 2827 
 2828 int
 2829 freebsd32_oaio_read(struct thread *td, struct freebsd32_oaio_read_args *uap)
 2830 {
 2831 
 2832         return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
 2833             &aiocb32_ops_osigevent));
 2834 }
 2835 
 2836 int
 2837 freebsd32_aio_read(struct thread *td, struct freebsd32_aio_read_args *uap)
 2838 {
 2839 
 2840         return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
 2841             &aiocb32_ops));
 2842 }
 2843 
 2844 int
 2845 freebsd32_oaio_write(struct thread *td, struct freebsd32_oaio_write_args *uap)
 2846 {
 2847 
 2848         return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
 2849             &aiocb32_ops_osigevent));
 2850 }
 2851 
 2852 int
 2853 freebsd32_aio_write(struct thread *td, struct freebsd32_aio_write_args *uap)
 2854 {
 2855 
 2856         return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
 2857             &aiocb32_ops));
 2858 }
 2859 
 2860 int
 2861 freebsd32_aio_waitcomplete(struct thread *td,
 2862     struct freebsd32_aio_waitcomplete_args *uap)
 2863 {
 2864         struct timespec32 ts32;
 2865         struct timespec ts, *tsp;
 2866         int error;
 2867 
 2868         if (uap->timeout) {
 2869                 /* Get timespec struct. */
 2870                 error = copyin(uap->timeout, &ts32, sizeof(ts32));
 2871                 if (error)
 2872                         return (error);
 2873                 CP(ts32, ts, tv_sec);
 2874                 CP(ts32, ts, tv_nsec);
 2875                 tsp = &ts;
 2876         } else
 2877                 tsp = NULL;
 2878 
 2879         return (kern_aio_waitcomplete(td, (struct aiocb **)uap->aiocbp, tsp,
 2880             &aiocb32_ops));
 2881 }
 2882 
 2883 int
 2884 freebsd32_aio_fsync(struct thread *td, struct freebsd32_aio_fsync_args *uap)
 2885 {
 2886 
 2887         return (kern_aio_fsync(td, uap->op, (struct aiocb *)uap->aiocbp,
 2888             &aiocb32_ops));
 2889 }
 2890 
 2891 int
 2892 freebsd32_olio_listio(struct thread *td, struct freebsd32_olio_listio_args *uap)
 2893 {
 2894         struct aiocb **acb_list;
 2895         struct sigevent *sigp, sig;
 2896         struct osigevent32 osig;
 2897         uint32_t *acb_list32;
 2898         int error, i, nent;
 2899 
 2900         if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
 2901                 return (EINVAL);
 2902 
 2903         nent = uap->nent;
 2904         if (nent < 0 || nent > AIO_LISTIO_MAX)
 2905                 return (EINVAL);
 2906 
 2907         if (uap->sig && (uap->mode == LIO_NOWAIT)) {
 2908                 error = copyin(uap->sig, &osig, sizeof(osig));
 2909                 if (error)
 2910                         return (error);
 2911                 error = convert_old_sigevent32(&osig, &sig);
 2912                 if (error)
 2913                         return (error);
 2914                 sigp = &sig;
 2915         } else
 2916                 sigp = NULL;
 2917 
 2918         acb_list32 = malloc(sizeof(uint32_t) * nent, M_LIO, M_WAITOK);
 2919         error = copyin(uap->acb_list, acb_list32, nent * sizeof(uint32_t));
 2920         if (error) {
 2921                 free(acb_list32, M_LIO);
 2922                 return (error);
 2923         }
 2924         acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
 2925         for (i = 0; i < nent; i++)
 2926                 acb_list[i] = PTRIN(acb_list32[i]);
 2927         free(acb_list32, M_LIO);
 2928 
 2929         error = kern_lio_listio(td, uap->mode,
 2930             (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
 2931             &aiocb32_ops_osigevent);
 2932         free(acb_list, M_LIO);
 2933         return (error);
 2934 }
 2935 
 2936 int
 2937 freebsd32_lio_listio(struct thread *td, struct freebsd32_lio_listio_args *uap)
 2938 {
 2939         struct aiocb **acb_list;
 2940         struct sigevent *sigp, sig;
 2941         struct sigevent32 sig32;
 2942         uint32_t *acb_list32;
 2943         int error, i, nent;
 2944 
 2945         if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
 2946                 return (EINVAL);
 2947 
 2948         nent = uap->nent;
 2949         if (nent < 0 || nent > AIO_LISTIO_MAX)
 2950                 return (EINVAL);
 2951 
 2952         if (uap->sig && (uap->mode == LIO_NOWAIT)) {
 2953                 error = copyin(uap->sig, &sig32, sizeof(sig32));
 2954                 if (error)
 2955                         return (error);
 2956                 error = convert_sigevent32(&sig32, &sig);
 2957                 if (error)
 2958                         return (error);
 2959                 sigp = &sig;
 2960         } else
 2961                 sigp = NULL;
 2962 
 2963         acb_list32 = malloc(sizeof(uint32_t) * nent, M_LIO, M_WAITOK);
 2964         error = copyin(uap->acb_list, acb_list32, nent * sizeof(uint32_t));
 2965         if (error) {
 2966                 free(acb_list32, M_LIO);
 2967                 return (error);
 2968         }
 2969         acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
 2970         for (i = 0; i < nent; i++)
 2971                 acb_list[i] = PTRIN(acb_list32[i]);
 2972         free(acb_list32, M_LIO);
 2973 
 2974         error = kern_lio_listio(td, uap->mode,
 2975             (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
 2976             &aiocb32_ops);
 2977         free(acb_list, M_LIO);
 2978         return (error);
 2979 }
 2980 
 2981 #endif

Cache object: cc6b08723a903636873d4976e96ea5c3


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.