The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/geom/bde/g_bde_work.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2002 Poul-Henning Kamp
    3  * Copyright (c) 2002 Networks Associates Technology, Inc.
    4  * All rights reserved.
    5  *
    6  * This software was developed for the FreeBSD Project by Poul-Henning Kamp
    7  * and NAI Labs, the Security Research Division of Network Associates, Inc.
    8  * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
    9  * DARPA CHATS research program.
   10  *
   11  * Redistribution and use in source and binary forms, with or without
   12  * modification, are permitted provided that the following conditions
   13  * are met:
   14  * 1. Redistributions of source code must retain the above copyright
   15  *    notice, this list of conditions and the following disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  * $FreeBSD: releng/10.3/sys/geom/bde/g_bde_work.c 172836 2007-10-20 23:23:23Z julian $
   33  */
   34 /*
   35  * This source file contains the state-engine which makes things happen in the
   36  * right order.
   37  *
   38  * Outline:
   39  *   1) g_bde_start1()
   40  *      Break the struct bio into multiple work packets one per zone.
   41  *   2) g_bde_start2()
   42  *      Setup the necessary sector buffers and start those read operations
   43  *      which we can start at this time and put the item on the work-list.
   44  *   3) g_bde_worker()
   45  *      Scan the work-list for items which are ready for crypto processing
   46  *      and call the matching crypto function in g_bde_crypt.c and schedule
   47  *      any writes needed.  Read operations finish here by releasing the
   48  *      sector buffers and delivering the original bio request.
   49  *   4) g_bde_write_done()
   50  *      Release sector buffers and deliver the original bio request.
   51  *
   52  * Because of the C-scope rules, the functions are almost perfectly in the
   53  * opposite order in this source file.
   54  *
   55  * XXX: A switch to the hardware assisted crypto in src/sys/opencrypto will add
   56  * XXX: additional states to this state-engine.  Since no hardware available
   57  * XXX: at this time has AES support, implementing this has been postponed
   58  * XXX: until such time as it would result in a benefit.
   59  */
   60 
   61 #include <sys/param.h>
   62 #include <sys/bio.h>
   63 #include <sys/lock.h>
   64 #include <sys/mutex.h>
   65 #include <sys/queue.h>
   66 #include <sys/malloc.h>
   67 #include <sys/systm.h>
   68 #include <sys/kernel.h>
   69 #include <sys/sysctl.h>
   70 #include <sys/proc.h>
   71 #include <sys/kthread.h>
   72 
   73 #include <crypto/rijndael/rijndael-api-fst.h>
   74 #include <crypto/sha2/sha2.h>
   75 #include <geom/geom.h>
   76 #include <geom/bde/g_bde.h>
   77 
   78 static void g_bde_delete_sector(struct g_bde_softc *wp, struct g_bde_sector *sp);
   79 static struct g_bde_sector * g_bde_new_sector(struct g_bde_work *wp, u_int len);
   80 static void g_bde_release_keysector(struct g_bde_work *wp);
   81 static struct g_bde_sector *g_bde_get_keysector(struct g_bde_work *wp);
   82 static int g_bde_start_read(struct g_bde_sector *sp);
   83 static void g_bde_purge_sector(struct g_bde_softc *sc, int fraction);
   84 
   85 /*
   86  * Work item allocation.
   87  *
   88  * C++ would call these constructors and destructors.
   89  */
   90 static u_int g_bde_nwork;
   91 SYSCTL_UINT(_debug, OID_AUTO, gbde_nwork, CTLFLAG_RD, &g_bde_nwork, 0, "");
   92 
   93 static MALLOC_DEFINE(M_GBDE, "gbde", "GBDE data structures");
   94 
   95 static struct g_bde_work *
   96 g_bde_new_work(struct g_bde_softc *sc)
   97 {
   98         struct g_bde_work *wp;
   99 
  100         wp = malloc(sizeof *wp, M_GBDE, M_NOWAIT | M_ZERO);
  101         if (wp == NULL)
  102                 return (wp);
  103         wp->state = SETUP;
  104         wp->softc = sc;
  105         g_bde_nwork++;
  106         sc->nwork++;
  107         TAILQ_INSERT_TAIL(&sc->worklist, wp, list);
  108         return (wp);
  109 }
  110 
  111 static void
  112 g_bde_delete_work(struct g_bde_work *wp)
  113 {
  114         struct g_bde_softc *sc;
  115 
  116         sc = wp->softc;
  117         g_bde_nwork--;
  118         sc->nwork--;
  119         TAILQ_REMOVE(&sc->worklist, wp, list);
  120         free(wp, M_GBDE);
  121 }
  122 
  123 /*
  124  * Sector buffer allocation
  125  *
  126  * These two functions allocate and free back variable sized sector buffers
  127  */
  128 
  129 static u_int g_bde_nsect;
  130 SYSCTL_UINT(_debug, OID_AUTO, gbde_nsect, CTLFLAG_RD, &g_bde_nsect, 0, "");
  131 
  132 static void
  133 g_bde_delete_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
  134 {
  135 
  136         g_bde_nsect--;
  137         sc->nsect--;
  138         if (sp->malloc)
  139                 free(sp->data, M_GBDE);
  140         free(sp, M_GBDE);
  141 }
  142 
  143 static struct g_bde_sector *
  144 g_bde_new_sector(struct g_bde_work *wp, u_int len)
  145 {
  146         struct g_bde_sector *sp;
  147 
  148         sp = malloc(sizeof *sp, M_GBDE, M_NOWAIT | M_ZERO);
  149         if (sp == NULL)
  150                 return (sp);
  151         if (len > 0) {
  152                 sp->data = malloc(len, M_GBDE, M_NOWAIT | M_ZERO);
  153                 if (sp->data == NULL) {
  154                         free(sp, M_GBDE);
  155                         return (NULL);
  156                 }
  157                 sp->malloc = 1;
  158         }
  159         g_bde_nsect++;
  160         wp->softc->nsect++;
  161         sp->size = len;
  162         sp->softc = wp->softc;
  163         sp->ref = 1;
  164         sp->owner = wp;
  165         sp->offset = wp->so;
  166         sp->state = JUNK;
  167         return (sp);
  168 }
  169 
  170 /*
  171  * Skey sector cache.
  172  *
  173  * Nothing prevents two separate I/O requests from addressing the same zone
  174  * and thereby needing the same skey sector.  We therefore need to sequence
  175  * I/O operations to the skey sectors.  A certain amount of caching is also
  176  * desirable, although the extent of benefit from this is not at this point
  177  * determined.
  178  *
  179  * XXX: GEOM may be able to grow a generic caching facility at some point
  180  * XXX: to support such needs.
  181  */
  182 
  183 static u_int g_bde_ncache;
  184 SYSCTL_UINT(_debug, OID_AUTO, gbde_ncache, CTLFLAG_RD, &g_bde_ncache, 0, "");
  185 
  186 static void
  187 g_bde_purge_one_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
  188 {
  189 
  190         g_trace(G_T_TOPOLOGY, "g_bde_purge_one_sector(%p, %p)", sc, sp);
  191         if (sp->ref != 0)
  192                 return;
  193         TAILQ_REMOVE(&sc->freelist, sp, list);
  194         g_bde_ncache--;
  195         sc->ncache--;
  196         bzero(sp->data, sp->size);
  197         g_bde_delete_sector(sc, sp);
  198 }
  199 
  200 static struct g_bde_sector *
  201 g_bde_get_keysector(struct g_bde_work *wp)
  202 {
  203         struct g_bde_sector *sp;
  204         struct g_bde_softc *sc;
  205         off_t offset;
  206 
  207         offset = wp->kso;
  208         g_trace(G_T_TOPOLOGY, "g_bde_get_keysector(%p, %jd)", wp, (intmax_t)offset);
  209         sc = wp->softc;
  210 
  211         if (malloc_last_fail() < g_bde_ncache)
  212                 g_bde_purge_sector(sc, -1);
  213 
  214         sp = TAILQ_FIRST(&sc->freelist);
  215         if (sp != NULL && sp->ref == 0 && sp->used + 300 < time_uptime)
  216                 g_bde_purge_one_sector(sc, sp);
  217 
  218         TAILQ_FOREACH(sp, &sc->freelist, list) {
  219                 if (sp->offset == offset)
  220                         break;
  221         }
  222         if (sp != NULL) {
  223                 sp->ref++;
  224                 KASSERT(sp->offset == offset, ("wrong offset"));
  225                 KASSERT(sp->softc == wp->softc, ("wrong softc"));
  226                 if (sp->ref == 1)
  227                         sp->owner = wp;
  228         } else {
  229                 if (malloc_last_fail() < g_bde_ncache) {
  230                         TAILQ_FOREACH(sp, &sc->freelist, list)
  231                                 if (sp->ref == 0)
  232                                         break;
  233                 }
  234                 if (sp == NULL && !TAILQ_EMPTY(&sc->freelist))
  235                         sp = TAILQ_FIRST(&sc->freelist);
  236                 if (sp != NULL && sp->ref > 0)
  237                         sp = NULL;
  238                 if (sp == NULL) {
  239                         sp = g_bde_new_sector(wp, sc->sectorsize);
  240                         if (sp != NULL) {
  241                                 g_bde_ncache++;
  242                                 sc->ncache++;
  243                                 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
  244                                 sp->malloc = 2;
  245                         }
  246                 }
  247                 if (sp != NULL) {
  248                         sp->offset = offset;
  249                         sp->softc = wp->softc;
  250                         sp->ref = 1;
  251                         sp->owner = wp;
  252                         sp->state = JUNK;
  253                         sp->error = 0;
  254                 }
  255         }
  256         if (sp != NULL) {
  257                 TAILQ_REMOVE(&sc->freelist, sp, list);
  258                 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
  259                 sp->used = time_uptime;
  260         }
  261         wp->ksp = sp;
  262         return(sp);
  263 }
  264 
  265 static void
  266 g_bde_release_keysector(struct g_bde_work *wp)
  267 {
  268         struct g_bde_softc *sc;
  269         struct g_bde_work *wp2;
  270         struct g_bde_sector *sp;
  271 
  272         sp = wp->ksp;
  273         g_trace(G_T_TOPOLOGY, "g_bde_release_keysector(%p)", sp);
  274         KASSERT(sp->malloc == 2, ("Wrong sector released"));
  275         sc = sp->softc;
  276         KASSERT(sc != NULL, ("NULL sp->softc"));
  277         KASSERT(wp == sp->owner, ("Releasing, not owner"));
  278         sp->owner = NULL;
  279         wp->ksp = NULL;
  280         sp->ref--;
  281         if (sp->ref > 0) {
  282                 TAILQ_REMOVE(&sc->freelist, sp, list);
  283                 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
  284                 TAILQ_FOREACH(wp2, &sc->worklist, list) {
  285                         if (wp2->ksp == sp) {
  286                                 KASSERT(wp2 != wp, ("Self-reowning"));
  287                                 sp->owner = wp2;
  288                                 wakeup(sp->softc);
  289                                 break;
  290                         }
  291                 }
  292                 KASSERT(wp2 != NULL, ("Failed to pick up owner for %p\n", sp));
  293         } else if (sp->error != 0) {
  294                 sp->offset = ~0;
  295                 sp->error = 0;
  296                 sp->state = JUNK;
  297         }
  298         TAILQ_REMOVE(&sc->freelist, sp, list);
  299         TAILQ_INSERT_HEAD(&sc->freelist, sp, list);
  300 }
  301 
  302 static void
  303 g_bde_purge_sector(struct g_bde_softc *sc, int fraction)
  304 {
  305         struct g_bde_sector *sp;
  306         int n;
  307 
  308         g_trace(G_T_TOPOLOGY, "g_bde_purge_sector(%p)", sc);
  309         if (fraction > 0)
  310                 n = sc->ncache / fraction + 1;
  311         else 
  312                 n = g_bde_ncache - malloc_last_fail();
  313         if (n < 0)
  314                 return;
  315         if (n > sc->ncache)
  316                 n = sc->ncache;
  317         while(n--) {
  318                 TAILQ_FOREACH(sp, &sc->freelist, list) {
  319                         if (sp->ref != 0)
  320                                 continue;
  321                         TAILQ_REMOVE(&sc->freelist, sp, list);
  322                         g_bde_ncache--;
  323                         sc->ncache--;
  324                         bzero(sp->data, sp->size);
  325                         g_bde_delete_sector(sc, sp);
  326                         break;
  327                 }
  328         }
  329 }
  330 
  331 static struct g_bde_sector *
  332 g_bde_read_keysector(struct g_bde_softc *sc, struct g_bde_work *wp)
  333 {
  334         struct g_bde_sector *sp;
  335 
  336         g_trace(G_T_TOPOLOGY, "g_bde_read_keysector(%p)", wp);
  337         sp = g_bde_get_keysector(wp);
  338         if (sp == NULL) {
  339                 g_bde_purge_sector(sc, -1);
  340                 sp = g_bde_get_keysector(wp);
  341         }
  342         if (sp == NULL)
  343                 return (sp);
  344         if (sp->owner != wp)
  345                 return (sp);
  346         if (sp->state == VALID)
  347                 return (sp);
  348         if (g_bde_start_read(sp) == 0)
  349                 return (sp);
  350         g_bde_release_keysector(wp);
  351         return (NULL);
  352 }
  353 
  354 /*
  355  * Contribute to the completion of the original bio request.
  356  *
  357  * We have no simple way to tell how many bits the original bio request has
  358  * been segmented into, so the easiest way to determine when we can deliver
  359  * it is to keep track of the number of bytes we have completed.  We keep
  360  * track of any errors underway and latch onto the first one.
  361  *
  362  * We always report "nothing done" in case of error, because random bits here
  363  * and there may be completed and returning a number of completed bytes does
  364  * not convey any useful information about which bytes they were.  If some
  365  * piece of broken code somewhere interprets this to mean that nothing has
  366  * changed on the underlying media they deserve the lossage headed for them.
  367  *
  368  * A single mutex per g_bde instance is used to prevent contention.
  369  */
  370 
  371 static void
  372 g_bde_contribute(struct bio *bp, off_t bytes, int error)
  373 {
  374 
  375         g_trace(G_T_TOPOLOGY, "g_bde_contribute bp %p bytes %jd error %d",
  376              bp, (intmax_t)bytes, error);
  377         if (bp->bio_error == 0)
  378                 bp->bio_error = error;
  379         bp->bio_completed += bytes;
  380         KASSERT(bp->bio_completed <= bp->bio_length, ("Too large contribution"));
  381         if (bp->bio_completed == bp->bio_length) {
  382                 if (bp->bio_error != 0)
  383                         bp->bio_completed = 0;
  384                 g_io_deliver(bp, bp->bio_error);
  385         }
  386 }
  387 
  388 /*
  389  * This is the common case "we're done with this work package" function
  390  */
  391 
  392 static void
  393 g_bde_work_done(struct g_bde_work *wp, int error)
  394 {
  395 
  396         g_bde_contribute(wp->bp, wp->length, error);
  397         if (wp->sp != NULL)
  398                 g_bde_delete_sector(wp->softc, wp->sp);
  399         if (wp->ksp != NULL)
  400                 g_bde_release_keysector(wp);
  401         g_bde_delete_work(wp);
  402 }
  403 
  404 /*
  405  * A write operation has finished.  When we have all expected cows in the
  406  * barn close the door and call it a day.
  407  */
  408 
  409 static void
  410 g_bde_write_done(struct bio *bp)
  411 {
  412         struct g_bde_sector *sp;
  413         struct g_bde_work *wp;
  414         struct g_bde_softc *sc;
  415 
  416         sp = bp->bio_caller1;
  417         sc = bp->bio_caller2;
  418         mtx_lock(&sc->worklist_mutex);
  419         KASSERT(sp != NULL, ("NULL sp"));
  420         KASSERT(sc != NULL, ("NULL sc"));
  421         KASSERT(sp->owner != NULL, ("NULL sp->owner"));
  422         g_trace(G_T_TOPOLOGY, "g_bde_write_done(%p)", sp);
  423         if (bp->bio_error == 0 && bp->bio_completed != sp->size)
  424                 bp->bio_error = EIO;
  425         sp->error = bp->bio_error;
  426         g_destroy_bio(bp);
  427         wp = sp->owner;
  428         if (wp->error == 0)
  429                 wp->error = sp->error;
  430 
  431         if (wp->bp->bio_cmd == BIO_DELETE) {
  432                 KASSERT(sp == wp->sp, ("trashed delete op"));
  433                 g_bde_work_done(wp, wp->error);
  434                 mtx_unlock(&sc->worklist_mutex);
  435                 return;
  436         }
  437 
  438         KASSERT(wp->bp->bio_cmd == BIO_WRITE, ("Confused in g_bde_write_done()"));
  439         KASSERT(sp == wp->sp || sp == wp->ksp, ("trashed write op"));
  440         if (wp->sp == sp) {
  441                 g_bde_delete_sector(sc, wp->sp);
  442                 wp->sp = NULL;
  443         } else {
  444                 sp->state = VALID;
  445         }
  446         if (wp->sp == NULL && wp->ksp != NULL && wp->ksp->state == VALID)
  447                 g_bde_work_done(wp, wp->error);
  448         mtx_unlock(&sc->worklist_mutex);
  449         return;
  450 }
  451 
  452 /*
  453  * Send a write request for the given sector down the pipeline.
  454  */
  455 
  456 static int
  457 g_bde_start_write(struct g_bde_sector *sp)
  458 {
  459         struct bio *bp;
  460         struct g_bde_softc *sc;
  461 
  462         g_trace(G_T_TOPOLOGY, "g_bde_start_write(%p)", sp);
  463         sc = sp->softc;
  464         KASSERT(sc != NULL, ("NULL sc in g_bde_start_write"));
  465         KASSERT(sp->owner != NULL, ("NULL sp->owner in g_bde_start_write"));
  466         bp = g_new_bio();
  467         if (bp == NULL)
  468                 return (ENOMEM);
  469         bp->bio_cmd = BIO_WRITE;
  470         bp->bio_offset = sp->offset;
  471         bp->bio_data = sp->data;
  472         bp->bio_length = sp->size;
  473         bp->bio_done = g_bde_write_done;
  474         bp->bio_caller1 = sp;
  475         bp->bio_caller2 = sc;
  476         sp->state = IO;
  477         g_io_request(bp, sc->consumer);
  478         return(0);
  479 }
  480 
  481 /*
  482  * A read operation has finished.  Mark the sector no longer iobusy and
  483  * wake up the worker thread and let it do its thing.
  484  */
  485 
  486 static void
  487 g_bde_read_done(struct bio *bp)
  488 {
  489         struct g_bde_sector *sp;
  490         struct g_bde_softc *sc;
  491 
  492         sp = bp->bio_caller1;
  493         g_trace(G_T_TOPOLOGY, "g_bde_read_done(%p)", sp);
  494         sc = bp->bio_caller2;
  495         mtx_lock(&sc->worklist_mutex);
  496         if (bp->bio_error == 0 && bp->bio_completed != sp->size)
  497                 bp->bio_error = EIO;
  498         sp->error = bp->bio_error;
  499         if (sp->error == 0)
  500                 sp->state = VALID;
  501         else
  502                 sp->state = JUNK;
  503         wakeup(sc);
  504         g_destroy_bio(bp);
  505         mtx_unlock(&sc->worklist_mutex);
  506 }
  507 
  508 /*
  509  * Send a read request for the given sector down the pipeline.
  510  */
  511 
  512 static int
  513 g_bde_start_read(struct g_bde_sector *sp)
  514 {
  515         struct bio *bp;
  516         struct g_bde_softc *sc;
  517 
  518         g_trace(G_T_TOPOLOGY, "g_bde_start_read(%p)", sp);
  519         sc = sp->softc;
  520         KASSERT(sc != NULL, ("Null softc in sp %p", sp));
  521         bp = g_new_bio();
  522         if (bp == NULL)
  523                 return (ENOMEM);
  524         bp->bio_cmd = BIO_READ;
  525         bp->bio_offset = sp->offset;
  526         bp->bio_data = sp->data;
  527         bp->bio_length = sp->size;
  528         bp->bio_done = g_bde_read_done;
  529         bp->bio_caller1 = sp;
  530         bp->bio_caller2 = sc;
  531         sp->state = IO;
  532         g_io_request(bp, sc->consumer);
  533         return(0);
  534 }
  535 
  536 /*
  537  * The worker thread.
  538  *
  539  * The up/down path of GEOM is not allowed to sleep or do any major work
  540  * so we use this thread to do the actual crypto operations and to push
  541  * the state engine onwards.
  542  *
  543  * XXX: if we switch to the src/sys/opencrypt hardware assisted encryption
  544  * XXX: using a thread here is probably not needed.
  545  */
  546 
  547 void
  548 g_bde_worker(void *arg)
  549 {
  550         struct g_bde_softc *sc;
  551         struct g_bde_work *wp, *twp;
  552         struct g_geom *gp;
  553         int restart, error;
  554 
  555         gp = arg;
  556         sc = gp->softc;
  557 
  558         mtx_lock(&sc->worklist_mutex);
  559         for (;;) {
  560                 restart = 0;
  561                 g_trace(G_T_TOPOLOGY, "g_bde_worker scan");
  562                 TAILQ_FOREACH_SAFE(wp, &sc->worklist, list, twp) {
  563                         KASSERT(wp != NULL, ("NULL wp"));
  564                         KASSERT(wp->softc != NULL, ("NULL wp->softc"));
  565                         if (wp->state != WAIT)
  566                                 continue;       /* Not interesting here */
  567 
  568                         KASSERT(wp->bp != NULL, ("NULL wp->bp"));
  569                         KASSERT(wp->sp != NULL, ("NULL wp->sp"));
  570 
  571                         if (wp->ksp != NULL) {
  572                                 if (wp->ksp->owner != wp)
  573                                         continue;
  574                                 if (wp->ksp->state == IO)
  575                                         continue;
  576                                 KASSERT(wp->ksp->state == VALID,
  577                                     ("Illegal sector state (%d)",
  578                                     wp->ksp->state));
  579                         }
  580 
  581                         if (wp->bp->bio_cmd == BIO_READ && wp->sp->state == IO)
  582                                 continue;
  583 
  584                         if (wp->ksp != NULL && wp->ksp->error != 0) {
  585                                 g_bde_work_done(wp, wp->ksp->error);
  586                                 continue;
  587                         } 
  588                         switch(wp->bp->bio_cmd) {
  589                         case BIO_READ:
  590                                 if (wp->ksp == NULL) {
  591                                         KASSERT(wp->error != 0,
  592                                             ("BIO_READ, no ksp and no error"));
  593                                         g_bde_work_done(wp, wp->error);
  594                                         break;
  595                                 }
  596                                 if (wp->sp->error != 0) {
  597                                         g_bde_work_done(wp, wp->sp->error);
  598                                         break;
  599                                 }
  600                                 mtx_unlock(&sc->worklist_mutex);
  601                                 g_bde_crypt_read(wp);
  602                                 mtx_lock(&sc->worklist_mutex);
  603                                 restart++;
  604                                 g_bde_work_done(wp, wp->sp->error);
  605                                 break;
  606                         case BIO_WRITE:
  607                                 wp->state = FINISH;
  608                                 KASSERT(wp->sp->owner == wp,
  609                                     ("Write not owner sp"));
  610                                 KASSERT(wp->ksp->owner == wp,
  611                                     ("Write not owner ksp"));
  612                                 mtx_unlock(&sc->worklist_mutex);
  613                                 g_bde_crypt_write(wp);
  614                                 mtx_lock(&sc->worklist_mutex);
  615                                 restart++;
  616                                 error = g_bde_start_write(wp->sp);
  617                                 if (error) {
  618                                         g_bde_work_done(wp, error);
  619                                         break;
  620                                 }
  621                                 error = g_bde_start_write(wp->ksp);
  622                                 if (wp->error != 0)
  623                                         wp->error = error;
  624                                 break;
  625                         case BIO_DELETE:
  626                                 wp->state = FINISH;
  627                                 mtx_unlock(&sc->worklist_mutex);
  628                                 g_bde_crypt_delete(wp);
  629                                 mtx_lock(&sc->worklist_mutex);
  630                                 restart++;
  631                                 g_bde_start_write(wp->sp);
  632                                 break;
  633                         }
  634                         if (restart)
  635                                 break;
  636                 }
  637                 if (!restart) {
  638                         /*
  639                          * We don't look for our death-warrant until we are
  640                          * idle.  Shouldn't make a difference in practice.
  641                          */
  642                         if (sc->dead)
  643                                 break;
  644                         g_trace(G_T_TOPOLOGY, "g_bde_worker sleep");
  645                         error = msleep(sc, &sc->worklist_mutex,
  646                             PRIBIO, "-", hz);
  647                         if (error == EWOULDBLOCK) {
  648                                 /*
  649                                  * Lose our skey cache in an orderly fashion.
  650                                  * The exact rate can be tuned to be less
  651                                  * aggressive if this is desirable.  10% per
  652                                  * second means that the cache is gone in a
  653                                  * few minutes.
  654                                  */
  655                                 g_bde_purge_sector(sc, 10);
  656                         }
  657                 }
  658         }
  659         g_trace(G_T_TOPOLOGY, "g_bde_worker die");
  660         g_bde_purge_sector(sc, 1);
  661         KASSERT(sc->nwork == 0, ("Dead but %d work remaining", sc->nwork));
  662         KASSERT(sc->ncache == 0, ("Dead but %d cache remaining", sc->ncache));
  663         KASSERT(sc->nsect == 0, ("Dead but %d sect remaining", sc->nsect));
  664         mtx_unlock(&sc->worklist_mutex);
  665         sc->dead = 2;
  666         wakeup(sc);
  667         kproc_exit(0);
  668 }
  669 
  670 /*
  671  * g_bde_start1 has chopped the incoming request up so all the requests
  672  * we see here are inside a single zone.  Map the data and key locations
  673  * grab the buffers we need and fire off the first volley of read requests.
  674  */
  675 
  676 static void
  677 g_bde_start2(struct g_bde_work *wp)
  678 {
  679         struct g_bde_softc *sc;
  680 
  681         KASSERT(wp != NULL, ("NULL wp in g_bde_start2"));
  682         KASSERT(wp->softc != NULL, ("NULL wp->softc"));
  683         g_trace(G_T_TOPOLOGY, "g_bde_start2(%p)", wp);
  684         sc = wp->softc;
  685         switch (wp->bp->bio_cmd) {
  686         case BIO_READ:
  687                 wp->sp = g_bde_new_sector(wp, 0);
  688                 if (wp->sp == NULL) {
  689                         g_bde_work_done(wp, ENOMEM);
  690                         return;
  691                 }
  692                 wp->sp->size = wp->length;
  693                 wp->sp->data = wp->data;
  694                 if (g_bde_start_read(wp->sp) != 0) {
  695                         g_bde_work_done(wp, ENOMEM);
  696                         return;
  697                 }
  698                 g_bde_read_keysector(sc, wp);
  699                 if (wp->ksp == NULL)
  700                         wp->error = ENOMEM;
  701                 break;
  702         case BIO_DELETE:
  703                 wp->sp = g_bde_new_sector(wp, wp->length);
  704                 if (wp->sp == NULL) {
  705                         g_bde_work_done(wp, ENOMEM);
  706                         return;
  707                 }
  708                 break;
  709         case BIO_WRITE:
  710                 wp->sp = g_bde_new_sector(wp, wp->length);
  711                 if (wp->sp == NULL) {
  712                         g_bde_work_done(wp, ENOMEM);
  713                         return;
  714                 }
  715                 g_bde_read_keysector(sc, wp);
  716                 if (wp->ksp == NULL) {
  717                         g_bde_work_done(wp, ENOMEM);
  718                         return;
  719                 }
  720                 break;
  721         default:
  722                 KASSERT(0 == 1, 
  723                     ("Wrong bio_cmd %d in g_bde_start2", wp->bp->bio_cmd));
  724         }
  725 
  726         wp->state = WAIT;
  727         wakeup(sc);
  728 }
  729 
  730 /*
  731  * Create a sequence of work structures, and have g_bde_map_sector() determine
  732  * how long they each can be.  Feed them to g_bde_start2().
  733  */
  734 
  735 void
  736 g_bde_start1(struct bio *bp)
  737 {
  738         struct g_bde_softc *sc;
  739         struct g_bde_work *wp;
  740         off_t done;
  741 
  742         sc = bp->bio_to->geom->softc;
  743         bp->bio_driver1 = sc;
  744 
  745         mtx_lock(&sc->worklist_mutex);
  746         for(done = 0; done < bp->bio_length; ) {
  747                 wp = g_bde_new_work(sc);
  748                 if (wp != NULL) {
  749                         wp->bp = bp;
  750                         wp->offset = bp->bio_offset + done;
  751                         wp->data = bp->bio_data + done;
  752                         wp->length = bp->bio_length - done;
  753                         g_bde_map_sector(wp);
  754                         done += wp->length;
  755                         g_bde_start2(wp);
  756                 }
  757                 if (wp == NULL || bp->bio_error != 0) {
  758                         g_bde_contribute(bp, bp->bio_length - done, ENOMEM);
  759                         break;
  760                 }
  761         }
  762         mtx_unlock(&sc->worklist_mutex);
  763         return;
  764 }

Cache object: df779e0bab5a22f2b35d719d3b4b4cde


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.