The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/geom/bde/g_bde_work.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2002 Poul-Henning Kamp
    3  * Copyright (c) 2002 Networks Associates Technology, Inc.
    4  * All rights reserved.
    5  *
    6  * This software was developed for the FreeBSD Project by Poul-Henning Kamp
    7  * and NAI Labs, the Security Research Division of Network Associates, Inc.
    8  * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
    9  * DARPA CHATS research program.
   10  *
   11  * Redistribution and use in source and binary forms, with or without
   12  * modification, are permitted provided that the following conditions
   13  * are met:
   14  * 1. Redistributions of source code must retain the above copyright
   15  *    notice, this list of conditions and the following disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  * $FreeBSD: releng/5.3/sys/geom/bde/g_bde_work.c 126674 2004-03-05 22:42:17Z jhb $
   33  *
   34  * This source file contains the state-engine which makes things happen in the
   35  * right order.
   36  *
   37  * Outline:
   38  *   1) g_bde_start1()
   39  *      Break the struct bio into multiple work packets one per zone.
   40  *   2) g_bde_start2()
   41  *      Setup the necessary sector buffers and start those read operations
   42  *      which we can start at this time and put the item on the work-list.
   43  *   3) g_bde_worker()
   44  *      Scan the work-list for items which are ready for crypto processing
   45  *      and call the matching crypto function in g_bde_crypt.c and schedule
   46  *      any writes needed.  Read operations finish here by releasing the
   47  *      sector buffers and delivering the original bio request.
   48  *   4) g_bde_write_done()
   49  *      Release sector buffers and deliver the original bio request.
   50  *
   51  * Because of the C-scope rules, the functions are almost perfectly in the
   52  * opposite order in this source file.
   53  *
   54  * XXX: A switch to the hardware assisted crypto in src/sys/opencrypto will add
   55  * XXX: additional states to this state-engine.  Since no hardware available
   56  * XXX: at this time has AES support, implementing this has been postponed
   57  * XXX: until such time as it would result in a benefit.
   58  */
   59 
   60 #include <sys/param.h>
   61 #include <sys/bio.h>
   62 #include <sys/lock.h>
   63 #include <sys/mutex.h>
   64 #include <sys/queue.h>
   65 #include <sys/malloc.h>
   66 #include <sys/systm.h>
   67 #include <sys/kernel.h>
   68 #include <sys/sysctl.h>
   69 #include <sys/proc.h>
   70 #include <sys/kthread.h>
   71 
   72 #include <crypto/rijndael/rijndael.h>
   73 #include <crypto/sha2/sha2.h>
   74 #include <geom/geom.h>
   75 #include <geom/bde/g_bde.h>
   76 
   77 static void g_bde_delete_sector(struct g_bde_softc *wp, struct g_bde_sector *sp);
   78 static struct g_bde_sector * g_bde_new_sector(struct g_bde_work *wp, u_int len);
   79 static void g_bde_release_keysector(struct g_bde_work *wp);
   80 static struct g_bde_sector *g_bde_get_keysector(struct g_bde_work *wp);
   81 static int g_bde_start_read(struct g_bde_sector *sp);
   82 static void g_bde_purge_sector(struct g_bde_softc *sc, int fraction);
   83 
   84 /*
   85  * Work item allocation.
   86  *
   87  * C++ would call these constructors and destructors.
   88  */
   89 static u_int g_bde_nwork;
   90 SYSCTL_UINT(_debug, OID_AUTO, gbde_nwork, CTLFLAG_RD, &g_bde_nwork, 0, "");
   91 
   92 static MALLOC_DEFINE(M_GBDE, "GBDE", "GBDE data structures");
   93 
   94 static struct g_bde_work *
   95 g_bde_new_work(struct g_bde_softc *sc)
   96 {
   97         struct g_bde_work *wp;
   98 
   99         wp = malloc(sizeof *wp, M_GBDE, M_NOWAIT | M_ZERO);
  100         if (wp == NULL)
  101                 return (wp);
  102         wp->state = SETUP;
  103         wp->softc = sc;
  104         g_bde_nwork++;
  105         sc->nwork++;
  106         TAILQ_INSERT_TAIL(&sc->worklist, wp, list);
  107         return (wp);
  108 }
  109 
  110 static void
  111 g_bde_delete_work(struct g_bde_work *wp)
  112 {
  113         struct g_bde_softc *sc;
  114 
  115         sc = wp->softc;
  116         g_bde_nwork--;
  117         sc->nwork--;
  118         TAILQ_REMOVE(&sc->worklist, wp, list);
  119         free(wp, M_GBDE);
  120 }
  121 
  122 /*
  123  * Sector buffer allocation
  124  *
  125  * These two functions allocate and free back variable sized sector buffers
  126  */
  127 
  128 static u_int g_bde_nsect;
  129 SYSCTL_UINT(_debug, OID_AUTO, gbde_nsect, CTLFLAG_RD, &g_bde_nsect, 0, "");
  130 
  131 static void
  132 g_bde_delete_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
  133 {
  134 
  135         g_bde_nsect--;
  136         sc->nsect--;
  137         if (sp->malloc)
  138                 free(sp->data, M_GBDE);
  139         free(sp, M_GBDE);
  140 }
  141 
  142 static struct g_bde_sector *
  143 g_bde_new_sector(struct g_bde_work *wp, u_int len)
  144 {
  145         struct g_bde_sector *sp;
  146 
  147         sp = malloc(sizeof *sp, M_GBDE, M_NOWAIT | M_ZERO);
  148         if (sp == NULL)
  149                 return (sp);
  150         if (len > 0) {
  151                 sp->data = malloc(len, M_GBDE, M_NOWAIT | M_ZERO);
  152                 if (sp->data == NULL) {
  153                         free(sp, M_GBDE);
  154                         return (NULL);
  155                 }
  156                 sp->malloc = 1;
  157         }
  158         g_bde_nsect++;
  159         wp->softc->nsect++;
  160         sp->size = len;
  161         sp->softc = wp->softc;
  162         sp->ref = 1;
  163         sp->owner = wp;
  164         sp->offset = wp->so;
  165         sp->state = JUNK;
  166         return (sp);
  167 }
  168 
  169 /*
  170  * Skey sector cache.
  171  *
  172  * Nothing prevents two separate I/O requests from addressing the same zone
  173  * and thereby needing the same skey sector.  We therefore need to sequence
  174  * I/O operations to the skey sectors.  A certain amount of caching is also
  175  * desirable, although the extent of benefit from this is not at this point
  176  * determined.
  177  *
  178  * XXX: GEOM may be able to grow a generic caching facility at some point
  179  * XXX: to support such needs.
  180  */
  181 
  182 static u_int g_bde_ncache;
  183 SYSCTL_UINT(_debug, OID_AUTO, gbde_ncache, CTLFLAG_RD, &g_bde_ncache, 0, "");
  184 
  185 static void
  186 g_bde_purge_one_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
  187 {
  188 
  189         g_trace(G_T_TOPOLOGY, "g_bde_purge_one_sector(%p, %p)", sc, sp);
  190         if (sp->ref != 0)
  191                 return;
  192         TAILQ_REMOVE(&sc->freelist, sp, list);
  193         g_bde_ncache--;
  194         sc->ncache--;
  195         bzero(sp->data, sp->size);
  196         g_bde_delete_sector(sc, sp);
  197 }
  198 
  199 static struct g_bde_sector *
  200 g_bde_get_keysector(struct g_bde_work *wp)
  201 {
  202         struct g_bde_sector *sp;
  203         struct g_bde_softc *sc;
  204         off_t offset;
  205 
  206         offset = wp->kso;
  207         g_trace(G_T_TOPOLOGY, "g_bde_get_keysector(%p, %jd)", wp, (intmax_t)offset);
  208         sc = wp->softc;
  209 
  210         if (malloc_last_fail() < g_bde_ncache)
  211                 g_bde_purge_sector(sc, -1);
  212 
  213         sp = TAILQ_FIRST(&sc->freelist);
  214         if (sp != NULL && sp->ref == 0 && sp->used + 300 < time_uptime)
  215                 g_bde_purge_one_sector(sc, sp);
  216 
  217         TAILQ_FOREACH(sp, &sc->freelist, list) {
  218                 if (sp->offset == offset)
  219                         break;
  220         }
  221         if (sp != NULL) {
  222                 sp->ref++;
  223                 KASSERT(sp->offset == offset, ("wrong offset"));
  224                 KASSERT(sp->softc == wp->softc, ("wrong softc"));
  225                 if (sp->ref == 1)
  226                         sp->owner = wp;
  227         } else {
  228                 if (malloc_last_fail() < g_bde_ncache) {
  229                         TAILQ_FOREACH(sp, &sc->freelist, list)
  230                                 if (sp->ref == 0)
  231                                         break;
  232                 }
  233                 if (sp == NULL && !TAILQ_EMPTY(&sc->freelist))
  234                         sp = TAILQ_FIRST(&sc->freelist);
  235                 if (sp != NULL && sp->ref > 0)
  236                         sp = NULL;
  237                 if (sp == NULL) {
  238                         sp = g_bde_new_sector(wp, sc->sectorsize);
  239                         if (sp != NULL) {
  240                                 g_bde_ncache++;
  241                                 sc->ncache++;
  242                                 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
  243                                 sp->malloc = 2;
  244                         }
  245                 }
  246                 if (sp != NULL) {
  247                         sp->offset = offset;
  248                         sp->softc = wp->softc;
  249                         sp->ref = 1;
  250                         sp->owner = wp;
  251                         sp->state = JUNK;
  252                         sp->error = 0;
  253                 }
  254         }
  255         if (sp != NULL) {
  256                 TAILQ_REMOVE(&sc->freelist, sp, list);
  257                 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
  258                 sp->used = time_uptime;
  259         }
  260         wp->ksp = sp;
  261         return(sp);
  262 }
  263 
  264 static void
  265 g_bde_release_keysector(struct g_bde_work *wp)
  266 {
  267         struct g_bde_softc *sc;
  268         struct g_bde_work *wp2;
  269         struct g_bde_sector *sp;
  270 
  271         sp = wp->ksp;
  272         g_trace(G_T_TOPOLOGY, "g_bde_release_keysector(%p)", sp);
  273         KASSERT(sp->malloc == 2, ("Wrong sector released"));
  274         sc = sp->softc;
  275         KASSERT(sc != NULL, ("NULL sp->softc"));
  276         KASSERT(wp == sp->owner, ("Releasing, not owner"));
  277         sp->owner = NULL;
  278         wp->ksp = NULL;
  279         sp->ref--;
  280         if (sp->ref > 0) {
  281                 TAILQ_REMOVE(&sc->freelist, sp, list);
  282                 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
  283                 TAILQ_FOREACH(wp2, &sc->worklist, list) {
  284                         if (wp2->ksp == sp) {
  285                                 KASSERT(wp2 != wp, ("Self-reowning"));
  286                                 sp->owner = wp2;
  287                                 wakeup(sp->softc);
  288                                 break;
  289                         }
  290                 }
  291                 KASSERT(wp2 != NULL, ("Failed to pick up owner for %p\n", sp));
  292         } else if (sp->error != 0) {
  293                 sp->offset = ~0;
  294                 sp->error = 0;
  295                 sp->state = JUNK;
  296         }
  297         TAILQ_REMOVE(&sc->freelist, sp, list);
  298         TAILQ_INSERT_HEAD(&sc->freelist, sp, list);
  299 }
  300 
  301 static void
  302 g_bde_purge_sector(struct g_bde_softc *sc, int fraction)
  303 {
  304         struct g_bde_sector *sp;
  305         int n;
  306 
  307         g_trace(G_T_TOPOLOGY, "g_bde_purge_sector(%p)", sc);
  308         if (fraction > 0)
  309                 n = sc->ncache / fraction + 1;
  310         else 
  311                 n = g_bde_ncache - malloc_last_fail();
  312         if (n < 0)
  313                 return;
  314         if (n > sc->ncache)
  315                 n = sc->ncache;
  316         while(n--) {
  317                 TAILQ_FOREACH(sp, &sc->freelist, list) {
  318                         if (sp->ref != 0)
  319                                 continue;
  320                         TAILQ_REMOVE(&sc->freelist, sp, list);
  321                         g_bde_ncache--;
  322                         sc->ncache--;
  323                         bzero(sp->data, sp->size);
  324                         g_bde_delete_sector(sc, sp);
  325                         break;
  326                 }
  327         }
  328 }
  329 
  330 static struct g_bde_sector *
  331 g_bde_read_keysector(struct g_bde_softc *sc, struct g_bde_work *wp)
  332 {
  333         struct g_bde_sector *sp;
  334 
  335         g_trace(G_T_TOPOLOGY, "g_bde_read_keysector(%p)", wp);
  336         sp = g_bde_get_keysector(wp);
  337         if (sp == NULL) {
  338                 g_bde_purge_sector(sc, -1);
  339                 sp = g_bde_get_keysector(wp);
  340         }
  341         if (sp == NULL)
  342                 return (sp);
  343         if (sp->owner != wp)
  344                 return (sp);
  345         if (sp->state == VALID)
  346                 return (sp);
  347         if (g_bde_start_read(sp) == 0)
  348                 return (sp);
  349         g_bde_release_keysector(wp);
  350         return (NULL);
  351 }
  352 
  353 /*
  354  * Contribute to the completion of the original bio request.
  355  *
  356  * We have no simple way to tell how many bits the original bio request has
  357  * been segmented into, so the easiest way to determine when we can deliver
  358  * it is to keep track of the number of bytes we have completed.  We keep
  359  * track of any errors underway and latch onto the first one.
  360  *
  361  * We always report "nothing done" in case of error, because random bits here
  362  * and there may be completed and returning a number of completed bytes does
  363  * not convey any useful information about which bytes they were.  If some
  364  * piece of broken code somewhere interprets this to mean that nothing has
  365  * changed on the underlying media they deserve the lossage headed for them.
  366  *
  367  * A single mutex per g_bde instance is used to prevent contention.
  368  */
  369 
  370 static void
  371 g_bde_contribute(struct bio *bp, off_t bytes, int error)
  372 {
  373 
  374         g_trace(G_T_TOPOLOGY, "g_bde_contribute bp %p bytes %jd error %d",
  375              bp, (intmax_t)bytes, error);
  376         if (bp->bio_error == 0)
  377                 bp->bio_error = error;
  378         bp->bio_completed += bytes;
  379         KASSERT(bp->bio_completed <= bp->bio_length, ("Too large contribution"));
  380         if (bp->bio_completed == bp->bio_length) {
  381                 if (bp->bio_error != 0)
  382                         bp->bio_completed = 0;
  383                 g_io_deliver(bp, bp->bio_error);
  384         }
  385 }
  386 
  387 /*
  388  * This is the common case "we're done with this work package" function
  389  */
  390 
  391 static void
  392 g_bde_work_done(struct g_bde_work *wp, int error)
  393 {
  394 
  395         g_bde_contribute(wp->bp, wp->length, error);
  396         if (wp->sp != NULL)
  397                 g_bde_delete_sector(wp->softc, wp->sp);
  398         if (wp->ksp != NULL)
  399                 g_bde_release_keysector(wp);
  400         g_bde_delete_work(wp);
  401 }
  402 
  403 /*
  404  * A write operation has finished.  When we have all expected cows in the
  405  * barn close the door and call it a day.
  406  */
  407 
  408 static void
  409 g_bde_write_done(struct bio *bp)
  410 {
  411         struct g_bde_sector *sp;
  412         struct g_bde_work *wp;
  413         struct g_bde_softc *sc;
  414 
  415         sp = bp->bio_caller1;
  416         sc = bp->bio_caller2;
  417         mtx_lock(&sc->worklist_mutex);
  418         KASSERT(sp != NULL, ("NULL sp"));
  419         KASSERT(sc != NULL, ("NULL sc"));
  420         KASSERT(sp->owner != NULL, ("NULL sp->owner"));
  421         g_trace(G_T_TOPOLOGY, "g_bde_write_done(%p)", sp);
  422         if (bp->bio_error == 0 && bp->bio_completed != sp->size)
  423                 bp->bio_error = EIO;
  424         sp->error = bp->bio_error;
  425         g_destroy_bio(bp);
  426         wp = sp->owner;
  427         if (wp->error == 0)
  428                 wp->error = sp->error;
  429 
  430         if (wp->bp->bio_cmd == BIO_DELETE) {
  431                 KASSERT(sp == wp->sp, ("trashed delete op"));
  432                 g_bde_work_done(wp, wp->error);
  433                 mtx_unlock(&sc->worklist_mutex);
  434                 return;
  435         }
  436 
  437         KASSERT(wp->bp->bio_cmd == BIO_WRITE, ("Confused in g_bde_write_done()"));
  438         KASSERT(sp == wp->sp || sp == wp->ksp, ("trashed write op"));
  439         if (wp->sp == sp) {
  440                 g_bde_delete_sector(sc, wp->sp);
  441                 wp->sp = NULL;
  442         } else {
  443                 sp->state = VALID;
  444         }
  445         if (wp->sp == NULL && wp->ksp != NULL && wp->ksp->state == VALID)
  446                 g_bde_work_done(wp, wp->error);
  447         mtx_unlock(&sc->worklist_mutex);
  448         return;
  449 }
  450 
  451 /*
  452  * Send a write request for the given sector down the pipeline.
  453  */
  454 
  455 static int
  456 g_bde_start_write(struct g_bde_sector *sp)
  457 {
  458         struct bio *bp;
  459         struct g_bde_softc *sc;
  460 
  461         g_trace(G_T_TOPOLOGY, "g_bde_start_write(%p)", sp);
  462         sc = sp->softc;
  463         KASSERT(sc != NULL, ("NULL sc in g_bde_start_write"));
  464         KASSERT(sp->owner != NULL, ("NULL sp->owner in g_bde_start_write"));
  465         bp = g_new_bio();
  466         if (bp == NULL)
  467                 return (ENOMEM);
  468         bp->bio_cmd = BIO_WRITE;
  469         bp->bio_offset = sp->offset;
  470         bp->bio_data = sp->data;
  471         bp->bio_length = sp->size;
  472         bp->bio_done = g_bde_write_done;
  473         bp->bio_caller1 = sp;
  474         bp->bio_caller2 = sc;
  475         sp->state = IO;
  476         g_io_request(bp, sc->consumer);
  477         return(0);
  478 }
  479 
  480 /*
  481  * A read operation has finished.  Mark the sector no longer iobusy and
  482  * wake up the worker thread and let it do its thing.
  483  */
  484 
  485 static void
  486 g_bde_read_done(struct bio *bp)
  487 {
  488         struct g_bde_sector *sp;
  489         struct g_bde_softc *sc;
  490 
  491         sp = bp->bio_caller1;
  492         g_trace(G_T_TOPOLOGY, "g_bde_read_done(%p)", sp);
  493         sc = bp->bio_caller2;
  494         mtx_lock(&sc->worklist_mutex);
  495         if (bp->bio_error == 0 && bp->bio_completed != sp->size)
  496                 bp->bio_error = EIO;
  497         sp->error = bp->bio_error;
  498         if (sp->error == 0)
  499                 sp->state = VALID;
  500         else
  501                 sp->state = JUNK;
  502         wakeup(sc);
  503         g_destroy_bio(bp);
  504         mtx_unlock(&sc->worklist_mutex);
  505 }
  506 
  507 /*
  508  * Send a read request for the given sector down the pipeline.
  509  */
  510 
  511 static int
  512 g_bde_start_read(struct g_bde_sector *sp)
  513 {
  514         struct bio *bp;
  515         struct g_bde_softc *sc;
  516 
  517         g_trace(G_T_TOPOLOGY, "g_bde_start_read(%p)", sp);
  518         sc = sp->softc;
  519         KASSERT(sc != NULL, ("Null softc in sp %p", sp));
  520         bp = g_new_bio();
  521         if (bp == NULL)
  522                 return (ENOMEM);
  523         bp->bio_cmd = BIO_READ;
  524         bp->bio_offset = sp->offset;
  525         bp->bio_data = sp->data;
  526         bp->bio_length = sp->size;
  527         bp->bio_done = g_bde_read_done;
  528         bp->bio_caller1 = sp;
  529         bp->bio_caller2 = sc;
  530         sp->state = IO;
  531         g_io_request(bp, sc->consumer);
  532         return(0);
  533 }
  534 
  535 /*
  536  * The worker thread.
  537  *
  538  * The up/down path of GEOM is not allowed to sleep or do any major work
  539  * so we use this thread to do the actual crypto operations and to push
  540  * the state engine onwards.
  541  *
  542  * XXX: if we switch to the src/sys/opencrypt hardware assisted encryption
  543  * XXX: using a thread here is probably not needed.
  544  */
  545 
  546 void
  547 g_bde_worker(void *arg)
  548 {
  549         struct g_bde_softc *sc;
  550         struct g_bde_work *wp, *twp;
  551         struct g_geom *gp;
  552         int restart, error;
  553 
  554         gp = arg;
  555         sc = gp->softc;
  556 
  557         mtx_lock(&sc->worklist_mutex);
  558         for (;;) {
  559                 restart = 0;
  560                 g_trace(G_T_TOPOLOGY, "g_bde_worker scan");
  561                 TAILQ_FOREACH_SAFE(wp, &sc->worklist, list, twp) {
  562                         KASSERT(wp != NULL, ("NULL wp"));
  563                         KASSERT(wp->softc != NULL, ("NULL wp->softc"));
  564                         if (wp->state != WAIT)
  565                                 continue;       /* Not interesting here */
  566 
  567                         KASSERT(wp->bp != NULL, ("NULL wp->bp"));
  568                         KASSERT(wp->sp != NULL, ("NULL wp->sp"));
  569 
  570                         if (wp->ksp != NULL) {
  571                                 if (wp->ksp->owner != wp)
  572                                         continue;
  573                                 if (wp->ksp->state == IO)
  574                                         continue;
  575                                 KASSERT(wp->ksp->state == VALID,
  576                                     ("Illegal sector state (%d)",
  577                                     wp->ksp->state));
  578                         }
  579 
  580                         if (wp->bp->bio_cmd == BIO_READ && wp->sp->state == IO)
  581                                 continue;
  582 
  583                         if (wp->ksp != NULL && wp->ksp->error != 0) {
  584                                 g_bde_work_done(wp, wp->ksp->error);
  585                                 continue;
  586                         } 
  587                         switch(wp->bp->bio_cmd) {
  588                         case BIO_READ:
  589                                 if (wp->ksp == NULL) {
  590                                         KASSERT(wp->error != 0,
  591                                             ("BIO_READ, no ksp and no error"));
  592                                         g_bde_work_done(wp, wp->error);
  593                                         break;
  594                                 }
  595                                 if (wp->sp->error != 0) {
  596                                         g_bde_work_done(wp, wp->sp->error);
  597                                         break;
  598                                 }
  599                                 mtx_unlock(&sc->worklist_mutex);
  600                                 g_bde_crypt_read(wp);
  601                                 mtx_lock(&sc->worklist_mutex);
  602                                 restart++;
  603                                 g_bde_work_done(wp, wp->sp->error);
  604                                 break;
  605                         case BIO_WRITE:
  606                                 wp->state = FINISH;
  607                                 KASSERT(wp->sp->owner == wp,
  608                                     ("Write not owner sp"));
  609                                 KASSERT(wp->ksp->owner == wp,
  610                                     ("Write not owner ksp"));
  611                                 mtx_unlock(&sc->worklist_mutex);
  612                                 g_bde_crypt_write(wp);
  613                                 mtx_lock(&sc->worklist_mutex);
  614                                 restart++;
  615                                 error = g_bde_start_write(wp->sp);
  616                                 if (error) {
  617                                         g_bde_work_done(wp, error);
  618                                         break;
  619                                 }
  620                                 error = g_bde_start_write(wp->ksp);
  621                                 if (wp->error != 0)
  622                                         wp->error = error;
  623                                 break;
  624                         case BIO_DELETE:
  625                                 wp->state = FINISH;
  626                                 mtx_unlock(&sc->worklist_mutex);
  627                                 g_bde_crypt_delete(wp);
  628                                 mtx_lock(&sc->worklist_mutex);
  629                                 restart++;
  630                                 g_bde_start_write(wp->sp);
  631                                 break;
  632                         }
  633                         if (restart)
  634                                 break;
  635                 }
  636                 if (!restart) {
  637                         /*
  638                          * We don't look for our death-warrant until we are
  639                          * idle.  Shouldn't make a difference in practice.
  640                          */
  641                         if (sc->dead)
  642                                 break;
  643                         g_trace(G_T_TOPOLOGY, "g_bde_worker sleep");
  644                         error = msleep(sc, &sc->worklist_mutex,
  645                             PRIBIO, "-", hz);
  646                         if (error == EWOULDBLOCK) {
  647                                 /*
  648                                  * Loose our skey cache in an orderly fashion.
  649                                  * The exact rate can be tuned to be less
  650                                  * aggressive if this is desirable.  10% per
  651                                  * second means that the cache is gone in a
  652                                  * few minutes.
  653                                  */
  654                                 g_bde_purge_sector(sc, 10);
  655                         }
  656                 }
  657         }
  658         g_trace(G_T_TOPOLOGY, "g_bde_worker die");
  659         g_bde_purge_sector(sc, 1);
  660         KASSERT(sc->nwork == 0, ("Dead but %d work remaining", sc->nwork));
  661         KASSERT(sc->ncache == 0, ("Dead but %d cache remaining", sc->ncache));
  662         KASSERT(sc->nsect == 0, ("Dead but %d sect remaining", sc->nsect));
  663         mtx_unlock(&sc->worklist_mutex);
  664         sc->dead = 2;
  665         wakeup(sc);
  666         kthread_exit(0);
  667 }
  668 
  669 /*
  670  * g_bde_start1 has chopped the incoming request up so all the requests
  671  * we see here are inside a single zone.  Map the data and key locations
  672  * grab the buffers we need and fire off the first volley of read requests.
  673  */
  674 
  675 static void
  676 g_bde_start2(struct g_bde_work *wp)
  677 {
  678         struct g_bde_softc *sc;
  679 
  680         KASSERT(wp != NULL, ("NULL wp in g_bde_start2"));
  681         KASSERT(wp->softc != NULL, ("NULL wp->softc"));
  682         g_trace(G_T_TOPOLOGY, "g_bde_start2(%p)", wp);
  683         sc = wp->softc;
  684         switch (wp->bp->bio_cmd) {
  685         case BIO_READ:
  686                 wp->sp = g_bde_new_sector(wp, 0);
  687                 if (wp->sp == NULL) {
  688                         g_bde_work_done(wp, ENOMEM);
  689                         return;
  690                 }
  691                 wp->sp->size = wp->length;
  692                 wp->sp->data = wp->data;
  693                 if (g_bde_start_read(wp->sp) != 0) {
  694                         g_bde_work_done(wp, ENOMEM);
  695                         return;
  696                 }
  697                 g_bde_read_keysector(sc, wp);
  698                 if (wp->ksp == NULL)
  699                         wp->error = ENOMEM;
  700                 break;
  701         case BIO_DELETE:
  702                 wp->sp = g_bde_new_sector(wp, wp->length);
  703                 if (wp->sp == NULL) {
  704                         g_bde_work_done(wp, ENOMEM);
  705                         return;
  706                 }
  707                 break;
  708         case BIO_WRITE:
  709                 wp->sp = g_bde_new_sector(wp, wp->length);
  710                 if (wp->sp == NULL) {
  711                         g_bde_work_done(wp, ENOMEM);
  712                         return;
  713                 }
  714                 g_bde_read_keysector(sc, wp);
  715                 if (wp->ksp == NULL) {
  716                         g_bde_work_done(wp, ENOMEM);
  717                         return;
  718                 }
  719                 break;
  720         default:
  721                 KASSERT(0 == 1, 
  722                     ("Wrong bio_cmd %d in g_bde_start2", wp->bp->bio_cmd));
  723         }
  724 
  725         wp->state = WAIT;
  726         wakeup(sc);
  727 }
  728 
  729 /*
  730  * Create a sequence of work structures, and have g_bde_map_sector() determine
  731  * how long they each can be.  Feed them to g_bde_start2().
  732  */
  733 
  734 void
  735 g_bde_start1(struct bio *bp)
  736 {
  737         struct g_bde_softc *sc;
  738         struct g_bde_work *wp;
  739         off_t done;
  740 
  741         sc = bp->bio_to->geom->softc;
  742         bp->bio_driver1 = sc;
  743 
  744         mtx_lock(&sc->worklist_mutex);
  745         for(done = 0; done < bp->bio_length; ) {
  746                 wp = g_bde_new_work(sc);
  747                 if (wp != NULL) {
  748                         wp->bp = bp;
  749                         wp->offset = bp->bio_offset + done;
  750                         wp->data = bp->bio_data + done;
  751                         wp->length = bp->bio_length - done;
  752                         g_bde_map_sector(wp);
  753                         done += wp->length;
  754                         g_bde_start2(wp);
  755                 }
  756                 if (wp == NULL || bp->bio_error != 0) {
  757                         g_bde_contribute(bp, bp->bio_length - done, ENOMEM);
  758                         break;
  759                 }
  760         }
  761         mtx_unlock(&sc->worklist_mutex);
  762         return;
  763 }

Cache object: 1b147e4823db0951a7e6cd49ce7b0ac5


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.