Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/geom/geom_io.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2002 Poul-Henning Kamp 3 * Copyright (c) 2002 Networks Associates Technology, Inc. 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed for the FreeBSD Project by Poul-Henning Kamp 8 * and NAI Labs, the Security Research Division of Network Associates, Inc. 9 * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the 10 * DARPA CHATS research program. 11 * 12 * Portions of this software were developed by Konstantin Belousov 13 * under sponsorship from the FreeBSD Foundation. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. The names of the authors may not be used to endorse or promote 24 * products derived from this software without specific prior written 25 * permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #include <sys/cdefs.h> 41 __FBSDID("$FreeBSD$"); 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/malloc.h> 47 #include <sys/bio.h> 48 #include <sys/ktr.h> 49 #include <sys/proc.h> 50 #include <sys/stack.h> 51 #include <sys/sysctl.h> 52 53 #include <sys/errno.h> 54 #include <geom/geom.h> 55 #include <geom/geom_int.h> 56 #include <sys/devicestat.h> 57 58 #include <vm/uma.h> 59 #include <vm/vm.h> 60 #include <vm/vm_param.h> 61 #include <vm/vm_kern.h> 62 #include <vm/vm_page.h> 63 #include <vm/vm_object.h> 64 #include <vm/vm_extern.h> 65 #include <vm/vm_map.h> 66 67 static struct g_bioq g_bio_run_down; 68 static struct g_bioq g_bio_run_up; 69 static struct g_bioq g_bio_run_task; 70 71 static u_int pace; 72 static uma_zone_t biozone; 73 74 /* 75 * The head of the list of classifiers used in g_io_request. 76 * Use g_register_classifier() and g_unregister_classifier() 77 * to add/remove entries to the list. 78 * Classifiers are invoked in registration order. 79 */ 80 static TAILQ_HEAD(g_classifier_tailq, g_classifier_hook) 81 g_classifier_tailq = TAILQ_HEAD_INITIALIZER(g_classifier_tailq); 82 83 #include <machine/atomic.h> 84 85 static void 86 g_bioq_lock(struct g_bioq *bq) 87 { 88 89 mtx_lock(&bq->bio_queue_lock); 90 } 91 92 static void 93 g_bioq_unlock(struct g_bioq *bq) 94 { 95 96 mtx_unlock(&bq->bio_queue_lock); 97 } 98 99 #if 0 100 static void 101 g_bioq_destroy(struct g_bioq *bq) 102 { 103 104 mtx_destroy(&bq->bio_queue_lock); 105 } 106 #endif 107 108 static void 109 g_bioq_init(struct g_bioq *bq) 110 { 111 112 TAILQ_INIT(&bq->bio_queue); 113 mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF); 114 } 115 116 static struct bio * 117 g_bioq_first(struct g_bioq *bq) 118 { 119 struct bio *bp; 120 121 bp = TAILQ_FIRST(&bq->bio_queue); 122 if (bp != NULL) { 123 KASSERT((bp->bio_flags & BIO_ONQUEUE), 124 ("Bio not on queue bp=%p target %p", bp, bq)); 125 bp->bio_flags &= ~BIO_ONQUEUE; 126 TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue); 127 bq->bio_queue_length--; 128 } 129 return (bp); 130 } 131 132 struct bio * 133 g_new_bio(void) 134 { 135 struct bio *bp; 136 137 bp = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 138 #ifdef KTR 139 if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 140 struct stack st; 141 142 CTR1(KTR_GEOM, "g_new_bio(): %p", bp); 143 stack_save(&st); 144 CTRSTACK(KTR_GEOM, &st, 3, 0); 145 } 146 #endif 147 return (bp); 148 } 149 150 struct bio * 151 g_alloc_bio(void) 152 { 153 struct bio *bp; 154 155 bp = uma_zalloc(biozone, M_WAITOK | M_ZERO); 156 #ifdef KTR 157 if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 158 struct stack st; 159 160 CTR1(KTR_GEOM, "g_alloc_bio(): %p", bp); 161 stack_save(&st); 162 CTRSTACK(KTR_GEOM, &st, 3, 0); 163 } 164 #endif 165 return (bp); 166 } 167 168 void 169 g_destroy_bio(struct bio *bp) 170 { 171 #ifdef KTR 172 if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 173 struct stack st; 174 175 CTR1(KTR_GEOM, "g_destroy_bio(): %p", bp); 176 stack_save(&st); 177 CTRSTACK(KTR_GEOM, &st, 3, 0); 178 } 179 #endif 180 uma_zfree(biozone, bp); 181 } 182 183 struct bio * 184 g_clone_bio(struct bio *bp) 185 { 186 struct bio *bp2; 187 188 bp2 = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 189 if (bp2 != NULL) { 190 bp2->bio_parent = bp; 191 bp2->bio_cmd = bp->bio_cmd; 192 /* 193 * BIO_ORDERED flag may be used by disk drivers to enforce 194 * ordering restrictions, so this flag needs to be cloned. 195 * BIO_UNMAPPED should be inherited, to properly indicate 196 * which way the buffer is passed. 197 * Other bio flags are not suitable for cloning. 198 */ 199 bp2->bio_flags = bp->bio_flags & (BIO_ORDERED | BIO_UNMAPPED); 200 bp2->bio_length = bp->bio_length; 201 bp2->bio_offset = bp->bio_offset; 202 bp2->bio_data = bp->bio_data; 203 bp2->bio_ma = bp->bio_ma; 204 bp2->bio_ma_n = bp->bio_ma_n; 205 bp2->bio_ma_offset = bp->bio_ma_offset; 206 bp2->bio_attribute = bp->bio_attribute; 207 /* Inherit classification info from the parent */ 208 bp2->bio_classifier1 = bp->bio_classifier1; 209 bp2->bio_classifier2 = bp->bio_classifier2; 210 bp->bio_children++; 211 } 212 #ifdef KTR 213 if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 214 struct stack st; 215 216 CTR2(KTR_GEOM, "g_clone_bio(%p): %p", bp, bp2); 217 stack_save(&st); 218 CTRSTACK(KTR_GEOM, &st, 3, 0); 219 } 220 #endif 221 return(bp2); 222 } 223 224 struct bio * 225 g_duplicate_bio(struct bio *bp) 226 { 227 struct bio *bp2; 228 229 bp2 = uma_zalloc(biozone, M_WAITOK | M_ZERO); 230 bp2->bio_flags = bp->bio_flags & BIO_UNMAPPED; 231 bp2->bio_parent = bp; 232 bp2->bio_cmd = bp->bio_cmd; 233 bp2->bio_length = bp->bio_length; 234 bp2->bio_offset = bp->bio_offset; 235 bp2->bio_data = bp->bio_data; 236 bp2->bio_ma = bp->bio_ma; 237 bp2->bio_ma_n = bp->bio_ma_n; 238 bp2->bio_ma_offset = bp->bio_ma_offset; 239 bp2->bio_attribute = bp->bio_attribute; 240 bp->bio_children++; 241 #ifdef KTR 242 if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 243 struct stack st; 244 245 CTR2(KTR_GEOM, "g_duplicate_bio(%p): %p", bp, bp2); 246 stack_save(&st); 247 CTRSTACK(KTR_GEOM, &st, 3, 0); 248 } 249 #endif 250 return(bp2); 251 } 252 253 void 254 g_io_init() 255 { 256 257 g_bioq_init(&g_bio_run_down); 258 g_bioq_init(&g_bio_run_up); 259 g_bioq_init(&g_bio_run_task); 260 biozone = uma_zcreate("g_bio", sizeof (struct bio), 261 NULL, NULL, 262 NULL, NULL, 263 0, 0); 264 } 265 266 int 267 g_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr) 268 { 269 struct bio *bp; 270 int error; 271 272 g_trace(G_T_BIO, "bio_getattr(%s)", attr); 273 bp = g_alloc_bio(); 274 bp->bio_cmd = BIO_GETATTR; 275 bp->bio_done = NULL; 276 bp->bio_attribute = attr; 277 bp->bio_length = *len; 278 bp->bio_data = ptr; 279 g_io_request(bp, cp); 280 error = biowait(bp, "ggetattr"); 281 *len = bp->bio_completed; 282 g_destroy_bio(bp); 283 return (error); 284 } 285 286 int 287 g_io_flush(struct g_consumer *cp) 288 { 289 struct bio *bp; 290 int error; 291 292 g_trace(G_T_BIO, "bio_flush(%s)", cp->provider->name); 293 bp = g_alloc_bio(); 294 bp->bio_cmd = BIO_FLUSH; 295 bp->bio_flags |= BIO_ORDERED; 296 bp->bio_done = NULL; 297 bp->bio_attribute = NULL; 298 bp->bio_offset = cp->provider->mediasize; 299 bp->bio_length = 0; 300 bp->bio_data = NULL; 301 g_io_request(bp, cp); 302 error = biowait(bp, "gflush"); 303 g_destroy_bio(bp); 304 return (error); 305 } 306 307 static int 308 g_io_check(struct bio *bp) 309 { 310 struct g_consumer *cp; 311 struct g_provider *pp; 312 313 cp = bp->bio_from; 314 pp = bp->bio_to; 315 316 /* Fail if access counters dont allow the operation */ 317 switch(bp->bio_cmd) { 318 case BIO_READ: 319 case BIO_GETATTR: 320 if (cp->acr == 0) 321 return (EPERM); 322 break; 323 case BIO_WRITE: 324 case BIO_DELETE: 325 case BIO_FLUSH: 326 if (cp->acw == 0) 327 return (EPERM); 328 break; 329 default: 330 return (EPERM); 331 } 332 /* if provider is marked for error, don't disturb. */ 333 if (pp->error) 334 return (pp->error); 335 if (cp->flags & G_CF_ORPHAN) 336 return (ENXIO); 337 338 switch(bp->bio_cmd) { 339 case BIO_READ: 340 case BIO_WRITE: 341 case BIO_DELETE: 342 /* Zero sectorsize or mediasize is probably a lack of media. */ 343 if (pp->sectorsize == 0 || pp->mediasize == 0) 344 return (ENXIO); 345 /* Reject I/O not on sector boundary */ 346 if (bp->bio_offset % pp->sectorsize) 347 return (EINVAL); 348 /* Reject I/O not integral sector long */ 349 if (bp->bio_length % pp->sectorsize) 350 return (EINVAL); 351 /* Reject requests before or past the end of media. */ 352 if (bp->bio_offset < 0) 353 return (EIO); 354 if (bp->bio_offset > pp->mediasize) 355 return (EIO); 356 break; 357 default: 358 break; 359 } 360 return (0); 361 } 362 363 /* 364 * bio classification support. 365 * 366 * g_register_classifier() and g_unregister_classifier() 367 * are used to add/remove a classifier from the list. 368 * The list is protected using the g_bio_run_down lock, 369 * because the classifiers are called in this path. 370 * 371 * g_io_request() passes bio's that are not already classified 372 * (i.e. those with bio_classifier1 == NULL) to g_run_classifiers(). 373 * Classifiers can store their result in the two fields 374 * bio_classifier1 and bio_classifier2. 375 * A classifier that updates one of the fields should 376 * return a non-zero value. 377 * If no classifier updates the field, g_run_classifiers() sets 378 * bio_classifier1 = BIO_NOTCLASSIFIED to avoid further calls. 379 */ 380 381 int 382 g_register_classifier(struct g_classifier_hook *hook) 383 { 384 385 g_bioq_lock(&g_bio_run_down); 386 TAILQ_INSERT_TAIL(&g_classifier_tailq, hook, link); 387 g_bioq_unlock(&g_bio_run_down); 388 389 return (0); 390 } 391 392 void 393 g_unregister_classifier(struct g_classifier_hook *hook) 394 { 395 struct g_classifier_hook *entry; 396 397 g_bioq_lock(&g_bio_run_down); 398 TAILQ_FOREACH(entry, &g_classifier_tailq, link) { 399 if (entry == hook) { 400 TAILQ_REMOVE(&g_classifier_tailq, hook, link); 401 break; 402 } 403 } 404 g_bioq_unlock(&g_bio_run_down); 405 } 406 407 static void 408 g_run_classifiers(struct bio *bp) 409 { 410 struct g_classifier_hook *hook; 411 int classified = 0; 412 413 TAILQ_FOREACH(hook, &g_classifier_tailq, link) 414 classified |= hook->func(hook->arg, bp); 415 416 if (!classified) 417 bp->bio_classifier1 = BIO_NOTCLASSIFIED; 418 } 419 420 void 421 g_io_request(struct bio *bp, struct g_consumer *cp) 422 { 423 struct g_provider *pp; 424 int first; 425 426 KASSERT(cp != NULL, ("NULL cp in g_io_request")); 427 KASSERT(bp != NULL, ("NULL bp in g_io_request")); 428 pp = cp->provider; 429 KASSERT(pp != NULL, ("consumer not attached in g_io_request")); 430 #ifdef DIAGNOSTIC 431 KASSERT(bp->bio_driver1 == NULL, 432 ("bio_driver1 used by the consumer (geom %s)", cp->geom->name)); 433 KASSERT(bp->bio_driver2 == NULL, 434 ("bio_driver2 used by the consumer (geom %s)", cp->geom->name)); 435 KASSERT(bp->bio_pflags == 0, 436 ("bio_pflags used by the consumer (geom %s)", cp->geom->name)); 437 /* 438 * Remember consumer's private fields, so we can detect if they were 439 * modified by the provider. 440 */ 441 bp->_bio_caller1 = bp->bio_caller1; 442 bp->_bio_caller2 = bp->bio_caller2; 443 bp->_bio_cflags = bp->bio_cflags; 444 #endif 445 446 if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_GETATTR)) { 447 KASSERT(bp->bio_data != NULL, 448 ("NULL bp->data in g_io_request(cmd=%hhu)", bp->bio_cmd)); 449 } 450 if (bp->bio_cmd & (BIO_DELETE|BIO_FLUSH)) { 451 KASSERT(bp->bio_data == NULL, 452 ("non-NULL bp->data in g_io_request(cmd=%hhu)", 453 bp->bio_cmd)); 454 } 455 if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) { 456 KASSERT(bp->bio_offset % cp->provider->sectorsize == 0, 457 ("wrong offset %jd for sectorsize %u", 458 bp->bio_offset, cp->provider->sectorsize)); 459 KASSERT(bp->bio_length % cp->provider->sectorsize == 0, 460 ("wrong length %jd for sectorsize %u", 461 bp->bio_length, cp->provider->sectorsize)); 462 } 463 464 g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d", 465 bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd); 466 467 bp->bio_from = cp; 468 bp->bio_to = pp; 469 bp->bio_error = 0; 470 bp->bio_completed = 0; 471 472 KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 473 ("Bio already on queue bp=%p", bp)); 474 bp->bio_flags |= BIO_ONQUEUE; 475 476 if (g_collectstats) 477 binuptime(&bp->bio_t0); 478 else 479 getbinuptime(&bp->bio_t0); 480 481 /* 482 * The statistics collection is lockless, as such, but we 483 * can not update one instance of the statistics from more 484 * than one thread at a time, so grab the lock first. 485 * 486 * We also use the lock to protect the list of classifiers. 487 */ 488 g_bioq_lock(&g_bio_run_down); 489 490 if (!TAILQ_EMPTY(&g_classifier_tailq) && !bp->bio_classifier1) 491 g_run_classifiers(bp); 492 493 if (g_collectstats & 1) 494 devstat_start_transaction(pp->stat, &bp->bio_t0); 495 if (g_collectstats & 2) 496 devstat_start_transaction(cp->stat, &bp->bio_t0); 497 498 pp->nstart++; 499 cp->nstart++; 500 first = TAILQ_EMPTY(&g_bio_run_down.bio_queue); 501 TAILQ_INSERT_TAIL(&g_bio_run_down.bio_queue, bp, bio_queue); 502 g_bio_run_down.bio_queue_length++; 503 g_bioq_unlock(&g_bio_run_down); 504 505 /* Pass it on down. */ 506 if (first) 507 wakeup(&g_wait_down); 508 } 509 510 void 511 g_io_deliver(struct bio *bp, int error) 512 { 513 struct bintime now; 514 struct g_consumer *cp; 515 struct g_provider *pp; 516 int first; 517 518 KASSERT(bp != NULL, ("NULL bp in g_io_deliver")); 519 pp = bp->bio_to; 520 KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver")); 521 cp = bp->bio_from; 522 if (cp == NULL) { 523 bp->bio_error = error; 524 bp->bio_done(bp); 525 return; 526 } 527 KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver")); 528 KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver")); 529 #ifdef DIAGNOSTIC 530 /* 531 * Some classes - GJournal in particular - can modify bio's 532 * private fields while the bio is in transit; G_GEOM_VOLATILE_BIO 533 * flag means it's an expected behaviour for that particular geom. 534 */ 535 if ((cp->geom->flags & G_GEOM_VOLATILE_BIO) == 0) { 536 KASSERT(bp->bio_caller1 == bp->_bio_caller1, 537 ("bio_caller1 used by the provider %s", pp->name)); 538 KASSERT(bp->bio_caller2 == bp->_bio_caller2, 539 ("bio_caller2 used by the provider %s", pp->name)); 540 KASSERT(bp->bio_cflags == bp->_bio_cflags, 541 ("bio_cflags used by the provider %s", pp->name)); 542 } 543 #endif 544 KASSERT(bp->bio_completed >= 0, ("bio_completed can't be less than 0")); 545 KASSERT(bp->bio_completed <= bp->bio_length, 546 ("bio_completed can't be greater than bio_length")); 547 548 g_trace(G_T_BIO, 549 "g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd", 550 bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error, 551 (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 552 553 KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 554 ("Bio already on queue bp=%p", bp)); 555 556 /* 557 * XXX: next two doesn't belong here 558 */ 559 bp->bio_bcount = bp->bio_length; 560 bp->bio_resid = bp->bio_bcount - bp->bio_completed; 561 562 /* 563 * The statistics collection is lockless, as such, but we 564 * can not update one instance of the statistics from more 565 * than one thread at a time, so grab the lock first. 566 */ 567 if (g_collectstats) 568 binuptime(&now); 569 g_bioq_lock(&g_bio_run_up); 570 if (g_collectstats & 1) 571 devstat_end_transaction_bio_bt(pp->stat, bp, &now); 572 if (g_collectstats & 2) 573 devstat_end_transaction_bio_bt(cp->stat, bp, &now); 574 575 cp->nend++; 576 pp->nend++; 577 if (error != ENOMEM) { 578 bp->bio_error = error; 579 first = TAILQ_EMPTY(&g_bio_run_up.bio_queue); 580 TAILQ_INSERT_TAIL(&g_bio_run_up.bio_queue, bp, bio_queue); 581 bp->bio_flags |= BIO_ONQUEUE; 582 g_bio_run_up.bio_queue_length++; 583 g_bioq_unlock(&g_bio_run_up); 584 if (first) 585 wakeup(&g_wait_up); 586 return; 587 } 588 g_bioq_unlock(&g_bio_run_up); 589 590 if (bootverbose) 591 printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name); 592 bp->bio_children = 0; 593 bp->bio_inbed = 0; 594 bp->bio_driver1 = NULL; 595 bp->bio_driver2 = NULL; 596 bp->bio_pflags = 0; 597 g_io_request(bp, cp); 598 pace++; 599 return; 600 } 601 602 SYSCTL_DECL(_kern_geom); 603 604 static long transient_maps; 605 SYSCTL_LONG(_kern_geom, OID_AUTO, transient_maps, CTLFLAG_RD, 606 &transient_maps, 0, 607 "Total count of the transient mapping requests"); 608 u_int transient_map_retries = 10; 609 SYSCTL_UINT(_kern_geom, OID_AUTO, transient_map_retries, CTLFLAG_RW, 610 &transient_map_retries, 0, 611 "Max count of retries used before giving up on creating transient map"); 612 int transient_map_hard_failures; 613 SYSCTL_INT(_kern_geom, OID_AUTO, transient_map_hard_failures, CTLFLAG_RD, 614 &transient_map_hard_failures, 0, 615 "Failures to establish the transient mapping due to retry attempts " 616 "exhausted"); 617 int transient_map_soft_failures; 618 SYSCTL_INT(_kern_geom, OID_AUTO, transient_map_soft_failures, CTLFLAG_RD, 619 &transient_map_soft_failures, 0, 620 "Count of retried failures to establish the transient mapping"); 621 int inflight_transient_maps; 622 SYSCTL_INT(_kern_geom, OID_AUTO, inflight_transient_maps, CTLFLAG_RD, 623 &inflight_transient_maps, 0, 624 "Current count of the active transient maps"); 625 626 static int 627 g_io_transient_map_bio(struct bio *bp) 628 { 629 vm_offset_t addr; 630 long size; 631 u_int retried; 632 int rv; 633 634 KASSERT(unmapped_buf_allowed, ("unmapped disabled")); 635 636 size = round_page(bp->bio_ma_offset + bp->bio_length); 637 KASSERT(size / PAGE_SIZE == bp->bio_ma_n, ("Bio too short %p", bp)); 638 addr = 0; 639 retried = 0; 640 atomic_add_long(&transient_maps, 1); 641 retry: 642 vm_map_lock(bio_transient_map); 643 if (vm_map_findspace(bio_transient_map, vm_map_min(bio_transient_map), 644 size, &addr)) { 645 vm_map_unlock(bio_transient_map); 646 if (transient_map_retries != 0 && 647 retried >= transient_map_retries) { 648 g_io_deliver(bp, EDEADLK/* XXXKIB */); 649 CTR2(KTR_GEOM, "g_down cannot map bp %p provider %s", 650 bp, bp->bio_to->name); 651 atomic_add_int(&transient_map_hard_failures, 1); 652 return (1); 653 } else { 654 /* 655 * Naive attempt to quisce the I/O to get more 656 * in-flight requests completed and defragment 657 * the bio_transient_map. 658 */ 659 CTR3(KTR_GEOM, "g_down retrymap bp %p provider %s r %d", 660 bp, bp->bio_to->name, retried); 661 pause("g_d_tra", hz / 10); 662 retried++; 663 atomic_add_int(&transient_map_soft_failures, 1); 664 goto retry; 665 } 666 } 667 rv = vm_map_insert(bio_transient_map, NULL, 0, addr, addr + size, 668 VM_PROT_RW, VM_PROT_RW, MAP_NOFAULT); 669 KASSERT(rv == KERN_SUCCESS, 670 ("vm_map_insert(bio_transient_map) rv %d %jx %lx", 671 rv, (uintmax_t)addr, size)); 672 vm_map_unlock(bio_transient_map); 673 atomic_add_int(&inflight_transient_maps, 1); 674 pmap_qenter((vm_offset_t)addr, bp->bio_ma, OFF_TO_IDX(size)); 675 bp->bio_data = (caddr_t)addr + bp->bio_ma_offset; 676 bp->bio_flags |= BIO_TRANSIENT_MAPPING; 677 bp->bio_flags &= ~BIO_UNMAPPED; 678 return (0); 679 } 680 681 void 682 g_io_schedule_down(struct thread *tp __unused) 683 { 684 struct bio *bp; 685 off_t excess; 686 int error; 687 688 for(;;) { 689 g_bioq_lock(&g_bio_run_down); 690 bp = g_bioq_first(&g_bio_run_down); 691 if (bp == NULL) { 692 CTR0(KTR_GEOM, "g_down going to sleep"); 693 msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock, 694 PRIBIO | PDROP, "-", 0); 695 continue; 696 } 697 CTR0(KTR_GEOM, "g_down has work to do"); 698 g_bioq_unlock(&g_bio_run_down); 699 if (pace > 0) { 700 CTR1(KTR_GEOM, "g_down pacing self (pace %d)", pace); 701 pause("g_down", hz/10); 702 pace--; 703 } 704 error = g_io_check(bp); 705 if (error) { 706 CTR3(KTR_GEOM, "g_down g_io_check on bp %p provider " 707 "%s returned %d", bp, bp->bio_to->name, error); 708 g_io_deliver(bp, error); 709 continue; 710 } 711 CTR2(KTR_GEOM, "g_down processing bp %p provider %s", bp, 712 bp->bio_to->name); 713 switch (bp->bio_cmd) { 714 case BIO_READ: 715 case BIO_WRITE: 716 case BIO_DELETE: 717 /* Truncate requests to the end of providers media. */ 718 /* 719 * XXX: What if we truncate because of offset being 720 * bad, not length? 721 */ 722 excess = bp->bio_offset + bp->bio_length; 723 if (excess > bp->bio_to->mediasize) { 724 KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 || 725 round_page(bp->bio_ma_offset + 726 bp->bio_length) / PAGE_SIZE == bp->bio_ma_n, 727 ("excess bio %p too short", bp)); 728 excess -= bp->bio_to->mediasize; 729 bp->bio_length -= excess; 730 if ((bp->bio_flags & BIO_UNMAPPED) != 0) { 731 bp->bio_ma_n = round_page( 732 bp->bio_ma_offset + 733 bp->bio_length) / PAGE_SIZE; 734 } 735 if (excess > 0) 736 CTR3(KTR_GEOM, "g_down truncated bio " 737 "%p provider %s by %d", bp, 738 bp->bio_to->name, excess); 739 } 740 /* Deliver zero length transfers right here. */ 741 if (bp->bio_length == 0) { 742 g_io_deliver(bp, 0); 743 CTR2(KTR_GEOM, "g_down terminated 0-length " 744 "bp %p provider %s", bp, bp->bio_to->name); 745 continue; 746 } 747 break; 748 default: 749 break; 750 } 751 if ((bp->bio_flags & BIO_UNMAPPED) != 0 && 752 (bp->bio_to->flags & G_PF_ACCEPT_UNMAPPED) == 0 && 753 (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE)) { 754 if (g_io_transient_map_bio(bp)) 755 continue; 756 } 757 THREAD_NO_SLEEPING(); 758 CTR4(KTR_GEOM, "g_down starting bp %p provider %s off %ld " 759 "len %ld", bp, bp->bio_to->name, bp->bio_offset, 760 bp->bio_length); 761 bp->bio_to->geom->start(bp); 762 THREAD_SLEEPING_OK(); 763 } 764 } 765 766 void 767 bio_taskqueue(struct bio *bp, bio_task_t *func, void *arg) 768 { 769 bp->bio_task = func; 770 bp->bio_task_arg = arg; 771 /* 772 * The taskqueue is actually just a second queue off the "up" 773 * queue, so we use the same lock. 774 */ 775 g_bioq_lock(&g_bio_run_up); 776 KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 777 ("Bio already on queue bp=%p target taskq", bp)); 778 bp->bio_flags |= BIO_ONQUEUE; 779 TAILQ_INSERT_TAIL(&g_bio_run_task.bio_queue, bp, bio_queue); 780 g_bio_run_task.bio_queue_length++; 781 wakeup(&g_wait_up); 782 g_bioq_unlock(&g_bio_run_up); 783 } 784 785 786 void 787 g_io_schedule_up(struct thread *tp __unused) 788 { 789 struct bio *bp; 790 for(;;) { 791 g_bioq_lock(&g_bio_run_up); 792 bp = g_bioq_first(&g_bio_run_task); 793 if (bp != NULL) { 794 g_bioq_unlock(&g_bio_run_up); 795 THREAD_NO_SLEEPING(); 796 CTR1(KTR_GEOM, "g_up processing task bp %p", bp); 797 bp->bio_task(bp->bio_task_arg); 798 THREAD_SLEEPING_OK(); 799 continue; 800 } 801 bp = g_bioq_first(&g_bio_run_up); 802 if (bp != NULL) { 803 g_bioq_unlock(&g_bio_run_up); 804 THREAD_NO_SLEEPING(); 805 CTR4(KTR_GEOM, "g_up biodone bp %p provider %s off " 806 "%jd len %ld", bp, bp->bio_to->name, 807 bp->bio_offset, bp->bio_length); 808 biodone(bp); 809 THREAD_SLEEPING_OK(); 810 continue; 811 } 812 CTR0(KTR_GEOM, "g_up going to sleep"); 813 msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock, 814 PRIBIO | PDROP, "-", 0); 815 } 816 } 817 818 void * 819 g_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error) 820 { 821 struct bio *bp; 822 void *ptr; 823 int errorc; 824 825 KASSERT(length > 0 && length >= cp->provider->sectorsize && 826 length <= MAXPHYS, ("g_read_data(): invalid length %jd", 827 (intmax_t)length)); 828 829 bp = g_alloc_bio(); 830 bp->bio_cmd = BIO_READ; 831 bp->bio_done = NULL; 832 bp->bio_offset = offset; 833 bp->bio_length = length; 834 ptr = g_malloc(length, M_WAITOK); 835 bp->bio_data = ptr; 836 g_io_request(bp, cp); 837 errorc = biowait(bp, "gread"); 838 if (error != NULL) 839 *error = errorc; 840 g_destroy_bio(bp); 841 if (errorc) { 842 g_free(ptr); 843 ptr = NULL; 844 } 845 return (ptr); 846 } 847 848 int 849 g_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length) 850 { 851 struct bio *bp; 852 int error; 853 854 KASSERT(length > 0 && length >= cp->provider->sectorsize && 855 length <= MAXPHYS, ("g_write_data(): invalid length %jd", 856 (intmax_t)length)); 857 858 bp = g_alloc_bio(); 859 bp->bio_cmd = BIO_WRITE; 860 bp->bio_done = NULL; 861 bp->bio_offset = offset; 862 bp->bio_length = length; 863 bp->bio_data = ptr; 864 g_io_request(bp, cp); 865 error = biowait(bp, "gwrite"); 866 g_destroy_bio(bp); 867 return (error); 868 } 869 870 int 871 g_delete_data(struct g_consumer *cp, off_t offset, off_t length) 872 { 873 struct bio *bp; 874 int error; 875 876 KASSERT(length > 0 && length >= cp->provider->sectorsize, 877 ("g_delete_data(): invalid length %jd", (intmax_t)length)); 878 879 bp = g_alloc_bio(); 880 bp->bio_cmd = BIO_DELETE; 881 bp->bio_done = NULL; 882 bp->bio_offset = offset; 883 bp->bio_length = length; 884 bp->bio_data = NULL; 885 g_io_request(bp, cp); 886 error = biowait(bp, "gdelete"); 887 g_destroy_bio(bp); 888 return (error); 889 } 890 891 void 892 g_print_bio(struct bio *bp) 893 { 894 const char *pname, *cmd = NULL; 895 896 if (bp->bio_to != NULL) 897 pname = bp->bio_to->name; 898 else 899 pname = "[unknown]"; 900 901 switch (bp->bio_cmd) { 902 case BIO_GETATTR: 903 cmd = "GETATTR"; 904 printf("%s[%s(attr=%s)]", pname, cmd, bp->bio_attribute); 905 return; 906 case BIO_FLUSH: 907 cmd = "FLUSH"; 908 printf("%s[%s]", pname, cmd); 909 return; 910 case BIO_READ: 911 cmd = "READ"; 912 break; 913 case BIO_WRITE: 914 cmd = "WRITE"; 915 break; 916 case BIO_DELETE: 917 cmd = "DELETE"; 918 break; 919 default: 920 cmd = "UNKNOWN"; 921 printf("%s[%s()]", pname, cmd); 922 return; 923 } 924 printf("%s[%s(offset=%jd, length=%jd)]", pname, cmd, 925 (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 926 }
Cache object: 3e0343f5f40a31a25e249a2650021a73
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]