Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/dev/virtio/balloon/virtio_balloon.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for VirtIO memory balloon devices. */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/endian.h> 38 #include <sys/kthread.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/sglist.h> 42 #include <sys/sysctl.h> 43 #include <sys/lock.h> 44 #include <sys/mutex.h> 45 #include <sys/queue.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_page.h> 49 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 #include <sys/bus.h> 53 #include <sys/rman.h> 54 55 #include <dev/virtio/virtio.h> 56 #include <dev/virtio/virtqueue.h> 57 #include <dev/virtio/balloon/virtio_balloon.h> 58 59 #include "virtio_if.h" 60 61 struct vtballoon_softc { 62 device_t vtballoon_dev; 63 struct mtx vtballoon_mtx; 64 uint64_t vtballoon_features; 65 uint32_t vtballoon_flags; 66 #define VTBALLOON_FLAG_DETACH 0x01 67 68 struct virtqueue *vtballoon_inflate_vq; 69 struct virtqueue *vtballoon_deflate_vq; 70 71 uint32_t vtballoon_desired_npages; 72 uint32_t vtballoon_current_npages; 73 TAILQ_HEAD(,vm_page) vtballoon_pages; 74 75 struct thread *vtballoon_td; 76 uint32_t *vtballoon_page_frames; 77 int vtballoon_timeout; 78 }; 79 80 static struct virtio_feature_desc vtballoon_feature_desc[] = { 81 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" }, 82 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" }, 83 { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" }, 84 85 { 0, NULL } 86 }; 87 88 static int vtballoon_probe(device_t); 89 static int vtballoon_attach(device_t); 90 static int vtballoon_detach(device_t); 91 static int vtballoon_config_change(device_t); 92 93 static int vtballoon_negotiate_features(struct vtballoon_softc *); 94 static int vtballoon_setup_features(struct vtballoon_softc *); 95 static int vtballoon_alloc_virtqueues(struct vtballoon_softc *); 96 97 static void vtballoon_vq_intr(void *); 98 99 static void vtballoon_inflate(struct vtballoon_softc *, int); 100 static void vtballoon_deflate(struct vtballoon_softc *, int); 101 102 static void vtballoon_send_page_frames(struct vtballoon_softc *, 103 struct virtqueue *, int); 104 105 static void vtballoon_pop(struct vtballoon_softc *); 106 static void vtballoon_stop(struct vtballoon_softc *); 107 108 static vm_page_t 109 vtballoon_alloc_page(struct vtballoon_softc *); 110 static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t); 111 112 static int vtballoon_sleep(struct vtballoon_softc *); 113 static void vtballoon_thread(void *); 114 static void vtballoon_setup_sysctl(struct vtballoon_softc *); 115 116 #define vtballoon_modern(_sc) \ 117 (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0) 118 119 /* Features desired/implemented by this driver. */ 120 #define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST 121 122 /* Timeout between retries when the balloon needs inflating. */ 123 #define VTBALLOON_LOWMEM_TIMEOUT hz 124 125 /* 126 * Maximum number of pages we'll request to inflate or deflate 127 * the balloon in one virtqueue request. Both Linux and NetBSD 128 * have settled on 256, doing up to 1MB at a time. 129 */ 130 #define VTBALLOON_PAGES_PER_REQUEST 256 131 132 /* Must be able to fix all pages frames in one page (segment). */ 133 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE); 134 135 #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx 136 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \ 137 "VirtIO Balloon Lock", MTX_DEF) 138 #define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc))) 139 #define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc))) 140 #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc))) 141 142 static device_method_t vtballoon_methods[] = { 143 /* Device methods. */ 144 DEVMETHOD(device_probe, vtballoon_probe), 145 DEVMETHOD(device_attach, vtballoon_attach), 146 DEVMETHOD(device_detach, vtballoon_detach), 147 148 /* VirtIO methods. */ 149 DEVMETHOD(virtio_config_change, vtballoon_config_change), 150 151 DEVMETHOD_END 152 }; 153 154 static driver_t vtballoon_driver = { 155 "vtballoon", 156 vtballoon_methods, 157 sizeof(struct vtballoon_softc) 158 }; 159 160 VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver, 0, 0); 161 MODULE_VERSION(virtio_balloon, 1); 162 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1); 163 164 VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON, 165 "VirtIO Balloon Adapter"); 166 167 static int 168 vtballoon_probe(device_t dev) 169 { 170 return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon)); 171 } 172 173 static int 174 vtballoon_attach(device_t dev) 175 { 176 struct vtballoon_softc *sc; 177 int error; 178 179 sc = device_get_softc(dev); 180 sc->vtballoon_dev = dev; 181 virtio_set_feature_desc(dev, vtballoon_feature_desc); 182 183 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev)); 184 TAILQ_INIT(&sc->vtballoon_pages); 185 186 vtballoon_setup_sysctl(sc); 187 188 error = vtballoon_setup_features(sc); 189 if (error) { 190 device_printf(dev, "cannot setup features\n"); 191 goto fail; 192 } 193 194 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST * 195 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO); 196 if (sc->vtballoon_page_frames == NULL) { 197 error = ENOMEM; 198 device_printf(dev, 199 "cannot allocate page frame request array\n"); 200 goto fail; 201 } 202 203 error = vtballoon_alloc_virtqueues(sc); 204 if (error) { 205 device_printf(dev, "cannot allocate virtqueues\n"); 206 goto fail; 207 } 208 209 error = virtio_setup_intr(dev, INTR_TYPE_MISC); 210 if (error) { 211 device_printf(dev, "cannot setup virtqueue interrupts\n"); 212 goto fail; 213 } 214 215 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td, 216 0, 0, "virtio_balloon"); 217 if (error) { 218 device_printf(dev, "cannot create balloon kthread\n"); 219 goto fail; 220 } 221 222 virtqueue_enable_intr(sc->vtballoon_inflate_vq); 223 virtqueue_enable_intr(sc->vtballoon_deflate_vq); 224 225 fail: 226 if (error) 227 vtballoon_detach(dev); 228 229 return (error); 230 } 231 232 static int 233 vtballoon_detach(device_t dev) 234 { 235 struct vtballoon_softc *sc; 236 237 sc = device_get_softc(dev); 238 239 if (sc->vtballoon_td != NULL) { 240 VTBALLOON_LOCK(sc); 241 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH; 242 wakeup_one(sc); 243 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0); 244 VTBALLOON_UNLOCK(sc); 245 246 sc->vtballoon_td = NULL; 247 } 248 249 if (device_is_attached(dev)) { 250 vtballoon_pop(sc); 251 vtballoon_stop(sc); 252 } 253 254 if (sc->vtballoon_page_frames != NULL) { 255 free(sc->vtballoon_page_frames, M_DEVBUF); 256 sc->vtballoon_page_frames = NULL; 257 } 258 259 VTBALLOON_LOCK_DESTROY(sc); 260 261 return (0); 262 } 263 264 static int 265 vtballoon_config_change(device_t dev) 266 { 267 struct vtballoon_softc *sc; 268 269 sc = device_get_softc(dev); 270 271 VTBALLOON_LOCK(sc); 272 wakeup_one(sc); 273 VTBALLOON_UNLOCK(sc); 274 275 return (1); 276 } 277 278 static int 279 vtballoon_negotiate_features(struct vtballoon_softc *sc) 280 { 281 device_t dev; 282 uint64_t features; 283 284 dev = sc->vtballoon_dev; 285 features = VTBALLOON_FEATURES; 286 287 sc->vtballoon_features = virtio_negotiate_features(dev, features); 288 return (virtio_finalize_features(dev)); 289 } 290 291 static int 292 vtballoon_setup_features(struct vtballoon_softc *sc) 293 { 294 int error; 295 296 error = vtballoon_negotiate_features(sc); 297 if (error) 298 return (error); 299 300 return (0); 301 } 302 303 static int 304 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc) 305 { 306 device_t dev; 307 struct vq_alloc_info vq_info[2]; 308 int nvqs; 309 310 dev = sc->vtballoon_dev; 311 nvqs = 2; 312 313 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc, 314 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev)); 315 316 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc, 317 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev)); 318 319 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 320 } 321 322 static void 323 vtballoon_vq_intr(void *xsc) 324 { 325 struct vtballoon_softc *sc; 326 327 sc = xsc; 328 329 VTBALLOON_LOCK(sc); 330 wakeup_one(sc); 331 VTBALLOON_UNLOCK(sc); 332 } 333 334 static void 335 vtballoon_inflate(struct vtballoon_softc *sc, int npages) 336 { 337 struct virtqueue *vq; 338 vm_page_t m; 339 int i; 340 341 vq = sc->vtballoon_inflate_vq; 342 343 if (npages > VTBALLOON_PAGES_PER_REQUEST) 344 npages = VTBALLOON_PAGES_PER_REQUEST; 345 346 for (i = 0; i < npages; i++) { 347 if ((m = vtballoon_alloc_page(sc)) == NULL) { 348 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT; 349 break; 350 } 351 352 sc->vtballoon_page_frames[i] = 353 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 354 355 KASSERT(m->a.queue == PQ_NONE, 356 ("%s: allocated page %p on queue", __func__, m)); 357 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q); 358 } 359 360 if (i > 0) 361 vtballoon_send_page_frames(sc, vq, i); 362 } 363 364 static void 365 vtballoon_deflate(struct vtballoon_softc *sc, int npages) 366 { 367 TAILQ_HEAD(, vm_page) free_pages; 368 struct virtqueue *vq; 369 vm_page_t m; 370 int i; 371 372 vq = sc->vtballoon_deflate_vq; 373 TAILQ_INIT(&free_pages); 374 375 if (npages > VTBALLOON_PAGES_PER_REQUEST) 376 npages = VTBALLOON_PAGES_PER_REQUEST; 377 378 for (i = 0; i < npages; i++) { 379 m = TAILQ_FIRST(&sc->vtballoon_pages); 380 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__)); 381 382 sc->vtballoon_page_frames[i] = 383 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 384 385 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q); 386 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q); 387 } 388 389 if (i > 0) { 390 /* Always tell host first before freeing the pages. */ 391 vtballoon_send_page_frames(sc, vq, i); 392 393 while ((m = TAILQ_FIRST(&free_pages)) != NULL) { 394 TAILQ_REMOVE(&free_pages, m, plinks.q); 395 vtballoon_free_page(sc, m); 396 } 397 } 398 399 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) && 400 sc->vtballoon_current_npages == 0) || 401 (!TAILQ_EMPTY(&sc->vtballoon_pages) && 402 sc->vtballoon_current_npages != 0), 403 ("%s: bogus page count %d", __func__, 404 sc->vtballoon_current_npages)); 405 } 406 407 static void 408 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq, 409 int npages) 410 { 411 struct sglist sg; 412 struct sglist_seg segs[1]; 413 void *c; 414 int error __diagused; 415 416 sglist_init(&sg, 1, segs); 417 418 error = sglist_append(&sg, sc->vtballoon_page_frames, 419 npages * sizeof(uint32_t)); 420 KASSERT(error == 0, ("error adding page frames to sglist")); 421 422 error = virtqueue_enqueue(vq, vq, &sg, 1, 0); 423 KASSERT(error == 0, ("error enqueuing page frames to virtqueue")); 424 virtqueue_notify(vq); 425 426 /* 427 * Inflate and deflate operations are done synchronously. The 428 * interrupt handler will wake us up. 429 */ 430 VTBALLOON_LOCK(sc); 431 while ((c = virtqueue_dequeue(vq, NULL)) == NULL) 432 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0); 433 VTBALLOON_UNLOCK(sc); 434 435 KASSERT(c == vq, ("unexpected balloon operation response")); 436 } 437 438 static void 439 vtballoon_pop(struct vtballoon_softc *sc) 440 { 441 442 while (!TAILQ_EMPTY(&sc->vtballoon_pages)) 443 vtballoon_deflate(sc, sc->vtballoon_current_npages); 444 } 445 446 static void 447 vtballoon_stop(struct vtballoon_softc *sc) 448 { 449 450 virtqueue_disable_intr(sc->vtballoon_inflate_vq); 451 virtqueue_disable_intr(sc->vtballoon_deflate_vq); 452 453 virtio_stop(sc->vtballoon_dev); 454 } 455 456 static vm_page_t 457 vtballoon_alloc_page(struct vtballoon_softc *sc) 458 { 459 vm_page_t m; 460 461 m = vm_page_alloc_noobj(VM_ALLOC_NODUMP); 462 if (m != NULL) 463 sc->vtballoon_current_npages++; 464 465 return (m); 466 } 467 468 static void 469 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m) 470 { 471 472 vm_page_free(m); 473 sc->vtballoon_current_npages--; 474 } 475 476 static uint32_t 477 vtballoon_desired_size(struct vtballoon_softc *sc) 478 { 479 uint32_t desired; 480 481 desired = virtio_read_dev_config_4(sc->vtballoon_dev, 482 offsetof(struct virtio_balloon_config, num_pages)); 483 484 if (vtballoon_modern(sc)) 485 return (desired); 486 else 487 return (le32toh(desired)); 488 } 489 490 static void 491 vtballoon_update_size(struct vtballoon_softc *sc) 492 { 493 uint32_t npages; 494 495 npages = sc->vtballoon_current_npages; 496 if (!vtballoon_modern(sc)) 497 npages = htole32(npages); 498 499 virtio_write_dev_config_4(sc->vtballoon_dev, 500 offsetof(struct virtio_balloon_config, actual), npages); 501 } 502 503 static int 504 vtballoon_sleep(struct vtballoon_softc *sc) 505 { 506 int rc, timeout; 507 uint32_t current, desired; 508 509 rc = 0; 510 current = sc->vtballoon_current_npages; 511 512 VTBALLOON_LOCK(sc); 513 for (;;) { 514 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) { 515 rc = 1; 516 break; 517 } 518 519 desired = vtballoon_desired_size(sc); 520 sc->vtballoon_desired_npages = desired; 521 522 /* 523 * If given, use non-zero timeout on the first time through 524 * the loop. On subsequent times, timeout will be zero so 525 * we will reevaluate the desired size of the balloon and 526 * break out to retry if needed. 527 */ 528 timeout = sc->vtballoon_timeout; 529 sc->vtballoon_timeout = 0; 530 531 if (current > desired) 532 break; 533 if (current < desired && timeout == 0) 534 break; 535 536 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout); 537 } 538 VTBALLOON_UNLOCK(sc); 539 540 return (rc); 541 } 542 543 static void 544 vtballoon_thread(void *xsc) 545 { 546 struct vtballoon_softc *sc; 547 uint32_t current, desired; 548 549 sc = xsc; 550 551 for (;;) { 552 if (vtballoon_sleep(sc) != 0) 553 break; 554 555 current = sc->vtballoon_current_npages; 556 desired = sc->vtballoon_desired_npages; 557 558 if (desired != current) { 559 if (desired > current) 560 vtballoon_inflate(sc, desired - current); 561 else 562 vtballoon_deflate(sc, current - desired); 563 564 vtballoon_update_size(sc); 565 } 566 } 567 568 kthread_exit(); 569 } 570 571 static void 572 vtballoon_setup_sysctl(struct vtballoon_softc *sc) 573 { 574 device_t dev; 575 struct sysctl_ctx_list *ctx; 576 struct sysctl_oid *tree; 577 struct sysctl_oid_list *child; 578 579 dev = sc->vtballoon_dev; 580 ctx = device_get_sysctl_ctx(dev); 581 tree = device_get_sysctl_tree(dev); 582 child = SYSCTL_CHILDREN(tree); 583 584 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired", 585 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t), 586 "Desired balloon size in pages"); 587 588 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current", 589 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t), 590 "Current balloon size in pages"); 591 }
Cache object: 11af6e22b0eaf4a80cbb477fcd424c8b
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]