Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/dev/hyperv/vmbus/vmbus_chan.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2009-2012,2016 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/bus.h> 34 #include <sys/callout.h> 35 #include <sys/kernel.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/mutex.h> 39 #include <sys/smp.h> 40 #include <sys/sysctl.h> 41 #include <sys/systm.h> 42 43 #include <machine/atomic.h> 44 #include <machine/stdarg.h> 45 46 #include <dev/hyperv/include/hyperv_busdma.h> 47 #include <dev/hyperv/include/vmbus_xact.h> 48 #include <dev/hyperv/vmbus/hyperv_var.h> 49 #include <dev/hyperv/vmbus/vmbus_reg.h> 50 #include <dev/hyperv/vmbus/vmbus_var.h> 51 #include <dev/hyperv/vmbus/vmbus_brvar.h> 52 #include <dev/hyperv/vmbus/vmbus_chanvar.h> 53 54 struct vmbus_chan_pollarg { 55 struct vmbus_channel *poll_chan; 56 u_int poll_hz; 57 }; 58 59 static void vmbus_chan_update_evtflagcnt( 60 struct vmbus_softc *, 61 const struct vmbus_channel *); 62 static int vmbus_chan_close_internal( 63 struct vmbus_channel *); 64 static int vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS); 65 static void vmbus_chan_sysctl_create( 66 struct vmbus_channel *); 67 static struct vmbus_channel *vmbus_chan_alloc(struct vmbus_softc *); 68 static void vmbus_chan_free(struct vmbus_channel *); 69 static int vmbus_chan_add(struct vmbus_channel *); 70 static void vmbus_chan_cpu_default(struct vmbus_channel *); 71 static int vmbus_chan_release(struct vmbus_channel *); 72 static void vmbus_chan_set_chmap(struct vmbus_channel *); 73 static void vmbus_chan_clear_chmap(struct vmbus_channel *); 74 static void vmbus_chan_detach(struct vmbus_channel *); 75 static bool vmbus_chan_wait_revoke( 76 const struct vmbus_channel *, bool); 77 static void vmbus_chan_poll_timeout(void *); 78 static bool vmbus_chan_poll_cancel_intq( 79 struct vmbus_channel *); 80 static void vmbus_chan_poll_cancel(struct vmbus_channel *); 81 82 static void vmbus_chan_ins_prilist(struct vmbus_softc *, 83 struct vmbus_channel *); 84 static void vmbus_chan_rem_prilist(struct vmbus_softc *, 85 struct vmbus_channel *); 86 static void vmbus_chan_ins_list(struct vmbus_softc *, 87 struct vmbus_channel *); 88 static void vmbus_chan_rem_list(struct vmbus_softc *, 89 struct vmbus_channel *); 90 static void vmbus_chan_ins_sublist(struct vmbus_channel *, 91 struct vmbus_channel *); 92 static void vmbus_chan_rem_sublist(struct vmbus_channel *, 93 struct vmbus_channel *); 94 95 static void vmbus_chan_task(void *, int); 96 static void vmbus_chan_task_nobatch(void *, int); 97 static void vmbus_chan_poll_task(void *, int); 98 static void vmbus_chan_clrchmap_task(void *, int); 99 static void vmbus_chan_pollcfg_task(void *, int); 100 static void vmbus_chan_polldis_task(void *, int); 101 static void vmbus_chan_poll_cancel_task(void *, int); 102 static void vmbus_prichan_attach_task(void *, int); 103 static void vmbus_subchan_attach_task(void *, int); 104 static void vmbus_prichan_detach_task(void *, int); 105 static void vmbus_subchan_detach_task(void *, int); 106 107 static void vmbus_chan_msgproc_choffer(struct vmbus_softc *, 108 const struct vmbus_message *); 109 static void vmbus_chan_msgproc_chrescind( 110 struct vmbus_softc *, 111 const struct vmbus_message *); 112 113 static int vmbus_chan_printf(const struct vmbus_channel *, 114 const char *, ...) __printflike(2, 3); 115 116 /* 117 * Vmbus channel message processing. 118 */ 119 static const vmbus_chanmsg_proc_t 120 vmbus_chan_msgprocs[VMBUS_CHANMSG_TYPE_MAX] = { 121 VMBUS_CHANMSG_PROC(CHOFFER, vmbus_chan_msgproc_choffer), 122 VMBUS_CHANMSG_PROC(CHRESCIND, vmbus_chan_msgproc_chrescind), 123 124 VMBUS_CHANMSG_PROC_WAKEUP(CHOPEN_RESP), 125 VMBUS_CHANMSG_PROC_WAKEUP(GPADL_CONNRESP), 126 VMBUS_CHANMSG_PROC_WAKEUP(GPADL_DISCONNRESP) 127 }; 128 129 /* 130 * Notify host that there are data pending on our TX bufring or 131 * we have put some data on the TX bufring. 132 */ 133 static __inline void 134 vmbus_chan_signal(const struct vmbus_channel *chan) 135 { 136 atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask); 137 if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 138 atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask); 139 else 140 hypercall_signal_event(chan->ch_monprm_dma.hv_paddr); 141 } 142 143 static __inline void 144 vmbus_chan_signal_tx(struct vmbus_channel *chan) 145 { 146 chan->ch_txbr.txbr_intrcnt ++; 147 148 vmbus_chan_signal(chan); 149 } 150 151 static __inline void 152 vmbus_chan_signal_rx(struct vmbus_channel *chan) 153 { 154 chan->ch_rxbr.rxbr_intrcnt ++; 155 156 vmbus_chan_signal(chan); 157 } 158 159 static void 160 vmbus_chan_ins_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 161 { 162 163 mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 164 if (atomic_testandset_int(&chan->ch_stflags, 165 VMBUS_CHAN_ST_ONPRIL_SHIFT)) 166 panic("channel is already on the prilist"); 167 TAILQ_INSERT_TAIL(&sc->vmbus_prichans, chan, ch_prilink); 168 } 169 170 static void 171 vmbus_chan_rem_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan) 172 { 173 174 mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED); 175 if (atomic_testandclear_int(&chan->ch_stflags, 176 VMBUS_CHAN_ST_ONPRIL_SHIFT) == 0) 177 panic("channel is not on the prilist"); 178 TAILQ_REMOVE(&sc->vmbus_prichans, chan, ch_prilink); 179 } 180 181 static void 182 vmbus_chan_ins_sublist(struct vmbus_channel *prichan, 183 struct vmbus_channel *chan) 184 { 185 186 mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 187 188 if (atomic_testandset_int(&chan->ch_stflags, 189 VMBUS_CHAN_ST_ONSUBL_SHIFT)) 190 panic("channel is already on the sublist"); 191 TAILQ_INSERT_TAIL(&prichan->ch_subchans, chan, ch_sublink); 192 193 /* Bump sub-channel count. */ 194 prichan->ch_subchan_cnt++; 195 } 196 197 static void 198 vmbus_chan_rem_sublist(struct vmbus_channel *prichan, 199 struct vmbus_channel *chan) 200 { 201 202 mtx_assert(&prichan->ch_subchan_lock, MA_OWNED); 203 204 KASSERT(prichan->ch_subchan_cnt > 0, 205 ("invalid subchan_cnt %d", prichan->ch_subchan_cnt)); 206 prichan->ch_subchan_cnt--; 207 208 if (atomic_testandclear_int(&chan->ch_stflags, 209 VMBUS_CHAN_ST_ONSUBL_SHIFT) == 0) 210 panic("channel is not on the sublist"); 211 TAILQ_REMOVE(&prichan->ch_subchans, chan, ch_sublink); 212 } 213 214 static void 215 vmbus_chan_ins_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 216 { 217 218 mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 219 if (atomic_testandset_int(&chan->ch_stflags, 220 VMBUS_CHAN_ST_ONLIST_SHIFT)) 221 panic("channel is already on the list"); 222 TAILQ_INSERT_TAIL(&sc->vmbus_chans, chan, ch_link); 223 } 224 225 static void 226 vmbus_chan_rem_list(struct vmbus_softc *sc, struct vmbus_channel *chan) 227 { 228 229 mtx_assert(&sc->vmbus_chan_lock, MA_OWNED); 230 if (atomic_testandclear_int(&chan->ch_stflags, 231 VMBUS_CHAN_ST_ONLIST_SHIFT) == 0) 232 panic("channel is not on the list"); 233 TAILQ_REMOVE(&sc->vmbus_chans, chan, ch_link); 234 } 235 236 static int 237 vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS) 238 { 239 struct vmbus_channel *chan = arg1; 240 int mnf = 0; 241 242 if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF) 243 mnf = 1; 244 return sysctl_handle_int(oidp, &mnf, 0, req); 245 } 246 247 static void 248 vmbus_chan_sysctl_create(struct vmbus_channel *chan) 249 { 250 struct sysctl_oid *ch_tree, *chid_tree, *br_tree; 251 struct sysctl_ctx_list *ctx; 252 uint32_t ch_id; 253 char name[16]; 254 255 /* 256 * Add sysctl nodes related to this channel to this 257 * channel's sysctl ctx, so that they can be destroyed 258 * independently upon close of this channel, which can 259 * happen even if the device is not detached. 260 */ 261 ctx = &chan->ch_sysctl_ctx; 262 sysctl_ctx_init(ctx); 263 264 /* 265 * Create dev.NAME.UNIT.channel tree. 266 */ 267 ch_tree = SYSCTL_ADD_NODE(ctx, 268 SYSCTL_CHILDREN(device_get_sysctl_tree(chan->ch_dev)), 269 OID_AUTO, "channel", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 270 if (ch_tree == NULL) 271 return; 272 273 /* 274 * Create dev.NAME.UNIT.channel.CHANID tree. 275 */ 276 if (VMBUS_CHAN_ISPRIMARY(chan)) 277 ch_id = chan->ch_id; 278 else 279 ch_id = chan->ch_prichan->ch_id; 280 snprintf(name, sizeof(name), "%d", ch_id); 281 chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 282 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 283 if (chid_tree == NULL) 284 return; 285 286 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 287 /* 288 * Create dev.NAME.UNIT.channel.CHANID.sub tree. 289 */ 290 ch_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), 291 OID_AUTO, "sub", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 292 if (ch_tree == NULL) 293 return; 294 295 /* 296 * Create dev.NAME.UNIT.channel.CHANID.sub.SUBIDX tree. 297 * 298 * NOTE: 299 * chid_tree is changed to this new sysctl tree. 300 */ 301 snprintf(name, sizeof(name), "%d", chan->ch_subidx); 302 chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree), 303 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 304 if (chid_tree == NULL) 305 return; 306 307 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 308 "chanid", CTLFLAG_RD, &chan->ch_id, 0, "channel id"); 309 } 310 311 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 312 "cpu", CTLFLAG_RD, &chan->ch_cpuid, 0, "owner CPU id"); 313 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 314 "mnf", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, 315 chan, 0, vmbus_chan_sysctl_mnf, "I", 316 "has monitor notification facilities"); 317 318 br_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO, 319 "br", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 320 if (br_tree != NULL) { 321 /* 322 * Create sysctl tree for RX bufring. 323 */ 324 vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx"); 325 /* 326 * Create sysctl tree for TX bufring. 327 */ 328 vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx"); 329 } 330 } 331 332 int 333 vmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size, 334 const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 335 { 336 struct vmbus_chan_br cbr; 337 int error; 338 339 /* 340 * Allocate the TX+RX bufrings. 341 */ 342 KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated")); 343 chan->ch_bufring = hyperv_dmamem_alloc(bus_get_dma_tag(chan->ch_dev), 344 PAGE_SIZE, 0, txbr_size + rxbr_size, &chan->ch_bufring_dma, 345 BUS_DMA_WAITOK); 346 if (chan->ch_bufring == NULL) { 347 vmbus_chan_printf(chan, "bufring allocation failed\n"); 348 return (ENOMEM); 349 } 350 351 cbr.cbr = chan->ch_bufring; 352 cbr.cbr_paddr = chan->ch_bufring_dma.hv_paddr; 353 cbr.cbr_txsz = txbr_size; 354 cbr.cbr_rxsz = rxbr_size; 355 356 error = vmbus_chan_open_br(chan, &cbr, udata, udlen, cb, cbarg); 357 if (error) { 358 if (error == EISCONN) { 359 /* 360 * XXX 361 * The bufring GPADL is still connected; abandon 362 * this bufring, instead of having mysterious 363 * crash or trashed data later on. 364 */ 365 vmbus_chan_printf(chan, "chan%u bufring GPADL " 366 "is still connected upon channel open error; " 367 "leak %d bytes memory\n", chan->ch_id, 368 txbr_size + rxbr_size); 369 } else { 370 hyperv_dmamem_free(&chan->ch_bufring_dma, 371 chan->ch_bufring); 372 } 373 chan->ch_bufring = NULL; 374 } 375 return (error); 376 } 377 378 int 379 vmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr, 380 const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg) 381 { 382 struct vmbus_softc *sc = chan->ch_vmbus; 383 const struct vmbus_message *msg; 384 struct vmbus_chanmsg_chopen *req; 385 struct vmbus_msghc *mh; 386 uint32_t status; 387 int error, txbr_size, rxbr_size; 388 task_fn_t *task_fn; 389 uint8_t *br; 390 391 if (udlen > VMBUS_CHANMSG_CHOPEN_UDATA_SIZE) { 392 vmbus_chan_printf(chan, 393 "invalid udata len %d for chan%u\n", udlen, chan->ch_id); 394 return (EINVAL); 395 } 396 397 br = cbr->cbr; 398 txbr_size = cbr->cbr_txsz; 399 rxbr_size = cbr->cbr_rxsz; 400 KASSERT((txbr_size & PAGE_MASK) == 0, 401 ("send bufring size is not multiple page")); 402 KASSERT((rxbr_size & PAGE_MASK) == 0, 403 ("recv bufring size is not multiple page")); 404 KASSERT((cbr->cbr_paddr & PAGE_MASK) == 0, 405 ("bufring is not page aligned")); 406 407 /* 408 * Zero out the TX/RX bufrings, in case that they were used before. 409 */ 410 memset(br, 0, txbr_size + rxbr_size); 411 412 if (atomic_testandset_int(&chan->ch_stflags, 413 VMBUS_CHAN_ST_OPENED_SHIFT)) 414 panic("double-open chan%u", chan->ch_id); 415 416 chan->ch_cb = cb; 417 chan->ch_cbarg = cbarg; 418 419 vmbus_chan_update_evtflagcnt(sc, chan); 420 421 chan->ch_tq = VMBUS_PCPU_GET(chan->ch_vmbus, event_tq, chan->ch_cpuid); 422 if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 423 task_fn = vmbus_chan_task; 424 else 425 task_fn = vmbus_chan_task_nobatch; 426 TASK_INIT(&chan->ch_task, 0, task_fn, chan); 427 428 /* TX bufring comes first */ 429 vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size); 430 /* RX bufring immediately follows TX bufring */ 431 vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size); 432 433 /* Create sysctl tree for this channel */ 434 vmbus_chan_sysctl_create(chan); 435 436 /* 437 * Connect the bufrings, both RX and TX, to this channel. 438 */ 439 error = vmbus_chan_gpadl_connect(chan, cbr->cbr_paddr, 440 txbr_size + rxbr_size, &chan->ch_bufring_gpadl); 441 if (error) { 442 vmbus_chan_printf(chan, 443 "failed to connect bufring GPADL to chan%u\n", chan->ch_id); 444 goto failed; 445 } 446 447 /* 448 * Install this channel, before it is opened, but after everything 449 * else has been setup. 450 */ 451 vmbus_chan_set_chmap(chan); 452 453 /* 454 * Open channel w/ the bufring GPADL on the target CPU. 455 */ 456 mh = vmbus_msghc_get(sc, sizeof(*req)); 457 if (mh == NULL) { 458 vmbus_chan_printf(chan, 459 "can not get msg hypercall for chopen(chan%u)\n", 460 chan->ch_id); 461 error = ENXIO; 462 goto failed; 463 } 464 465 req = vmbus_msghc_dataptr(mh); 466 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHOPEN; 467 req->chm_chanid = chan->ch_id; 468 req->chm_openid = chan->ch_id; 469 req->chm_gpadl = chan->ch_bufring_gpadl; 470 req->chm_vcpuid = chan->ch_vcpuid; 471 req->chm_txbr_pgcnt = txbr_size >> PAGE_SHIFT; 472 if (udlen > 0) 473 memcpy(req->chm_udata, udata, udlen); 474 475 error = vmbus_msghc_exec(sc, mh); 476 if (error) { 477 vmbus_chan_printf(chan, 478 "chopen(chan%u) msg hypercall exec failed: %d\n", 479 chan->ch_id, error); 480 vmbus_msghc_put(sc, mh); 481 goto failed; 482 } 483 484 for (;;) { 485 msg = vmbus_msghc_poll_result(sc, mh); 486 if (msg != NULL) 487 break; 488 if (vmbus_chan_is_revoked(chan)) { 489 int i; 490 491 /* 492 * NOTE: 493 * Hypervisor does _not_ send response CHOPEN to 494 * a revoked channel. 495 */ 496 vmbus_chan_printf(chan, 497 "chan%u is revoked, when it is being opened\n", 498 chan->ch_id); 499 500 /* 501 * XXX 502 * Add extra delay before cancel the hypercall 503 * execution; mainly to close any possible 504 * CHRESCIND and CHOPEN_RESP races on the 505 * hypervisor side. 506 */ 507 #define REVOKE_LINGER 100 508 for (i = 0; i < REVOKE_LINGER; ++i) { 509 msg = vmbus_msghc_poll_result(sc, mh); 510 if (msg != NULL) 511 break; 512 pause("rchopen", 1); 513 } 514 #undef REVOKE_LINGER 515 if (msg == NULL) 516 vmbus_msghc_exec_cancel(sc, mh); 517 break; 518 } 519 pause("chopen", 1); 520 } 521 if (msg != NULL) { 522 status = ((const struct vmbus_chanmsg_chopen_resp *) 523 msg->msg_data)->chm_status; 524 } else { 525 /* XXX any non-0 value is ok here. */ 526 status = 0xff; 527 } 528 529 vmbus_msghc_put(sc, mh); 530 531 if (status == 0) { 532 if (bootverbose) 533 vmbus_chan_printf(chan, "chan%u opened\n", chan->ch_id); 534 return (0); 535 } 536 537 vmbus_chan_printf(chan, "failed to open chan%u\n", chan->ch_id); 538 error = ENXIO; 539 540 failed: 541 sysctl_ctx_free(&chan->ch_sysctl_ctx); 542 vmbus_chan_clear_chmap(chan); 543 if (chan->ch_bufring_gpadl != 0) { 544 int error1; 545 546 error1 = vmbus_chan_gpadl_disconnect(chan, 547 chan->ch_bufring_gpadl); 548 if (error1) { 549 /* 550 * Give caller a hint that the bufring GPADL is still 551 * connected. 552 */ 553 error = EISCONN; 554 } 555 chan->ch_bufring_gpadl = 0; 556 } 557 atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED); 558 return (error); 559 } 560 561 int 562 vmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr, 563 int size, uint32_t *gpadl0) 564 { 565 struct vmbus_softc *sc = chan->ch_vmbus; 566 struct vmbus_msghc *mh; 567 struct vmbus_chanmsg_gpadl_conn *req; 568 const struct vmbus_message *msg; 569 size_t reqsz; 570 uint32_t gpadl, status; 571 int page_count, range_len, i, cnt, error; 572 uint64_t page_id; 573 574 KASSERT(*gpadl0 == 0, ("GPADL is not zero")); 575 576 /* 577 * Preliminary checks. 578 */ 579 580 KASSERT((size & PAGE_MASK) == 0, 581 ("invalid GPA size %d, not multiple page size", size)); 582 page_count = size >> PAGE_SHIFT; 583 584 KASSERT((paddr & PAGE_MASK) == 0, 585 ("GPA is not page aligned %jx", (uintmax_t)paddr)); 586 page_id = paddr >> PAGE_SHIFT; 587 588 range_len = __offsetof(struct vmbus_gpa_range, gpa_page[page_count]); 589 /* 590 * We don't support multiple GPA ranges. 591 */ 592 if (range_len > UINT16_MAX) { 593 vmbus_chan_printf(chan, "GPA too large, %d pages\n", 594 page_count); 595 return EOPNOTSUPP; 596 } 597 598 /* 599 * Allocate GPADL id. 600 */ 601 gpadl = vmbus_gpadl_alloc(sc); 602 603 /* 604 * Connect this GPADL to the target channel. 605 * 606 * NOTE: 607 * Since each message can only hold small set of page 608 * addresses, several messages may be required to 609 * complete the connection. 610 */ 611 if (page_count > VMBUS_CHANMSG_GPADL_CONN_PGMAX) 612 cnt = VMBUS_CHANMSG_GPADL_CONN_PGMAX; 613 else 614 cnt = page_count; 615 page_count -= cnt; 616 617 reqsz = __offsetof(struct vmbus_chanmsg_gpadl_conn, 618 chm_range.gpa_page[cnt]); 619 mh = vmbus_msghc_get(sc, reqsz); 620 if (mh == NULL) { 621 vmbus_chan_printf(chan, 622 "can not get msg hypercall for gpadl_conn(chan%u)\n", 623 chan->ch_id); 624 return EIO; 625 } 626 627 req = vmbus_msghc_dataptr(mh); 628 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_CONN; 629 req->chm_chanid = chan->ch_id; 630 req->chm_gpadl = gpadl; 631 req->chm_range_len = range_len; 632 req->chm_range_cnt = 1; 633 req->chm_range.gpa_len = size; 634 req->chm_range.gpa_ofs = 0; 635 for (i = 0; i < cnt; ++i) 636 req->chm_range.gpa_page[i] = page_id++; 637 638 error = vmbus_msghc_exec(sc, mh); 639 if (error) { 640 vmbus_chan_printf(chan, 641 "gpadl_conn(chan%u) msg hypercall exec failed: %d\n", 642 chan->ch_id, error); 643 vmbus_msghc_put(sc, mh); 644 return error; 645 } 646 647 while (page_count > 0) { 648 struct vmbus_chanmsg_gpadl_subconn *subreq; 649 650 if (page_count > VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX) 651 cnt = VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX; 652 else 653 cnt = page_count; 654 page_count -= cnt; 655 656 reqsz = __offsetof(struct vmbus_chanmsg_gpadl_subconn, 657 chm_gpa_page[cnt]); 658 vmbus_msghc_reset(mh, reqsz); 659 660 subreq = vmbus_msghc_dataptr(mh); 661 subreq->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_SUBCONN; 662 subreq->chm_gpadl = gpadl; 663 for (i = 0; i < cnt; ++i) 664 subreq->chm_gpa_page[i] = page_id++; 665 666 vmbus_msghc_exec_noresult(mh); 667 } 668 KASSERT(page_count == 0, ("invalid page count %d", page_count)); 669 670 msg = vmbus_msghc_wait_result(sc, mh); 671 status = ((const struct vmbus_chanmsg_gpadl_connresp *) 672 msg->msg_data)->chm_status; 673 674 vmbus_msghc_put(sc, mh); 675 676 if (status != 0) { 677 vmbus_chan_printf(chan, "gpadl_conn(chan%u) failed: %u\n", 678 chan->ch_id, status); 679 return EIO; 680 } 681 682 /* Done; commit the GPADL id. */ 683 *gpadl0 = gpadl; 684 if (bootverbose) { 685 vmbus_chan_printf(chan, "gpadl_conn(chan%u) succeeded\n", 686 chan->ch_id); 687 } 688 return 0; 689 } 690 691 static bool 692 vmbus_chan_wait_revoke(const struct vmbus_channel *chan, bool can_sleep) 693 { 694 #define WAIT_COUNT 200 /* 200ms */ 695 696 int i; 697 698 for (i = 0; i < WAIT_COUNT; ++i) { 699 if (vmbus_chan_is_revoked(chan)) 700 return (true); 701 if (can_sleep) 702 pause("wchrev", 1); 703 else 704 DELAY(1000); 705 } 706 return (false); 707 708 #undef WAIT_COUNT 709 } 710 711 /* 712 * Disconnect the GPA from the target channel 713 */ 714 int 715 vmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl) 716 { 717 struct vmbus_softc *sc = chan->ch_vmbus; 718 struct vmbus_msghc *mh; 719 struct vmbus_chanmsg_gpadl_disconn *req; 720 int error; 721 722 KASSERT(gpadl != 0, ("GPADL is zero")); 723 724 mh = vmbus_msghc_get(sc, sizeof(*req)); 725 if (mh == NULL) { 726 vmbus_chan_printf(chan, 727 "can not get msg hypercall for gpadl_disconn(chan%u)\n", 728 chan->ch_id); 729 return (EBUSY); 730 } 731 732 req = vmbus_msghc_dataptr(mh); 733 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_DISCONN; 734 req->chm_chanid = chan->ch_id; 735 req->chm_gpadl = gpadl; 736 737 error = vmbus_msghc_exec(sc, mh); 738 if (error) { 739 vmbus_msghc_put(sc, mh); 740 741 if (vmbus_chan_wait_revoke(chan, true)) { 742 /* 743 * Error is benign; this channel is revoked, 744 * so this GPADL will not be touched anymore. 745 */ 746 vmbus_chan_printf(chan, 747 "gpadl_disconn(revoked chan%u) msg hypercall " 748 "exec failed: %d\n", chan->ch_id, error); 749 return (0); 750 } 751 vmbus_chan_printf(chan, 752 "gpadl_disconn(chan%u) msg hypercall exec failed: %d\n", 753 chan->ch_id, error); 754 return (error); 755 } 756 757 vmbus_msghc_wait_result(sc, mh); 758 /* Discard result; no useful information */ 759 vmbus_msghc_put(sc, mh); 760 761 return (0); 762 } 763 764 static void 765 vmbus_chan_detach(struct vmbus_channel *chan) 766 { 767 int refs; 768 769 KASSERT(chan->ch_refs > 0, ("chan%u: invalid refcnt %d", 770 chan->ch_id, chan->ch_refs)); 771 refs = atomic_fetchadd_int(&chan->ch_refs, -1); 772 #ifdef INVARIANTS 773 if (VMBUS_CHAN_ISPRIMARY(chan)) { 774 KASSERT(refs == 1, ("chan%u: invalid refcnt %d for prichan", 775 chan->ch_id, refs + 1)); 776 } 777 #endif 778 if (refs == 1) { 779 /* 780 * Detach the target channel. 781 */ 782 if (bootverbose) { 783 vmbus_chan_printf(chan, "chan%u detached\n", 784 chan->ch_id); 785 } 786 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 787 } 788 } 789 790 static void 791 vmbus_chan_clrchmap_task(void *xchan, int pending __unused) 792 { 793 struct vmbus_channel *chan = xchan; 794 795 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 796 } 797 798 static void 799 vmbus_chan_clear_chmap(struct vmbus_channel *chan) 800 { 801 struct task chmap_task; 802 803 TASK_INIT(&chmap_task, 0, vmbus_chan_clrchmap_task, chan); 804 vmbus_chan_run_task(chan, &chmap_task); 805 } 806 807 static void 808 vmbus_chan_set_chmap(struct vmbus_channel *chan) 809 { 810 __compiler_membar(); 811 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 812 } 813 814 static void 815 vmbus_chan_poll_cancel_task(void *xchan, int pending __unused) 816 { 817 818 vmbus_chan_poll_cancel_intq(xchan); 819 } 820 821 static void 822 vmbus_chan_poll_cancel(struct vmbus_channel *chan) 823 { 824 struct task poll_cancel; 825 826 TASK_INIT(&poll_cancel, 0, vmbus_chan_poll_cancel_task, chan); 827 vmbus_chan_run_task(chan, &poll_cancel); 828 } 829 830 static int 831 vmbus_chan_close_internal(struct vmbus_channel *chan) 832 { 833 struct vmbus_softc *sc = chan->ch_vmbus; 834 struct vmbus_msghc *mh; 835 struct vmbus_chanmsg_chclose *req; 836 uint32_t old_stflags; 837 int error; 838 839 /* 840 * NOTE: 841 * Sub-channels are closed upon their primary channel closing, 842 * so they can be closed even before they are opened. 843 */ 844 for (;;) { 845 old_stflags = chan->ch_stflags; 846 if (atomic_cmpset_int(&chan->ch_stflags, old_stflags, 847 old_stflags & ~VMBUS_CHAN_ST_OPENED)) 848 break; 849 } 850 if ((old_stflags & VMBUS_CHAN_ST_OPENED) == 0) { 851 /* Not opened yet; done */ 852 if (bootverbose) { 853 vmbus_chan_printf(chan, "chan%u not opened\n", 854 chan->ch_id); 855 } 856 return (0); 857 } 858 859 /* 860 * Free this channel's sysctl tree attached to its device's 861 * sysctl tree. 862 */ 863 sysctl_ctx_free(&chan->ch_sysctl_ctx); 864 865 /* 866 * Cancel polling, if it is enabled. 867 */ 868 vmbus_chan_poll_cancel(chan); 869 870 /* 871 * NOTE: 872 * Order is critical. This channel _must_ be uninstalled first, 873 * else the channel task may be enqueued by the IDT after it has 874 * been drained. 875 */ 876 vmbus_chan_clear_chmap(chan); 877 taskqueue_drain(chan->ch_tq, &chan->ch_task); 878 chan->ch_tq = NULL; 879 880 /* 881 * Close this channel. 882 */ 883 mh = vmbus_msghc_get(sc, sizeof(*req)); 884 if (mh == NULL) { 885 vmbus_chan_printf(chan, 886 "can not get msg hypercall for chclose(chan%u)\n", 887 chan->ch_id); 888 error = ENXIO; 889 goto disconnect; 890 } 891 892 req = vmbus_msghc_dataptr(mh); 893 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHCLOSE; 894 req->chm_chanid = chan->ch_id; 895 896 error = vmbus_msghc_exec_noresult(mh); 897 vmbus_msghc_put(sc, mh); 898 899 if (error) { 900 vmbus_chan_printf(chan, 901 "chclose(chan%u) msg hypercall exec failed: %d\n", 902 chan->ch_id, error); 903 goto disconnect; 904 } 905 906 if (bootverbose) 907 vmbus_chan_printf(chan, "chan%u closed\n", chan->ch_id); 908 909 disconnect: 910 /* 911 * Disconnect the TX+RX bufrings from this channel. 912 */ 913 if (chan->ch_bufring_gpadl != 0) { 914 int error1; 915 916 error1 = vmbus_chan_gpadl_disconnect(chan, 917 chan->ch_bufring_gpadl); 918 if (error1) { 919 /* 920 * XXX 921 * The bufring GPADL is still connected; abandon 922 * this bufring, instead of having mysterious 923 * crash or trashed data later on. 924 */ 925 vmbus_chan_printf(chan, "chan%u bufring GPADL " 926 "is still connected after close\n", chan->ch_id); 927 chan->ch_bufring = NULL; 928 /* 929 * Give caller a hint that the bufring GPADL is 930 * still connected. 931 */ 932 error = EISCONN; 933 } 934 chan->ch_bufring_gpadl = 0; 935 } 936 937 /* 938 * Destroy the TX+RX bufrings. 939 */ 940 if (chan->ch_bufring != NULL) { 941 hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring); 942 chan->ch_bufring = NULL; 943 } 944 return (error); 945 } 946 947 int 948 vmbus_chan_close_direct(struct vmbus_channel *chan) 949 { 950 int error; 951 952 #ifdef INVARIANTS 953 if (VMBUS_CHAN_ISPRIMARY(chan)) { 954 struct vmbus_channel *subchan; 955 956 /* 957 * All sub-channels _must_ have been closed, or are _not_ 958 * opened at all. 959 */ 960 mtx_lock(&chan->ch_subchan_lock); 961 TAILQ_FOREACH(subchan, &chan->ch_subchans, ch_sublink) { 962 KASSERT( 963 (subchan->ch_stflags & VMBUS_CHAN_ST_OPENED) == 0, 964 ("chan%u: subchan%u is still opened", 965 chan->ch_id, subchan->ch_subidx)); 966 } 967 mtx_unlock(&chan->ch_subchan_lock); 968 } 969 #endif 970 971 error = vmbus_chan_close_internal(chan); 972 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 973 /* 974 * This sub-channel is referenced, when it is linked to 975 * the primary channel; drop that reference now. 976 */ 977 vmbus_chan_detach(chan); 978 } 979 return (error); 980 } 981 982 /* 983 * Caller should make sure that all sub-channels have 984 * been added to 'chan' and all to-be-closed channels 985 * are not being opened. 986 */ 987 void 988 vmbus_chan_close(struct vmbus_channel *chan) 989 { 990 int subchan_cnt; 991 992 if (!VMBUS_CHAN_ISPRIMARY(chan)) { 993 /* 994 * Sub-channel is closed when its primary channel 995 * is closed; done. 996 */ 997 return; 998 } 999 1000 /* 1001 * Close all sub-channels, if any. 1002 */ 1003 subchan_cnt = chan->ch_subchan_cnt; 1004 if (subchan_cnt > 0) { 1005 struct vmbus_channel **subchan; 1006 int i; 1007 1008 subchan = vmbus_subchan_get(chan, subchan_cnt); 1009 for (i = 0; i < subchan_cnt; ++i) { 1010 vmbus_chan_close_internal(subchan[i]); 1011 /* 1012 * This sub-channel is referenced, when it is 1013 * linked to the primary channel; drop that 1014 * reference now. 1015 */ 1016 vmbus_chan_detach(subchan[i]); 1017 } 1018 vmbus_subchan_rel(subchan, subchan_cnt); 1019 } 1020 1021 /* Then close the primary channel. */ 1022 vmbus_chan_close_internal(chan); 1023 } 1024 1025 void 1026 vmbus_chan_intr_drain(struct vmbus_channel *chan) 1027 { 1028 1029 taskqueue_drain(chan->ch_tq, &chan->ch_task); 1030 } 1031 1032 uint32_t 1033 vmbus_chan_write_available(struct vmbus_channel *chan) 1034 { 1035 return (vmbus_txbr_available(&chan->ch_txbr)); 1036 } 1037 1038 bool 1039 vmbus_chan_write_signal(struct vmbus_channel *chan, 1040 int32_t min_signal_size) 1041 { 1042 if (min_signal_size >= 0 && 1043 vmbus_chan_write_available(chan) > min_signal_size) { 1044 return false; 1045 } 1046 1047 if (!vmbus_txbr_get_imask(&chan->ch_txbr)) { 1048 /* txbr imask is not set, signal the reader */ 1049 vmbus_chan_signal_tx(chan); 1050 return true; 1051 } 1052 1053 return false; 1054 } 1055 1056 void 1057 vmbus_chan_set_pending_send_size(struct vmbus_channel *chan, 1058 uint32_t size) 1059 { 1060 if (chan) 1061 vmbus_txbr_set_pending_snd_sz(&chan->ch_txbr, size); 1062 } 1063 1064 int 1065 vmbus_chan_iov_send(struct vmbus_channel *chan, 1066 const struct iovec iov[], int iovlen, 1067 vmbus_br_copy_callback_t cb, void *cbarg) 1068 { 1069 int error; 1070 boolean_t send_evt; 1071 1072 if (iovlen == 0) 1073 return (0); 1074 1075 error = vmbus_txbr_write_call(&chan->ch_txbr, iov, iovlen, 1076 cb, cbarg, &send_evt); 1077 1078 if (!error && send_evt) { 1079 vmbus_chan_signal_tx(chan); 1080 } 1081 1082 return error; 1083 } 1084 1085 int 1086 vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags, 1087 void *data, int dlen, uint64_t xactid) 1088 { 1089 struct vmbus_chanpkt pkt; 1090 int pktlen, pad_pktlen, hlen, error; 1091 uint64_t pad = 0; 1092 struct iovec iov[3]; 1093 boolean_t send_evt; 1094 1095 hlen = sizeof(pkt); 1096 pktlen = hlen + dlen; 1097 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1098 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1099 ("invalid packet size %d", pad_pktlen)); 1100 1101 pkt.cp_hdr.cph_type = type; 1102 pkt.cp_hdr.cph_flags = flags; 1103 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1104 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1105 pkt.cp_hdr.cph_xactid = xactid; 1106 1107 iov[0].iov_base = &pkt; 1108 iov[0].iov_len = hlen; 1109 iov[1].iov_base = data; 1110 iov[1].iov_len = dlen; 1111 iov[2].iov_base = &pad; 1112 iov[2].iov_len = pad_pktlen - pktlen; 1113 1114 error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt); 1115 if (!error && send_evt) 1116 vmbus_chan_signal_tx(chan); 1117 return error; 1118 } 1119 1120 int 1121 vmbus_chan_send_sglist(struct vmbus_channel *chan, 1122 struct vmbus_gpa sg[], int sglen, void *data, int dlen, uint64_t xactid) 1123 { 1124 struct vmbus_chanpkt_sglist pkt; 1125 int pktlen, pad_pktlen, hlen, error; 1126 struct iovec iov[4]; 1127 boolean_t send_evt; 1128 uint64_t pad = 0; 1129 1130 hlen = __offsetof(struct vmbus_chanpkt_sglist, cp_gpa[sglen]); 1131 pktlen = hlen + dlen; 1132 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1133 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1134 ("invalid packet size %d", pad_pktlen)); 1135 1136 pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1137 pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1138 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1139 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1140 pkt.cp_hdr.cph_xactid = xactid; 1141 pkt.cp_rsvd = 0; 1142 pkt.cp_gpa_cnt = sglen; 1143 1144 iov[0].iov_base = &pkt; 1145 iov[0].iov_len = sizeof(pkt); 1146 iov[1].iov_base = sg; 1147 iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen; 1148 iov[2].iov_base = data; 1149 iov[2].iov_len = dlen; 1150 iov[3].iov_base = &pad; 1151 iov[3].iov_len = pad_pktlen - pktlen; 1152 1153 error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1154 if (!error && send_evt) 1155 vmbus_chan_signal_tx(chan); 1156 return error; 1157 } 1158 1159 int 1160 vmbus_chan_send_prplist(struct vmbus_channel *chan, 1161 struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen, 1162 uint64_t xactid) 1163 { 1164 struct vmbus_chanpkt_prplist pkt; 1165 int pktlen, pad_pktlen, hlen, error; 1166 struct iovec iov[4]; 1167 boolean_t send_evt; 1168 uint64_t pad = 0; 1169 1170 hlen = __offsetof(struct vmbus_chanpkt_prplist, 1171 cp_range[0].gpa_page[prp_cnt]); 1172 pktlen = hlen + dlen; 1173 pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); 1174 KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), 1175 ("invalid packet size %d", pad_pktlen)); 1176 1177 pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1178 pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1179 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen); 1180 VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen); 1181 pkt.cp_hdr.cph_xactid = xactid; 1182 pkt.cp_rsvd = 0; 1183 pkt.cp_range_cnt = 1; 1184 1185 iov[0].iov_base = &pkt; 1186 iov[0].iov_len = sizeof(pkt); 1187 iov[1].iov_base = prp; 1188 iov[1].iov_len = __offsetof(struct vmbus_gpa_range, gpa_page[prp_cnt]); 1189 iov[2].iov_base = data; 1190 iov[2].iov_len = dlen; 1191 iov[3].iov_base = &pad; 1192 iov[3].iov_len = pad_pktlen - pktlen; 1193 1194 error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); 1195 if (!error && send_evt) 1196 vmbus_chan_signal_tx(chan); 1197 return error; 1198 } 1199 1200 int 1201 vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0, 1202 uint64_t *xactid) 1203 { 1204 struct vmbus_chanpkt_hdr pkt; 1205 int error, dlen, hlen; 1206 1207 error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); 1208 if (error) 1209 return (error); 1210 1211 if (__predict_false(pkt.cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1212 vmbus_chan_printf(chan, "invalid hlen %u\n", pkt.cph_hlen); 1213 /* XXX this channel is dead actually. */ 1214 return (EIO); 1215 } 1216 if (__predict_false(pkt.cph_hlen > pkt.cph_tlen)) { 1217 vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1218 pkt.cph_hlen, pkt.cph_tlen); 1219 /* XXX this channel is dead actually. */ 1220 return (EIO); 1221 } 1222 1223 hlen = VMBUS_CHANPKT_GETLEN(pkt.cph_hlen); 1224 dlen = VMBUS_CHANPKT_GETLEN(pkt.cph_tlen) - hlen; 1225 1226 if (*dlen0 < dlen) { 1227 /* Return the size of this packet's data. */ 1228 *dlen0 = dlen; 1229 return (ENOBUFS); 1230 } 1231 1232 *xactid = pkt.cph_xactid; 1233 *dlen0 = dlen; 1234 1235 /* Skip packet header */ 1236 error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen); 1237 KASSERT(!error, ("vmbus_rxbr_read failed")); 1238 1239 return (0); 1240 } 1241 1242 int 1243 vmbus_chan_recv_pkt(struct vmbus_channel *chan, 1244 struct vmbus_chanpkt_hdr *pkt, int *pktlen0) 1245 { 1246 int error, pktlen, pkt_hlen; 1247 1248 pkt_hlen = sizeof(*pkt); 1249 error = vmbus_rxbr_peek(&chan->ch_rxbr, pkt, pkt_hlen); 1250 if (error) 1251 return (error); 1252 1253 if (__predict_false(pkt->cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) { 1254 vmbus_chan_printf(chan, "invalid hlen %u\n", pkt->cph_hlen); 1255 /* XXX this channel is dead actually. */ 1256 return (EIO); 1257 } 1258 if (__predict_false(pkt->cph_hlen > pkt->cph_tlen)) { 1259 vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n", 1260 pkt->cph_hlen, pkt->cph_tlen); 1261 /* XXX this channel is dead actually. */ 1262 return (EIO); 1263 } 1264 1265 pktlen = VMBUS_CHANPKT_GETLEN(pkt->cph_tlen); 1266 if (*pktlen0 < pktlen) { 1267 /* Return the size of this packet. */ 1268 *pktlen0 = pktlen; 1269 return (ENOBUFS); 1270 } 1271 *pktlen0 = pktlen; 1272 1273 /* 1274 * Skip the fixed-size packet header, which has been filled 1275 * by the above vmbus_rxbr_peek(). 1276 */ 1277 error = vmbus_rxbr_read(&chan->ch_rxbr, pkt + 1, 1278 pktlen - pkt_hlen, pkt_hlen); 1279 KASSERT(!error, ("vmbus_rxbr_read failed")); 1280 1281 return (0); 1282 } 1283 1284 uint32_t 1285 vmbus_chan_read_available(struct vmbus_channel *chan) 1286 { 1287 return (vmbus_rxbr_available(&chan->ch_rxbr)); 1288 } 1289 1290 /* 1291 * This routine does: 1292 * - Advance the channel read index for 'advance' bytes 1293 * - Copy data_len bytes in to the buffer pointed by 'data' 1294 * Return 0 if operation succeed. EAGAIN if operations if failed. 1295 * If failed, the buffer pointed by 'data' is intact, and the 1296 * channel read index is not advanced at all. 1297 */ 1298 int 1299 vmbus_chan_recv_peek(struct vmbus_channel *chan, 1300 void *data, int data_len, uint32_t advance) 1301 { 1302 int error; 1303 boolean_t sig_event; 1304 1305 if (data == NULL || data_len <= 0) 1306 return (EINVAL); 1307 1308 error = vmbus_rxbr_idxadv_peek(&chan->ch_rxbr, 1309 data, data_len, advance, &sig_event); 1310 1311 if (!error && sig_event) { 1312 vmbus_chan_signal_rx(chan); 1313 } 1314 1315 return (error); 1316 } 1317 1318 /* 1319 * This routine does: 1320 * - Advance the channel read index for 'advance' bytes 1321 */ 1322 int 1323 vmbus_chan_recv_idxadv(struct vmbus_channel *chan, uint32_t advance) 1324 { 1325 int error; 1326 boolean_t sig_event; 1327 1328 if (advance == 0) 1329 return (EINVAL); 1330 1331 error = vmbus_rxbr_idxadv(&chan->ch_rxbr, advance, &sig_event); 1332 1333 if (!error && sig_event) { 1334 vmbus_chan_signal_rx(chan); 1335 } 1336 1337 return (error); 1338 } 1339 1340 1341 /* 1342 * Caller should hold its own lock to serialize the ring buffer 1343 * copy. 1344 */ 1345 int 1346 vmbus_chan_recv_peek_call(struct vmbus_channel *chan, int data_len, 1347 uint32_t skip, vmbus_br_copy_callback_t cb, void *cbarg) 1348 { 1349 if (!chan || data_len <= 0 || cb == NULL) 1350 return (EINVAL); 1351 1352 return (vmbus_rxbr_peek_call(&chan->ch_rxbr, data_len, skip, 1353 cb, cbarg)); 1354 } 1355 1356 static void 1357 vmbus_chan_task(void *xchan, int pending __unused) 1358 { 1359 struct vmbus_channel *chan = xchan; 1360 vmbus_chan_callback_t cb = chan->ch_cb; 1361 void *cbarg = chan->ch_cbarg; 1362 1363 KASSERT(chan->ch_poll_intvl == 0, 1364 ("chan%u: interrupted in polling mode", chan->ch_id)); 1365 1366 /* 1367 * Optimize host to guest signaling by ensuring: 1368 * 1. While reading the channel, we disable interrupts from 1369 * host. 1370 * 2. Ensure that we process all posted messages from the host 1371 * before returning from this callback. 1372 * 3. Once we return, enable signaling from the host. Once this 1373 * state is set we check to see if additional packets are 1374 * available to read. In this case we repeat the process. 1375 * 1376 * NOTE: Interrupt has been disabled in the ISR. 1377 */ 1378 for (;;) { 1379 uint32_t left; 1380 1381 cb(chan, cbarg); 1382 1383 left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1384 if (left == 0) { 1385 /* No more data in RX bufring; done */ 1386 break; 1387 } 1388 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1389 } 1390 } 1391 1392 static void 1393 vmbus_chan_task_nobatch(void *xchan, int pending __unused) 1394 { 1395 struct vmbus_channel *chan = xchan; 1396 1397 KASSERT(chan->ch_poll_intvl == 0, 1398 ("chan%u: interrupted in polling mode", chan->ch_id)); 1399 chan->ch_cb(chan, chan->ch_cbarg); 1400 } 1401 1402 static void 1403 vmbus_chan_poll_timeout(void *xchan) 1404 { 1405 struct vmbus_channel *chan = xchan; 1406 1407 KASSERT(chan->ch_poll_intvl != 0, 1408 ("chan%u: polling timeout in interrupt mode", chan->ch_id)); 1409 taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task); 1410 } 1411 1412 static void 1413 vmbus_chan_poll_task(void *xchan, int pending __unused) 1414 { 1415 struct vmbus_channel *chan = xchan; 1416 1417 KASSERT(chan->ch_poll_intvl != 0, 1418 ("chan%u: polling in interrupt mode", chan->ch_id)); 1419 callout_reset_sbt_curcpu(&chan->ch_poll_timeo, chan->ch_poll_intvl, 0, 1420 vmbus_chan_poll_timeout, chan, chan->ch_poll_flags); 1421 chan->ch_cb(chan, chan->ch_cbarg); 1422 } 1423 1424 static void 1425 vmbus_chan_pollcfg_task(void *xarg, int pending __unused) 1426 { 1427 const struct vmbus_chan_pollarg *arg = xarg; 1428 struct vmbus_channel *chan = arg->poll_chan; 1429 sbintime_t intvl; 1430 int poll_flags; 1431 1432 /* 1433 * Save polling interval. 1434 */ 1435 intvl = SBT_1S / arg->poll_hz; 1436 if (intvl == 0) 1437 intvl = 1; 1438 if (intvl == chan->ch_poll_intvl) { 1439 /* Nothing changes; done */ 1440 return; 1441 } 1442 chan->ch_poll_intvl = intvl; 1443 1444 /* Adjust callout flags. */ 1445 poll_flags = C_DIRECT_EXEC; 1446 if (arg->poll_hz <= hz) 1447 poll_flags |= C_HARDCLOCK; 1448 chan->ch_poll_flags = poll_flags; 1449 1450 /* 1451 * Disconnect this channel from the channel map to make sure that 1452 * the RX bufring interrupt enabling bit can not be touched, and 1453 * ISR can not enqueue this channel task anymore. THEN, disable 1454 * interrupt from the RX bufring (TX bufring does not generate 1455 * interrupt to VM). 1456 * 1457 * NOTE: order is critical. 1458 */ 1459 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL; 1460 __compiler_membar(); 1461 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1462 1463 /* 1464 * NOTE: 1465 * At this point, this channel task will not be enqueued by 1466 * the ISR anymore, time to cancel the pending one. 1467 */ 1468 taskqueue_cancel(chan->ch_tq, &chan->ch_task, NULL); 1469 1470 /* Kick start! */ 1471 taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task); 1472 } 1473 1474 static bool 1475 vmbus_chan_poll_cancel_intq(struct vmbus_channel *chan) 1476 { 1477 1478 if (chan->ch_poll_intvl == 0) { 1479 /* Not enabled. */ 1480 return (false); 1481 } 1482 1483 /* 1484 * Stop polling callout, so that channel polling task 1485 * will not be enqueued anymore. 1486 */ 1487 callout_drain(&chan->ch_poll_timeo); 1488 1489 /* 1490 * Disable polling by resetting polling interval. 1491 * 1492 * NOTE: 1493 * The polling interval resetting MUST be conducted 1494 * after the callout is drained; mainly to keep the 1495 * proper assertion in place. 1496 */ 1497 chan->ch_poll_intvl = 0; 1498 1499 /* 1500 * NOTE: 1501 * At this point, this channel polling task will not be 1502 * enqueued by the callout anymore, time to cancel the 1503 * pending one. 1504 */ 1505 taskqueue_cancel(chan->ch_tq, &chan->ch_poll_task, NULL); 1506 1507 /* Polling was enabled. */ 1508 return (true); 1509 } 1510 1511 static void 1512 vmbus_chan_polldis_task(void *xchan, int pending __unused) 1513 { 1514 struct vmbus_channel *chan = xchan; 1515 1516 if (!vmbus_chan_poll_cancel_intq(chan)) { 1517 /* Already disabled; done. */ 1518 return; 1519 } 1520 1521 /* 1522 * Plug this channel back to the channel map and unmask 1523 * the RX bufring interrupt. 1524 */ 1525 chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan; 1526 __compiler_membar(); 1527 vmbus_rxbr_intr_unmask(&chan->ch_rxbr); 1528 1529 /* 1530 * Kick start the interrupt task, just in case unmasking 1531 * interrupt races ISR. 1532 */ 1533 taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1534 } 1535 1536 static __inline void 1537 vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *event_flags, 1538 int flag_cnt) 1539 { 1540 int f; 1541 1542 for (f = 0; f < flag_cnt; ++f) { 1543 uint32_t chid_base; 1544 u_long flags; 1545 int chid_ofs; 1546 1547 if (event_flags[f] == 0) 1548 continue; 1549 1550 flags = atomic_swap_long(&event_flags[f], 0); 1551 chid_base = f << VMBUS_EVTFLAG_SHIFT; 1552 1553 while ((chid_ofs = ffsl(flags)) != 0) { 1554 struct vmbus_channel *chan; 1555 1556 --chid_ofs; /* NOTE: ffsl is 1-based */ 1557 flags &= ~(1UL << chid_ofs); 1558 1559 chan = sc->vmbus_chmap[chid_base + chid_ofs]; 1560 if (__predict_false(chan == NULL)) { 1561 /* Channel is closed. */ 1562 continue; 1563 } 1564 __compiler_membar(); 1565 1566 if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) 1567 vmbus_rxbr_intr_mask(&chan->ch_rxbr); 1568 taskqueue_enqueue(chan->ch_tq, &chan->ch_task); 1569 } 1570 } 1571 } 1572 1573 void 1574 vmbus_event_proc(struct vmbus_softc *sc, int cpu) 1575 { 1576 struct vmbus_evtflags *eventf; 1577 1578 /* 1579 * On Host with Win8 or above, the event page can be checked directly 1580 * to get the id of the channel that has the pending interrupt. 1581 */ 1582 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1583 vmbus_event_flags_proc(sc, eventf->evt_flags, 1584 VMBUS_PCPU_GET(sc, event_flags_cnt, cpu)); 1585 } 1586 1587 void 1588 vmbus_event_proc_compat(struct vmbus_softc *sc, int cpu) 1589 { 1590 struct vmbus_evtflags *eventf; 1591 1592 eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE; 1593 if (atomic_testandclear_long(&eventf->evt_flags[0], 0)) { 1594 vmbus_event_flags_proc(sc, sc->vmbus_rx_evtflags, 1595 VMBUS_CHAN_MAX_COMPAT >> VMBUS_EVTFLAG_SHIFT); 1596 } 1597 } 1598 1599 static void 1600 vmbus_chan_update_evtflagcnt(struct vmbus_softc *sc, 1601 const struct vmbus_channel *chan) 1602 { 1603 volatile int *flag_cnt_ptr; 1604 int flag_cnt; 1605 1606 flag_cnt = (chan->ch_id / VMBUS_EVTFLAG_LEN) + 1; 1607 flag_cnt_ptr = VMBUS_PCPU_PTR(sc, event_flags_cnt, chan->ch_cpuid); 1608 1609 for (;;) { 1610 int old_flag_cnt; 1611 1612 old_flag_cnt = *flag_cnt_ptr; 1613 if (old_flag_cnt >= flag_cnt) 1614 break; 1615 if (atomic_cmpset_int(flag_cnt_ptr, old_flag_cnt, flag_cnt)) { 1616 if (bootverbose) { 1617 vmbus_chan_printf(chan, 1618 "chan%u update cpu%d flag_cnt to %d\n", 1619 chan->ch_id, chan->ch_cpuid, flag_cnt); 1620 } 1621 break; 1622 } 1623 } 1624 } 1625 1626 static struct vmbus_channel * 1627 vmbus_chan_alloc(struct vmbus_softc *sc) 1628 { 1629 struct vmbus_channel *chan; 1630 1631 chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO); 1632 1633 chan->ch_monprm = hyperv_dmamem_alloc(bus_get_dma_tag(sc->vmbus_dev), 1634 HYPERCALL_PARAM_ALIGN, 0, sizeof(struct hyperv_mon_param), 1635 &chan->ch_monprm_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 1636 if (chan->ch_monprm == NULL) { 1637 device_printf(sc->vmbus_dev, "monprm alloc failed\n"); 1638 free(chan, M_DEVBUF); 1639 return NULL; 1640 } 1641 1642 chan->ch_refs = 1; 1643 chan->ch_vmbus = sc; 1644 mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF); 1645 sx_init(&chan->ch_orphan_lock, "vmbus chorphan"); 1646 TAILQ_INIT(&chan->ch_subchans); 1647 vmbus_rxbr_init(&chan->ch_rxbr); 1648 vmbus_txbr_init(&chan->ch_txbr); 1649 1650 TASK_INIT(&chan->ch_poll_task, 0, vmbus_chan_poll_task, chan); 1651 callout_init(&chan->ch_poll_timeo, 1); 1652 1653 return chan; 1654 } 1655 1656 static void 1657 vmbus_chan_free(struct vmbus_channel *chan) 1658 { 1659 1660 KASSERT(TAILQ_EMPTY(&chan->ch_subchans) && chan->ch_subchan_cnt == 0, 1661 ("still owns sub-channels")); 1662 KASSERT((chan->ch_stflags & 1663 (VMBUS_CHAN_ST_OPENED | 1664 VMBUS_CHAN_ST_ONPRIL | 1665 VMBUS_CHAN_ST_ONSUBL | 1666 VMBUS_CHAN_ST_ONLIST)) == 0, ("free busy channel")); 1667 KASSERT(chan->ch_orphan_xact == NULL, 1668 ("still has orphan xact installed")); 1669 KASSERT(chan->ch_refs == 0, ("chan%u: invalid refcnt %d", 1670 chan->ch_id, chan->ch_refs)); 1671 KASSERT(chan->ch_poll_intvl == 0, ("chan%u: polling is activated", 1672 chan->ch_id)); 1673 1674 hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm); 1675 mtx_destroy(&chan->ch_subchan_lock); 1676 sx_destroy(&chan->ch_orphan_lock); 1677 vmbus_rxbr_deinit(&chan->ch_rxbr); 1678 vmbus_txbr_deinit(&chan->ch_txbr); 1679 free(chan, M_DEVBUF); 1680 } 1681 1682 static int 1683 vmbus_chan_add(struct vmbus_channel *newchan) 1684 { 1685 struct vmbus_softc *sc = newchan->ch_vmbus; 1686 struct vmbus_channel *prichan; 1687 1688 if (newchan->ch_id == 0) { 1689 /* 1690 * XXX 1691 * Chan0 will neither be processed nor should be offered; 1692 * skip it. 1693 */ 1694 device_printf(sc->vmbus_dev, "got chan0 offer, discard\n"); 1695 return EINVAL; 1696 } else if (newchan->ch_id >= VMBUS_CHAN_MAX) { 1697 device_printf(sc->vmbus_dev, "invalid chan%u offer\n", 1698 newchan->ch_id); 1699 return EINVAL; 1700 } 1701 1702 mtx_lock(&sc->vmbus_prichan_lock); 1703 TAILQ_FOREACH(prichan, &sc->vmbus_prichans, ch_prilink) { 1704 /* 1705 * Sub-channel will have the same type GUID and instance 1706 * GUID as its primary channel. 1707 */ 1708 if (memcmp(&prichan->ch_guid_type, &newchan->ch_guid_type, 1709 sizeof(struct hyperv_guid)) == 0 && 1710 memcmp(&prichan->ch_guid_inst, &newchan->ch_guid_inst, 1711 sizeof(struct hyperv_guid)) == 0) 1712 break; 1713 } 1714 if (VMBUS_CHAN_ISPRIMARY(newchan)) { 1715 if (prichan == NULL) { 1716 /* Install the new primary channel */ 1717 vmbus_chan_ins_prilist(sc, newchan); 1718 mtx_unlock(&sc->vmbus_prichan_lock); 1719 goto done; 1720 } else { 1721 mtx_unlock(&sc->vmbus_prichan_lock); 1722 device_printf(sc->vmbus_dev, 1723 "duplicated primary chan%u\n", newchan->ch_id); 1724 return EINVAL; 1725 } 1726 } else { /* Sub-channel */ 1727 if (prichan == NULL) { 1728 mtx_unlock(&sc->vmbus_prichan_lock); 1729 device_printf(sc->vmbus_dev, 1730 "no primary chan for chan%u\n", newchan->ch_id); 1731 return EINVAL; 1732 } 1733 /* 1734 * Found the primary channel for this sub-channel and 1735 * move on. 1736 * 1737 * XXX refcnt prichan 1738 */ 1739 } 1740 mtx_unlock(&sc->vmbus_prichan_lock); 1741 1742 /* 1743 * This is a sub-channel; link it with the primary channel. 1744 */ 1745 KASSERT(!VMBUS_CHAN_ISPRIMARY(newchan), 1746 ("new channel is not sub-channel")); 1747 KASSERT(prichan != NULL, ("no primary channel")); 1748 1749 /* 1750 * Reference count this sub-channel; it will be dereferenced 1751 * when this sub-channel is closed. 1752 */ 1753 KASSERT(newchan->ch_refs == 1, ("chan%u: invalid refcnt %d", 1754 newchan->ch_id, newchan->ch_refs)); 1755 atomic_add_int(&newchan->ch_refs, 1); 1756 1757 newchan->ch_prichan = prichan; 1758 newchan->ch_dev = prichan->ch_dev; 1759 1760 mtx_lock(&prichan->ch_subchan_lock); 1761 vmbus_chan_ins_sublist(prichan, newchan); 1762 mtx_unlock(&prichan->ch_subchan_lock); 1763 /* 1764 * Notify anyone that is interested in this sub-channel, 1765 * after this sub-channel is setup. 1766 */ 1767 wakeup(prichan); 1768 done: 1769 /* 1770 * Hook this channel up for later revocation. 1771 */ 1772 mtx_lock(&sc->vmbus_chan_lock); 1773 vmbus_chan_ins_list(sc, newchan); 1774 mtx_unlock(&sc->vmbus_chan_lock); 1775 1776 if (bootverbose) { 1777 vmbus_chan_printf(newchan, "chan%u subidx%u offer\n", 1778 newchan->ch_id, newchan->ch_subidx); 1779 } 1780 1781 /* Select default cpu for this channel. */ 1782 vmbus_chan_cpu_default(newchan); 1783 1784 return 0; 1785 } 1786 1787 void 1788 vmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu) 1789 { 1790 KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu %d", cpu)); 1791 1792 if (chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WS2008 || 1793 chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WIN7) { 1794 /* Only cpu0 is supported */ 1795 cpu = 0; 1796 } 1797 1798 chan->ch_cpuid = cpu; 1799 chan->ch_vcpuid = VMBUS_PCPU_GET(chan->ch_vmbus, vcpuid, cpu); 1800 1801 if (bootverbose) { 1802 vmbus_chan_printf(chan, 1803 "chan%u assigned to cpu%u [vcpu%u]\n", 1804 chan->ch_id, chan->ch_cpuid, chan->ch_vcpuid); 1805 } 1806 } 1807 1808 void 1809 vmbus_chan_cpu_rr(struct vmbus_channel *chan) 1810 { 1811 static uint32_t vmbus_chan_nextcpu; 1812 int cpu; 1813 1814 cpu = atomic_fetchadd_int(&vmbus_chan_nextcpu, 1) % mp_ncpus; 1815 vmbus_chan_cpu_set(chan, cpu); 1816 } 1817 1818 static void 1819 vmbus_chan_cpu_default(struct vmbus_channel *chan) 1820 { 1821 /* 1822 * By default, pin the channel to cpu0. Devices having 1823 * special channel-cpu mapping requirement should call 1824 * vmbus_chan_cpu_{set,rr}(). 1825 */ 1826 vmbus_chan_cpu_set(chan, 0); 1827 } 1828 1829 static void 1830 vmbus_chan_msgproc_choffer(struct vmbus_softc *sc, 1831 const struct vmbus_message *msg) 1832 { 1833 const struct vmbus_chanmsg_choffer *offer; 1834 struct vmbus_channel *chan; 1835 task_fn_t *detach_fn, *attach_fn; 1836 int error; 1837 1838 offer = (const struct vmbus_chanmsg_choffer *)msg->msg_data; 1839 1840 chan = vmbus_chan_alloc(sc); 1841 if (chan == NULL) { 1842 device_printf(sc->vmbus_dev, "allocate chan%u failed\n", 1843 offer->chm_chanid); 1844 return; 1845 } 1846 1847 chan->ch_id = offer->chm_chanid; 1848 chan->ch_subidx = offer->chm_subidx; 1849 chan->ch_guid_type = offer->chm_chtype; 1850 chan->ch_guid_inst = offer->chm_chinst; 1851 1852 /* Batch reading is on by default */ 1853 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 1854 1855 chan->ch_monprm->mp_connid = VMBUS_CONNID_EVENT; 1856 if (sc->vmbus_version != VMBUS_VERSION_WS2008) 1857 chan->ch_monprm->mp_connid = offer->chm_connid; 1858 1859 if (offer->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1860 int trig_idx; 1861 1862 /* 1863 * Setup MNF stuffs. 1864 */ 1865 chan->ch_txflags |= VMBUS_CHAN_TXF_HASMNF; 1866 1867 trig_idx = offer->chm_montrig / VMBUS_MONTRIG_LEN; 1868 if (trig_idx >= VMBUS_MONTRIGS_MAX) 1869 panic("invalid monitor trigger %u", offer->chm_montrig); 1870 chan->ch_montrig = 1871 &sc->vmbus_mnf2->mnf_trigs[trig_idx].mt_pending; 1872 1873 chan->ch_montrig_mask = 1874 1 << (offer->chm_montrig % VMBUS_MONTRIG_LEN); 1875 } 1876 1877 if (offer->chm_chflags & VMBUS_CHAN_TLNPI_PROVIDER_OFFER) { 1878 /* This is HyperV socket channel */ 1879 chan->ch_is_hvs = true; 1880 /* The first byte != 0 means the host initiated connection. */ 1881 chan->ch_hvs_conn_from_host = 1882 offer->chm_udata.pipe.user_def[0]; 1883 1884 if (bootverbose) { 1885 device_printf(sc->vmbus_dev, 1886 "chan%u is hyperv socket channel " 1887 "connected %s host\n", 1888 chan->ch_id, 1889 (chan->ch_hvs_conn_from_host != 0) ? 1890 "from" : "to"); 1891 } 1892 } else { 1893 chan->ch_is_hvs = false; 1894 } 1895 1896 /* 1897 * Setup event flag. 1898 */ 1899 chan->ch_evtflag = 1900 &sc->vmbus_tx_evtflags[chan->ch_id >> VMBUS_EVTFLAG_SHIFT]; 1901 chan->ch_evtflag_mask = 1UL << (chan->ch_id & VMBUS_EVTFLAG_MASK); 1902 1903 /* 1904 * Setup attach and detach tasks. 1905 */ 1906 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1907 chan->ch_mgmt_tq = sc->vmbus_devtq; 1908 attach_fn = vmbus_prichan_attach_task; 1909 detach_fn = vmbus_prichan_detach_task; 1910 } else { 1911 chan->ch_mgmt_tq = sc->vmbus_subchtq; 1912 attach_fn = vmbus_subchan_attach_task; 1913 detach_fn = vmbus_subchan_detach_task; 1914 } 1915 TASK_INIT(&chan->ch_attach_task, 0, attach_fn, chan); 1916 TASK_INIT(&chan->ch_detach_task, 0, detach_fn, chan); 1917 1918 error = vmbus_chan_add(chan); 1919 if (error) { 1920 device_printf(sc->vmbus_dev, "add chan%u failed: %d\n", 1921 chan->ch_id, error); 1922 atomic_subtract_int(&chan->ch_refs, 1); 1923 vmbus_chan_free(chan); 1924 return; 1925 } 1926 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_attach_task); 1927 } 1928 1929 static void 1930 vmbus_chan_msgproc_chrescind(struct vmbus_softc *sc, 1931 const struct vmbus_message *msg) 1932 { 1933 const struct vmbus_chanmsg_chrescind *note; 1934 struct vmbus_channel *chan; 1935 1936 note = (const struct vmbus_chanmsg_chrescind *)msg->msg_data; 1937 if (note->chm_chanid > VMBUS_CHAN_MAX) { 1938 device_printf(sc->vmbus_dev, "invalid revoked chan%u\n", 1939 note->chm_chanid); 1940 return; 1941 } 1942 1943 /* 1944 * Find and remove the target channel from the channel list. 1945 */ 1946 mtx_lock(&sc->vmbus_chan_lock); 1947 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 1948 if (chan->ch_id == note->chm_chanid) 1949 break; 1950 } 1951 if (chan == NULL) { 1952 mtx_unlock(&sc->vmbus_chan_lock); 1953 device_printf(sc->vmbus_dev, "chan%u is not offered\n", 1954 note->chm_chanid); 1955 return; 1956 } 1957 vmbus_chan_rem_list(sc, chan); 1958 mtx_unlock(&sc->vmbus_chan_lock); 1959 1960 if (VMBUS_CHAN_ISPRIMARY(chan)) { 1961 /* 1962 * The target channel is a primary channel; remove the 1963 * target channel from the primary channel list now, 1964 * instead of later, so that it will not be found by 1965 * other sub-channel offers, which are processed in 1966 * this thread. 1967 */ 1968 mtx_lock(&sc->vmbus_prichan_lock); 1969 vmbus_chan_rem_prilist(sc, chan); 1970 mtx_unlock(&sc->vmbus_prichan_lock); 1971 } 1972 1973 /* 1974 * NOTE: 1975 * The following processing order is critical: 1976 * Set the REVOKED state flag before orphaning the installed xact. 1977 */ 1978 1979 if (atomic_testandset_int(&chan->ch_stflags, 1980 VMBUS_CHAN_ST_REVOKED_SHIFT)) 1981 panic("channel has already been revoked"); 1982 1983 sx_xlock(&chan->ch_orphan_lock); 1984 if (chan->ch_orphan_xact != NULL) 1985 vmbus_xact_ctx_orphan(chan->ch_orphan_xact); 1986 sx_xunlock(&chan->ch_orphan_lock); 1987 1988 if (bootverbose) 1989 vmbus_chan_printf(chan, "chan%u revoked\n", note->chm_chanid); 1990 vmbus_chan_detach(chan); 1991 } 1992 1993 static int 1994 vmbus_chan_release(struct vmbus_channel *chan) 1995 { 1996 struct vmbus_softc *sc = chan->ch_vmbus; 1997 struct vmbus_chanmsg_chfree *req; 1998 struct vmbus_msghc *mh; 1999 int error; 2000 2001 mh = vmbus_msghc_get(sc, sizeof(*req)); 2002 if (mh == NULL) { 2003 vmbus_chan_printf(chan, 2004 "can not get msg hypercall for chfree(chan%u)\n", 2005 chan->ch_id); 2006 return (ENXIO); 2007 } 2008 2009 req = vmbus_msghc_dataptr(mh); 2010 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE; 2011 req->chm_chanid = chan->ch_id; 2012 2013 error = vmbus_msghc_exec_noresult(mh); 2014 vmbus_msghc_put(sc, mh); 2015 2016 if (error) { 2017 vmbus_chan_printf(chan, 2018 "chfree(chan%u) msg hypercall exec failed: %d\n", 2019 chan->ch_id, error); 2020 } else { 2021 if (bootverbose) 2022 vmbus_chan_printf(chan, "chan%u freed\n", chan->ch_id); 2023 } 2024 return (error); 2025 } 2026 2027 static void 2028 vmbus_prichan_detach_task(void *xchan, int pending __unused) 2029 { 2030 struct vmbus_channel *chan = xchan; 2031 2032 KASSERT(VMBUS_CHAN_ISPRIMARY(chan), 2033 ("chan%u is not primary channel", chan->ch_id)); 2034 2035 /* Delete and detach the device associated with this channel. */ 2036 vmbus_delete_child(chan); 2037 2038 /* Release this channel (back to vmbus). */ 2039 vmbus_chan_release(chan); 2040 2041 /* Free this channel's resource. */ 2042 vmbus_chan_free(chan); 2043 } 2044 2045 static void 2046 vmbus_subchan_detach_task(void *xchan, int pending __unused) 2047 { 2048 struct vmbus_channel *chan = xchan; 2049 struct vmbus_channel *pri_chan = chan->ch_prichan; 2050 2051 KASSERT(!VMBUS_CHAN_ISPRIMARY(chan), 2052 ("chan%u is primary channel", chan->ch_id)); 2053 2054 /* Release this channel (back to vmbus). */ 2055 vmbus_chan_release(chan); 2056 2057 /* Unlink from its primary channel's sub-channel list. */ 2058 mtx_lock(&pri_chan->ch_subchan_lock); 2059 vmbus_chan_rem_sublist(pri_chan, chan); 2060 mtx_unlock(&pri_chan->ch_subchan_lock); 2061 /* Notify anyone that is waiting for this sub-channel to vanish. */ 2062 wakeup(pri_chan); 2063 2064 /* Free this channel's resource. */ 2065 vmbus_chan_free(chan); 2066 } 2067 2068 static void 2069 vmbus_prichan_attach_task(void *xchan, int pending __unused) 2070 { 2071 2072 /* 2073 * Add device for this primary channel. 2074 */ 2075 vmbus_add_child(xchan); 2076 } 2077 2078 static void 2079 vmbus_subchan_attach_task(void *xchan __unused, int pending __unused) 2080 { 2081 2082 /* Nothing */ 2083 } 2084 2085 void 2086 vmbus_chan_destroy_all(struct vmbus_softc *sc) 2087 { 2088 2089 /* 2090 * Detach all devices and destroy the corresponding primary 2091 * channels. 2092 */ 2093 for (;;) { 2094 struct vmbus_channel *chan; 2095 2096 mtx_lock(&sc->vmbus_chan_lock); 2097 TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) { 2098 if (VMBUS_CHAN_ISPRIMARY(chan)) 2099 break; 2100 } 2101 if (chan == NULL) { 2102 /* No more primary channels; done. */ 2103 mtx_unlock(&sc->vmbus_chan_lock); 2104 break; 2105 } 2106 vmbus_chan_rem_list(sc, chan); 2107 mtx_unlock(&sc->vmbus_chan_lock); 2108 2109 mtx_lock(&sc->vmbus_prichan_lock); 2110 vmbus_chan_rem_prilist(sc, chan); 2111 mtx_unlock(&sc->vmbus_prichan_lock); 2112 2113 taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task); 2114 } 2115 } 2116 2117 struct vmbus_channel ** 2118 vmbus_subchan_get(struct vmbus_channel *pri_chan, int subchan_cnt) 2119 { 2120 struct vmbus_channel **ret, *chan; 2121 int i; 2122 2123 KASSERT(subchan_cnt > 0, ("invalid sub-channel count %d", subchan_cnt)); 2124 2125 ret = malloc(subchan_cnt * sizeof(struct vmbus_channel *), M_TEMP, 2126 M_WAITOK); 2127 2128 mtx_lock(&pri_chan->ch_subchan_lock); 2129 2130 while (pri_chan->ch_subchan_cnt < subchan_cnt) 2131 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "subch", 0); 2132 2133 i = 0; 2134 TAILQ_FOREACH(chan, &pri_chan->ch_subchans, ch_sublink) { 2135 /* TODO: refcnt chan */ 2136 ret[i] = chan; 2137 2138 ++i; 2139 if (i == subchan_cnt) 2140 break; 2141 } 2142 KASSERT(i == subchan_cnt, ("invalid subchan count %d, should be %d", 2143 pri_chan->ch_subchan_cnt, subchan_cnt)); 2144 2145 mtx_unlock(&pri_chan->ch_subchan_lock); 2146 2147 return ret; 2148 } 2149 2150 void 2151 vmbus_subchan_rel(struct vmbus_channel **subchan, int subchan_cnt __unused) 2152 { 2153 2154 free(subchan, M_TEMP); 2155 } 2156 2157 void 2158 vmbus_subchan_drain(struct vmbus_channel *pri_chan) 2159 { 2160 mtx_lock(&pri_chan->ch_subchan_lock); 2161 while (pri_chan->ch_subchan_cnt > 0) 2162 mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "dsubch", 0); 2163 mtx_unlock(&pri_chan->ch_subchan_lock); 2164 } 2165 2166 void 2167 vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg) 2168 { 2169 vmbus_chanmsg_proc_t msg_proc; 2170 uint32_t msg_type; 2171 2172 msg_type = ((const struct vmbus_chanmsg_hdr *)msg->msg_data)->chm_type; 2173 KASSERT(msg_type < VMBUS_CHANMSG_TYPE_MAX, 2174 ("invalid message type %u", msg_type)); 2175 2176 msg_proc = vmbus_chan_msgprocs[msg_type]; 2177 if (msg_proc != NULL) 2178 msg_proc(sc, msg); 2179 } 2180 2181 void 2182 vmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on) 2183 { 2184 if (!on) 2185 chan->ch_flags &= ~VMBUS_CHAN_FLAG_BATCHREAD; 2186 else 2187 chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD; 2188 } 2189 2190 uint32_t 2191 vmbus_chan_id(const struct vmbus_channel *chan) 2192 { 2193 return chan->ch_id; 2194 } 2195 2196 uint32_t 2197 vmbus_chan_subidx(const struct vmbus_channel *chan) 2198 { 2199 return chan->ch_subidx; 2200 } 2201 2202 bool 2203 vmbus_chan_is_primary(const struct vmbus_channel *chan) 2204 { 2205 if (VMBUS_CHAN_ISPRIMARY(chan)) 2206 return true; 2207 else 2208 return false; 2209 } 2210 2211 bool 2212 vmbus_chan_is_hvs(const struct vmbus_channel *chan) 2213 { 2214 return chan->ch_is_hvs; 2215 } 2216 2217 bool 2218 vmbus_chan_is_hvs_conn_from_host(const struct vmbus_channel *chan) 2219 { 2220 KASSERT(vmbus_chan_is_hvs(chan) == true, 2221 ("Not a HyperV Socket channel %u", chan->ch_id)); 2222 if (chan->ch_hvs_conn_from_host != 0) 2223 return true; 2224 else 2225 return false; 2226 } 2227 2228 struct hyperv_guid * 2229 vmbus_chan_guid_type(struct vmbus_channel *chan) 2230 { 2231 return &chan->ch_guid_type; 2232 } 2233 2234 struct hyperv_guid * 2235 vmbus_chan_guid_inst(struct vmbus_channel *chan) 2236 { 2237 return &chan->ch_guid_inst; 2238 } 2239 2240 int 2241 vmbus_chan_prplist_nelem(int br_size, int prpcnt_max, int dlen_max) 2242 { 2243 int elem_size; 2244 2245 elem_size = __offsetof(struct vmbus_chanpkt_prplist, 2246 cp_range[0].gpa_page[prpcnt_max]); 2247 elem_size += dlen_max; 2248 elem_size = VMBUS_CHANPKT_TOTLEN(elem_size); 2249 2250 return (vmbus_br_nelem(br_size, elem_size)); 2251 } 2252 2253 bool 2254 vmbus_chan_tx_empty(const struct vmbus_channel *chan) 2255 { 2256 2257 return (vmbus_txbr_empty(&chan->ch_txbr)); 2258 } 2259 2260 bool 2261 vmbus_chan_rx_empty(const struct vmbus_channel *chan) 2262 { 2263 2264 return (vmbus_rxbr_empty(&chan->ch_rxbr)); 2265 } 2266 2267 static int 2268 vmbus_chan_printf(const struct vmbus_channel *chan, const char *fmt, ...) 2269 { 2270 va_list ap; 2271 device_t dev; 2272 int retval; 2273 2274 if (chan->ch_dev == NULL || !device_is_alive(chan->ch_dev)) 2275 dev = chan->ch_vmbus->vmbus_dev; 2276 else 2277 dev = chan->ch_dev; 2278 2279 retval = device_print_prettyname(dev); 2280 va_start(ap, fmt); 2281 retval += vprintf(fmt, ap); 2282 va_end(ap); 2283 2284 return (retval); 2285 } 2286 2287 void 2288 vmbus_chan_run_task(struct vmbus_channel *chan, struct task *task) 2289 { 2290 2291 taskqueue_enqueue(chan->ch_tq, task); 2292 taskqueue_drain(chan->ch_tq, task); 2293 } 2294 2295 struct taskqueue * 2296 vmbus_chan_mgmt_tq(const struct vmbus_channel *chan) 2297 { 2298 2299 return (chan->ch_mgmt_tq); 2300 } 2301 2302 bool 2303 vmbus_chan_is_revoked(const struct vmbus_channel *chan) 2304 { 2305 2306 if (chan->ch_stflags & VMBUS_CHAN_ST_REVOKED) 2307 return (true); 2308 return (false); 2309 } 2310 2311 void 2312 vmbus_chan_set_orphan(struct vmbus_channel *chan, struct vmbus_xact_ctx *xact) 2313 { 2314 2315 sx_xlock(&chan->ch_orphan_lock); 2316 chan->ch_orphan_xact = xact; 2317 sx_xunlock(&chan->ch_orphan_lock); 2318 } 2319 2320 void 2321 vmbus_chan_unset_orphan(struct vmbus_channel *chan) 2322 { 2323 2324 sx_xlock(&chan->ch_orphan_lock); 2325 chan->ch_orphan_xact = NULL; 2326 sx_xunlock(&chan->ch_orphan_lock); 2327 } 2328 2329 const void * 2330 vmbus_chan_xact_wait(const struct vmbus_channel *chan, 2331 struct vmbus_xact *xact, size_t *resp_len, bool can_sleep) 2332 { 2333 const void *ret; 2334 2335 if (can_sleep) 2336 ret = vmbus_xact_wait(xact, resp_len); 2337 else 2338 ret = vmbus_xact_busywait(xact, resp_len); 2339 if (vmbus_chan_is_revoked(chan)) { 2340 /* 2341 * This xact probably is interrupted, and the 2342 * interruption can race the reply reception, 2343 * so we have to make sure that there are nothing 2344 * left on the RX bufring, i.e. this xact will 2345 * not be touched, once this function returns. 2346 * 2347 * Since the hypervisor will not put more data 2348 * onto the RX bufring once the channel is revoked, 2349 * the following loop will be terminated, once all 2350 * data are drained by the driver's channel 2351 * callback. 2352 */ 2353 while (!vmbus_chan_rx_empty(chan)) { 2354 if (can_sleep) 2355 pause("chxact", 1); 2356 else 2357 DELAY(1000); 2358 } 2359 } 2360 return (ret); 2361 } 2362 2363 void 2364 vmbus_chan_poll_enable(struct vmbus_channel *chan, u_int pollhz) 2365 { 2366 struct vmbus_chan_pollarg arg; 2367 struct task poll_cfg; 2368 2369 KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD, 2370 ("enable polling on non-batch chan%u", chan->ch_id)); 2371 KASSERT(pollhz >= VMBUS_CHAN_POLLHZ_MIN && 2372 pollhz <= VMBUS_CHAN_POLLHZ_MAX, ("invalid pollhz %u", pollhz)); 2373 2374 arg.poll_chan = chan; 2375 arg.poll_hz = pollhz; 2376 TASK_INIT(&poll_cfg, 0, vmbus_chan_pollcfg_task, &arg); 2377 vmbus_chan_run_task(chan, &poll_cfg); 2378 } 2379 2380 void 2381 vmbus_chan_poll_disable(struct vmbus_channel *chan) 2382 { 2383 struct task poll_dis; 2384 2385 KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD, 2386 ("disable polling on non-batch chan%u", chan->ch_id)); 2387 2388 TASK_INIT(&poll_dis, 0, vmbus_chan_polldis_task, chan); 2389 vmbus_chan_run_task(chan, &poll_dis); 2390 }
Cache object: 1b16d495aa07938cba2b365ab5b48fed
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]