Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/mips/rmi/fmn.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2003-2009 RMI Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. Neither the name of RMI Corporation, nor the names of its contributors, 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * RMI_BSD */ 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD: releng/10.1/sys/mips/rmi/fmn.c 223562 2011-06-26 10:07:48Z kevlo $"); 32 #include <sys/types.h> 33 #include <sys/systm.h> 34 #include <sys/param.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/proc.h> 38 #include <sys/limits.h> 39 #include <sys/bus.h> 40 41 #include <sys/ktr.h> 42 #include <sys/kernel.h> 43 #include <sys/kthread.h> 44 #include <sys/resourcevar.h> 45 #include <sys/sched.h> 46 #include <sys/unistd.h> 47 #include <sys/sysctl.h> 48 #include <sys/malloc.h> 49 50 #include <machine/reg.h> 51 #include <machine/cpu.h> 52 #include <machine/hwfunc.h> 53 #include <machine/mips_opcode.h> 54 55 #include <machine/param.h> 56 #include <machine/intr_machdep.h> 57 #include <mips/rmi/interrupt.h> 58 #include <mips/rmi/msgring.h> 59 #include <mips/rmi/pic.h> 60 #include <mips/rmi/board.h> 61 62 #define MSGRNG_CC_INIT_CPU_DEST(dest, counter) \ 63 do { \ 64 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][0], 0 ); \ 65 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][1], 1 ); \ 66 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][2], 2 ); \ 67 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][3], 3 ); \ 68 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][4], 4 ); \ 69 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][5], 5 ); \ 70 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][6], 6 ); \ 71 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][7], 7 ); \ 72 } while(0) 73 74 75 /* 76 * Keep track of our message ring handler threads, each core has a 77 * different message station. Ideally we will need to start a few 78 * message handling threads every core, and wake them up depending on 79 * load 80 */ 81 struct msgring_thread { 82 struct { 83 struct thread *thread; /* msgring handler threads */ 84 int needed; /* thread needs to wake up */ 85 } threads[XLR_NTHREADS]; 86 int running; /* number of threads running */ 87 int nthreads; /* number of threads started */ 88 struct mtx lock; /* for changing running/active */ 89 }; 90 static struct msgring_thread msgring_threads[XLR_MAX_CORES]; 91 static struct proc *msgring_proc; /* all threads are under a proc */ 92 93 /* 94 * The maximum number of software message handler threads to be started 95 * per core. Default is 3 per core 96 */ 97 static int msgring_maxthreads = 3; 98 TUNABLE_INT("hw.fmn.maxthreads", &msgring_maxthreads); 99 100 /* 101 * The device drivers can register a handler for the messages sent 102 * from a station (corresponding to the device). 103 */ 104 struct tx_stn_handler { 105 msgring_handler action; 106 void *arg; 107 }; 108 static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS]; 109 static struct mtx msgmap_lock; 110 111 /* 112 * Initialize the messaging subsystem. 113 * 114 * Message Stations are shared among all threads in a cpu core, this 115 * has to be called once from every core which is online. 116 */ 117 void 118 xlr_msgring_cpu_init(void) 119 { 120 struct stn_cc *cc_config; 121 struct bucket_size *bucket_sizes; 122 uint32_t flags; 123 int id; 124 125 KASSERT(xlr_thr_id() == 0, 126 ("xlr_msgring_cpu_init from non-zero thread")); 127 id = xlr_core_id(); 128 bucket_sizes = xlr_board_info.bucket_sizes; 129 cc_config = xlr_board_info.credit_configs[id]; 130 131 flags = msgrng_access_enable(); 132 133 /* 134 * FMN messages are received in 8 buckets per core, set up 135 * the bucket sizes for each bucket 136 */ 137 msgrng_write_bucksize(0, bucket_sizes->bucket[id * 8 + 0]); 138 msgrng_write_bucksize(1, bucket_sizes->bucket[id * 8 + 1]); 139 msgrng_write_bucksize(2, bucket_sizes->bucket[id * 8 + 2]); 140 msgrng_write_bucksize(3, bucket_sizes->bucket[id * 8 + 3]); 141 msgrng_write_bucksize(4, bucket_sizes->bucket[id * 8 + 4]); 142 msgrng_write_bucksize(5, bucket_sizes->bucket[id * 8 + 5]); 143 msgrng_write_bucksize(6, bucket_sizes->bucket[id * 8 + 6]); 144 msgrng_write_bucksize(7, bucket_sizes->bucket[id * 8 + 7]); 145 146 /* 147 * For sending FMN messages, we need credits on the destination 148 * bucket. Program the credits this core has on the 128 possible 149 * destination buckets. 150 * We cannot use a loop here, because the first argument has 151 * to be a constant integer value. 152 */ 153 MSGRNG_CC_INIT_CPU_DEST(0, cc_config->counters); 154 MSGRNG_CC_INIT_CPU_DEST(1, cc_config->counters); 155 MSGRNG_CC_INIT_CPU_DEST(2, cc_config->counters); 156 MSGRNG_CC_INIT_CPU_DEST(3, cc_config->counters); 157 MSGRNG_CC_INIT_CPU_DEST(4, cc_config->counters); 158 MSGRNG_CC_INIT_CPU_DEST(5, cc_config->counters); 159 MSGRNG_CC_INIT_CPU_DEST(6, cc_config->counters); 160 MSGRNG_CC_INIT_CPU_DEST(7, cc_config->counters); 161 MSGRNG_CC_INIT_CPU_DEST(8, cc_config->counters); 162 MSGRNG_CC_INIT_CPU_DEST(9, cc_config->counters); 163 MSGRNG_CC_INIT_CPU_DEST(10, cc_config->counters); 164 MSGRNG_CC_INIT_CPU_DEST(11, cc_config->counters); 165 MSGRNG_CC_INIT_CPU_DEST(12, cc_config->counters); 166 MSGRNG_CC_INIT_CPU_DEST(13, cc_config->counters); 167 MSGRNG_CC_INIT_CPU_DEST(14, cc_config->counters); 168 MSGRNG_CC_INIT_CPU_DEST(15, cc_config->counters); 169 msgrng_restore(flags); 170 } 171 172 /* 173 * Boot time init, called only once 174 */ 175 void 176 xlr_msgring_config(void) 177 { 178 mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN); 179 180 /* check value */ 181 if (msgring_maxthreads < 0 || msgring_maxthreads > XLR_NTHREADS) 182 msgring_maxthreads = XLR_NTHREADS; 183 } 184 185 /* 186 * Drain out max_messages for the buckets set in the bucket mask. 187 * Use max_messages = 0 to drain out all messages. 188 */ 189 uint32_t 190 xlr_msgring_handler(uint8_t bucket_mask, uint32_t max_messages) 191 { 192 int bucket = 0; 193 int size = 0, code = 0, rx_stid = 0; 194 struct msgrng_msg msg; 195 struct tx_stn_handler *he; 196 unsigned int status = 0; 197 unsigned long mflags; 198 uint32_t n_msgs; 199 uint32_t msgbuckets; 200 201 n_msgs = 0; 202 mflags = msgrng_access_enable(); 203 for (;;) { 204 msgbuckets = (~msgrng_read_status() >> 24) & bucket_mask; 205 206 /* all buckets empty, break */ 207 if (msgbuckets == 0) 208 break; 209 210 for (bucket = 0; bucket < 8; bucket++) { 211 if ((msgbuckets & (1 << bucket)) == 0) /* empty */ 212 continue; 213 214 status = message_receive(bucket, &size, &code, 215 &rx_stid, &msg); 216 if (status != 0) 217 continue; 218 n_msgs++; 219 he = &msgmap[rx_stid]; 220 if (he->action == NULL) { 221 printf("[%s]: No Handler for message from " 222 "stn_id=%d, bucket=%d, size=%d, msg0=%jx\n", 223 __func__, rx_stid, bucket, size, 224 (uintmax_t)msg.msg0); 225 } else { 226 msgrng_restore(mflags); 227 (*he->action)(bucket, size, code, rx_stid, 228 &msg, he->arg); 229 mflags = msgrng_access_enable(); 230 } 231 if (max_messages > 0 && n_msgs >= max_messages) 232 goto done; 233 } 234 } 235 236 done: 237 msgrng_restore(mflags); 238 return (n_msgs); 239 } 240 241 /* 242 * XLR COP2 supports watermark interrupts based on the number of 243 * messages pending in all the buckets in the core. We increase 244 * the watermark until all the possible handler threads in the core 245 * are woken up. 246 */ 247 static void 248 msgrng_setconfig(int running, int nthr) 249 { 250 uint32_t config, mflags; 251 int watermark = 1; /* non zero needed */ 252 int wm_intr_value; 253 254 KASSERT(nthr >= 0 && nthr <= msgring_maxthreads, 255 ("Bad value of nthr %d", nthr)); 256 KASSERT(running <= nthr, ("Bad value of running %d", running)); 257 258 if (running == nthr) { 259 wm_intr_value = 0; 260 } else { 261 switch (running) { 262 case 0: break; /* keep default */ 263 case 1: 264 watermark = 32; break; 265 case 2: 266 watermark = 48; break; 267 case 3: 268 watermark = 56; break; 269 } 270 wm_intr_value = 0x2; /* set watermark enable interrupt */ 271 } 272 mflags = msgrng_access_enable(); 273 config = (watermark << 24) | (IRQ_MSGRING << 16) | (1 << 8) | 274 wm_intr_value; 275 /* clear pending interrupts, they will get re-raised if still valid */ 276 write_c0_eirr64(1ULL << IRQ_MSGRING); 277 msgrng_write_config(config); 278 msgrng_restore(mflags); 279 } 280 281 /* Debug counters */ 282 static int msgring_nintr[XLR_MAX_CORES]; 283 static int msgring_badintr[XLR_MAX_CORES]; 284 static int msgring_wakeup_sleep[XLR_MAX_CORES * XLR_NTHREADS]; 285 static int msgring_wakeup_nosleep[XLR_MAX_CORES * XLR_NTHREADS]; 286 static int msgring_nmsgs[XLR_MAX_CORES * XLR_NTHREADS]; 287 288 static int 289 msgring_process_fast_intr(void *arg) 290 { 291 struct msgring_thread *mthd; 292 struct thread *td; 293 uint32_t mflags; 294 int core, nt; 295 296 core = xlr_core_id(); 297 mthd = &msgring_threads[core]; 298 msgring_nintr[core]++; 299 mtx_lock_spin(&mthd->lock); 300 nt = mthd->running; 301 if(nt >= mthd->nthreads) { 302 msgring_badintr[core]++; 303 mtx_unlock_spin(&mthd->lock); 304 return (FILTER_HANDLED); 305 } 306 307 td = mthd->threads[nt].thread; 308 mflags = msgrng_access_enable(); 309 310 /* default value with interrupts disabled */ 311 msgrng_write_config((1 << 24) | (IRQ_MSGRING << 16) | (1 << 8)); 312 /* clear pending interrupts */ 313 write_c0_eirr64(1ULL << IRQ_MSGRING); 314 msgrng_restore(mflags); 315 mtx_unlock_spin(&mthd->lock); 316 317 /* wake up the target thread */ 318 mthd->threads[nt].needed = 1; 319 thread_lock(td); 320 if (TD_AWAITING_INTR(td)) { 321 msgring_wakeup_sleep[core*4+nt]++; 322 TD_CLR_IWAIT(td); 323 sched_add(td, SRQ_INTR); 324 } else 325 msgring_wakeup_nosleep[core*4+nt]++; 326 thread_unlock(td); 327 return (FILTER_HANDLED); 328 } 329 330 static void 331 msgring_process(void *arg) 332 { 333 struct msgring_thread *mthd; 334 struct thread *td; 335 int hwtid, tid, core; 336 int nmsgs; 337 338 hwtid = (intptr_t)arg; 339 core = hwtid / 4; 340 tid = hwtid % 4; 341 mthd = &msgring_threads[core]; 342 td = mthd->threads[tid].thread; 343 KASSERT(curthread == td, 344 ("Incorrect thread core %d, thread %d", core, hwtid)); 345 346 /* First bind this thread to the right CPU */ 347 thread_lock(td); 348 sched_bind(td, xlr_hwtid_to_cpuid[hwtid]); 349 thread_unlock(td); 350 351 mtx_lock_spin(&mthd->lock); 352 ++mthd->nthreads; /* Active thread count */ 353 mtx_unlock_spin(&mthd->lock); 354 355 /* start processing messages */ 356 for(;;) { 357 mtx_lock_spin(&mthd->lock); 358 ++mthd->running; 359 msgrng_setconfig(mthd->running, mthd->nthreads); 360 mtx_unlock_spin(&mthd->lock); 361 362 atomic_store_rel_int(&mthd->threads[tid].needed, 0); 363 nmsgs = xlr_msgring_handler(0xff, 0); 364 msgring_nmsgs[hwtid] += nmsgs; 365 366 mtx_lock_spin(&mthd->lock); 367 --mthd->running; 368 msgrng_setconfig(mthd->running, mthd->nthreads); 369 mtx_unlock_spin(&mthd->lock); 370 371 /* sleep */ 372 thread_lock(td); 373 if (mthd->threads[tid].needed) { 374 thread_unlock(td); 375 continue; 376 } 377 sched_class(td, PRI_ITHD); 378 TD_SET_IWAIT(td); 379 mi_switch(SW_VOL, NULL); 380 thread_unlock(td); 381 } 382 } 383 384 static void 385 create_msgring_thread(int hwtid) 386 { 387 struct msgring_thread *mthd; 388 struct thread *td; 389 int tid, core; 390 int error; 391 392 core = hwtid / 4; 393 tid = hwtid % 4; 394 mthd = &msgring_threads[core]; 395 if (tid == 0) { 396 mtx_init(&mthd->lock, "msgrngcore", NULL, MTX_SPIN); 397 mthd->running = mthd->nthreads = 0; 398 } 399 error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid, 400 &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc", 401 "msgthr%d", hwtid); 402 if (error) 403 panic("kproc_kthread_add() failed with %d", error); 404 mthd->threads[tid].thread = td; 405 406 thread_lock(td); 407 sched_class(td, PRI_ITHD); 408 sched_add(td, SRQ_INTR); 409 thread_unlock(td); 410 CTR2(KTR_INTR, "%s: created %s", __func__, td->td_name); 411 } 412 413 int 414 register_msgring_handler(int startb, int endb, msgring_handler action, 415 void *arg) 416 { 417 void *cookie; 418 int i; 419 static int msgring_int_enabled = 0; 420 421 KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS, 422 ("Invalid value for for bucket range %d,%d", startb, endb)); 423 424 mtx_lock_spin(&msgmap_lock); 425 for (i = startb; i <= endb; i++) { 426 KASSERT(msgmap[i].action == NULL, 427 ("Bucket %d already used [action %p]", i, msgmap[i].action)); 428 msgmap[i].action = action; 429 msgmap[i].arg = arg; 430 } 431 mtx_unlock_spin(&msgmap_lock); 432 433 if (xlr_test_and_set(&msgring_int_enabled)) { 434 create_msgring_thread(0); 435 if (msgring_maxthreads > xlr_threads_per_core) 436 msgring_maxthreads = xlr_threads_per_core; 437 cpu_establish_hardintr("msgring", msgring_process_fast_intr, 438 NULL, NULL, IRQ_MSGRING, 439 INTR_TYPE_NET, &cookie); 440 } 441 return (0); 442 } 443 444 /* 445 * Start message ring processing threads on other CPUs, after SMP start 446 */ 447 static void 448 start_msgring_threads(void *arg) 449 { 450 int hwt, tid; 451 452 for (hwt = 1; hwt < XLR_MAX_CORES * XLR_NTHREADS; hwt++) { 453 if ((xlr_hw_thread_mask & (1 << hwt)) == 0) 454 continue; 455 tid = hwt % XLR_NTHREADS; 456 if (tid >= msgring_maxthreads) 457 continue; 458 create_msgring_thread(hwt); 459 } 460 } 461 462 SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE, 463 start_msgring_threads, NULL); 464 465 /* 466 * DEBUG support, XXX: static buffer, not locked 467 */ 468 static int 469 sys_print_debug(SYSCTL_HANDLER_ARGS) 470 { 471 int error, nb, i, fs; 472 static char xprintb[4096], *buf; 473 474 buf = xprintb; 475 fs = sizeof(xprintb); 476 nb = snprintf(buf, fs, 477 "\nID INTR ER WU-SLP WU-ERR MSGS\n"); 478 buf += nb; 479 fs -= nb; 480 for (i = 0; i < 32; i++) { 481 if ((xlr_hw_thread_mask & (1 << i)) == 0) 482 continue; 483 nb = snprintf(buf, fs, 484 "%2d: %8d %4d %8d %8d %8d\n", i, 485 msgring_nintr[i/4], msgring_badintr[i/4], 486 msgring_wakeup_sleep[i], msgring_wakeup_nosleep[i], 487 msgring_nmsgs[i]); 488 buf += nb; 489 fs -= nb; 490 } 491 error = SYSCTL_OUT(req, xprintb, buf - xprintb); 492 return (error); 493 } 494 495 SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0, 496 sys_print_debug, "A", "msgring debug info");
Cache object: 8db682a2d9ee4b63f2d9144616a174c9
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]