The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/mips/rmi/fmn.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2003-2009 RMI Corporation
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 3. Neither the name of RMI Corporation, nor the names of its contributors,
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  *
   29  * RMI_BSD */
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD: releng/8.4/sys/mips/rmi/fmn.c 218992 2011-02-24 10:23:22Z brucec $");
   32 #include <sys/types.h>
   33 #include <sys/systm.h>
   34 #include <sys/param.h>
   35 #include <sys/lock.h>
   36 #include <sys/mutex.h>
   37 #include <sys/proc.h>
   38 #include <sys/limits.h>
   39 #include <sys/bus.h>
   40 
   41 #include <sys/ktr.h>
   42 #include <sys/kernel.h>
   43 #include <sys/kthread.h>
   44 #include <sys/proc.h>
   45 #include <sys/resourcevar.h>
   46 #include <sys/sched.h>
   47 #include <sys/unistd.h>
   48 #include <sys/sysctl.h>
   49 #include <sys/malloc.h>
   50 
   51 #include <machine/reg.h>
   52 #include <machine/cpu.h>
   53 #include <machine/hwfunc.h>
   54 #include <machine/mips_opcode.h>
   55 
   56 #include <machine/param.h>
   57 #include <machine/intr_machdep.h>
   58 #include <mips/rmi/interrupt.h>
   59 #include <mips/rmi/msgring.h>
   60 #include <mips/rmi/pic.h>
   61 #include <mips/rmi/board.h>
   62 
   63 #define MSGRNG_CC_INIT_CPU_DEST(dest, counter) \
   64 do { \
   65      msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][0], 0 ); \
   66      msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][1], 1 ); \
   67      msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][2], 2 ); \
   68      msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][3], 3 ); \
   69      msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][4], 4 ); \
   70      msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][5], 5 ); \
   71      msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][6], 6 ); \
   72      msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][7], 7 ); \
   73 } while(0)
   74 
   75 
   76 /*
   77  * Keep track of our message ring handler threads, each core has a 
   78  * different message station. Ideally we will need to start a few
   79  * message handling threads every core, and wake them up depending on
   80  * load
   81  */
   82 struct msgring_thread {
   83         struct {
   84                 struct thread   *thread; /* msgring handler threads */
   85                 int     needed;         /* thread needs to wake up */
   86         } threads[XLR_NTHREADS];
   87         int     running;                /* number of threads running */
   88         int     nthreads;               /* number of threads started */
   89         struct mtx lock;                /* for changing running/active */
   90 };
   91 static struct msgring_thread msgring_threads[XLR_MAX_CORES];
   92 static struct proc *msgring_proc;       /* all threads are under a proc */
   93 
   94 /*
   95  * The maximum number of software message handler threads to be started 
   96  * per core. Default is 3 per core
   97  */
   98 static int      msgring_maxthreads = 3; 
   99 TUNABLE_INT("hw.fmn.maxthreads", &msgring_maxthreads);
  100 
  101 /* 
  102  * The device drivers can register a handler for the messages sent
  103  * from a station (corresponding to the device). 
  104  */
  105 struct tx_stn_handler {
  106         msgring_handler action;
  107         void *arg;
  108 };
  109 static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS];
  110 static struct mtx       msgmap_lock;
  111 
  112 /*
  113  * Initialize the messaging subsystem.
  114  * 
  115  * Message Stations are shared among all threads in a cpu core, this 
  116  * has to be called once from every core which is online.
  117  */
  118 void 
  119 xlr_msgring_cpu_init(void)
  120 {
  121         struct stn_cc *cc_config;
  122         struct bucket_size *bucket_sizes;
  123         uint32_t flags;
  124         int id;
  125 
  126         KASSERT(xlr_thr_id() == 0,
  127                 ("xlr_msgring_cpu_init from non-zero thread"));
  128         id = xlr_core_id();
  129         bucket_sizes = xlr_board_info.bucket_sizes;
  130         cc_config = xlr_board_info.credit_configs[id];
  131 
  132         flags = msgrng_access_enable();
  133 
  134         /*
  135          * FMN messages are received in 8 buckets per core, set up
  136          * the bucket sizes for each bucket
  137          */
  138         msgrng_write_bucksize(0, bucket_sizes->bucket[id * 8 + 0]);
  139         msgrng_write_bucksize(1, bucket_sizes->bucket[id * 8 + 1]);
  140         msgrng_write_bucksize(2, bucket_sizes->bucket[id * 8 + 2]);
  141         msgrng_write_bucksize(3, bucket_sizes->bucket[id * 8 + 3]);
  142         msgrng_write_bucksize(4, bucket_sizes->bucket[id * 8 + 4]);
  143         msgrng_write_bucksize(5, bucket_sizes->bucket[id * 8 + 5]);
  144         msgrng_write_bucksize(6, bucket_sizes->bucket[id * 8 + 6]);
  145         msgrng_write_bucksize(7, bucket_sizes->bucket[id * 8 + 7]);
  146 
  147         /* 
  148          * For sending FMN messages, we need credits on the destination
  149          * bucket.  Program the credits this core has on the 128 possible
  150          * destination buckets.
  151          * We cannot use a loop here, because the first argument has
  152          * to be a constant integer value.
  153          */
  154         MSGRNG_CC_INIT_CPU_DEST(0,  cc_config->counters);
  155         MSGRNG_CC_INIT_CPU_DEST(1,  cc_config->counters);
  156         MSGRNG_CC_INIT_CPU_DEST(2,  cc_config->counters);
  157         MSGRNG_CC_INIT_CPU_DEST(3,  cc_config->counters);
  158         MSGRNG_CC_INIT_CPU_DEST(4,  cc_config->counters);
  159         MSGRNG_CC_INIT_CPU_DEST(5,  cc_config->counters);
  160         MSGRNG_CC_INIT_CPU_DEST(6,  cc_config->counters);
  161         MSGRNG_CC_INIT_CPU_DEST(7,  cc_config->counters);
  162         MSGRNG_CC_INIT_CPU_DEST(8,  cc_config->counters);
  163         MSGRNG_CC_INIT_CPU_DEST(9,  cc_config->counters);
  164         MSGRNG_CC_INIT_CPU_DEST(10, cc_config->counters);
  165         MSGRNG_CC_INIT_CPU_DEST(11, cc_config->counters);
  166         MSGRNG_CC_INIT_CPU_DEST(12, cc_config->counters);
  167         MSGRNG_CC_INIT_CPU_DEST(13, cc_config->counters);
  168         MSGRNG_CC_INIT_CPU_DEST(14, cc_config->counters);
  169         MSGRNG_CC_INIT_CPU_DEST(15, cc_config->counters);
  170         msgrng_restore(flags);
  171 }
  172 
  173 /*
  174  * Boot time init, called only once
  175  */
  176 void 
  177 xlr_msgring_config(void)
  178 {
  179         mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN);
  180         
  181         /* check value */
  182         if (msgring_maxthreads < 0 || msgring_maxthreads > XLR_NTHREADS)
  183                 msgring_maxthreads = XLR_NTHREADS;
  184 }
  185 
  186 /*
  187  * Drain out max_messages for the buckets set in the bucket mask. 
  188  * Use max_messages = 0 to drain out all messages.
  189  */
  190 uint32_t
  191 xlr_msgring_handler(uint8_t bucket_mask, uint32_t max_messages)
  192 {
  193         int bucket = 0;
  194         int size = 0, code = 0, rx_stid = 0;
  195         struct msgrng_msg msg;
  196         struct tx_stn_handler *he;
  197         unsigned int status = 0;
  198         unsigned long mflags;
  199         uint32_t n_msgs;
  200         uint32_t msgbuckets;
  201 
  202         n_msgs = 0;
  203         mflags = msgrng_access_enable();
  204         for (;;) {
  205                 msgbuckets = (~msgrng_read_status() >> 24) & bucket_mask;
  206 
  207                 /* all buckets empty, break */
  208                 if (msgbuckets == 0)
  209                         break;
  210 
  211                 for (bucket = 0; bucket < 8; bucket++) {
  212                         if ((msgbuckets & (1 << bucket)) == 0) /* empty */
  213                                 continue;
  214 
  215                         status = message_receive(bucket, &size, &code,
  216                             &rx_stid, &msg);
  217                         if (status != 0)
  218                                 continue;
  219                         n_msgs++;
  220                         he = &msgmap[rx_stid];
  221                         if (he->action == NULL) {
  222                                 printf("[%s]: No Handler for message from "
  223                                     "stn_id=%d, bucket=%d, size=%d, msg0=%jx\n",
  224                                     __func__, rx_stid, bucket, size,
  225                                     (uintmax_t)msg.msg0);
  226                         } else {
  227                                 msgrng_restore(mflags);
  228                                 (*he->action)(bucket, size, code, rx_stid,
  229                                     &msg, he->arg);
  230                                 mflags = msgrng_access_enable();
  231                         }
  232                         if (max_messages > 0 && n_msgs >= max_messages)
  233                                 goto done;
  234                 }
  235         }
  236 
  237 done:
  238         msgrng_restore(mflags);
  239         return (n_msgs);
  240 }
  241 
  242 /* 
  243  * XLR COP2 supports watermark interrupts based on the number of 
  244  * messages pending in all the buckets in the core.  We increase 
  245  * the watermark until all the possible handler threads in the core
  246  * are woken up.
  247  */
  248 static void
  249 msgrng_setconfig(int running, int nthr)
  250 {
  251         uint32_t config, mflags;
  252         int watermark = 1;      /* non zero needed */
  253         int wm_intr_value;
  254 
  255         KASSERT(nthr >= 0 && nthr <= msgring_maxthreads,
  256             ("Bad value of nthr %d", nthr));
  257         KASSERT(running <= nthr, ("Bad value of running %d", running));
  258 
  259         if (running == nthr) {
  260                 wm_intr_value = 0;
  261         } else {
  262                 switch (running) {
  263                 case 0: break;          /* keep default */
  264                 case 1:
  265                         watermark = 32; break;
  266                 case 2:
  267                         watermark = 48; break;
  268                 case 3: 
  269                         watermark = 56; break;
  270                 }
  271                 wm_intr_value = 0x2;    /* set watermark enable interrupt */
  272         }
  273         mflags = msgrng_access_enable();
  274         config = (watermark << 24) | (IRQ_MSGRING << 16) | (1 << 8) |
  275                 wm_intr_value;
  276         /* clear pending interrupts, they will get re-raised if still valid */
  277         write_c0_eirr64(1ULL << IRQ_MSGRING);
  278         msgrng_write_config(config);
  279         msgrng_restore(mflags);
  280 }
  281 
  282 /* Debug counters */
  283 static int msgring_nintr[XLR_MAX_CORES];
  284 static int msgring_badintr[XLR_MAX_CORES];
  285 static int msgring_wakeup_sleep[XLR_MAX_CORES * XLR_NTHREADS];
  286 static int msgring_wakeup_nosleep[XLR_MAX_CORES * XLR_NTHREADS];
  287 static int msgring_nmsgs[XLR_MAX_CORES * XLR_NTHREADS];
  288 
  289 static int
  290 msgring_process_fast_intr(void *arg)
  291 {
  292         struct msgring_thread *mthd;
  293         struct thread   *td;
  294         uint32_t        mflags;
  295         int             core, nt;
  296 
  297         core = xlr_core_id();
  298         mthd = &msgring_threads[core];
  299         msgring_nintr[core]++;
  300         mtx_lock_spin(&mthd->lock);
  301         nt = mthd->running;
  302         if(nt >= mthd->nthreads) {
  303                 msgring_badintr[core]++;
  304                 mtx_unlock_spin(&mthd->lock);
  305                 return (FILTER_HANDLED);
  306         }
  307 
  308         td = mthd->threads[nt].thread;
  309         mflags = msgrng_access_enable();
  310 
  311         /* default value with interrupts disabled */
  312         msgrng_write_config((1 << 24) | (IRQ_MSGRING << 16) | (1 << 8));
  313         /* clear pending interrupts */
  314         write_c0_eirr64(1ULL << IRQ_MSGRING);
  315         msgrng_restore(mflags);
  316         mtx_unlock_spin(&mthd->lock);
  317 
  318         /* wake up the target thread */
  319         mthd->threads[nt].needed = 1;
  320         thread_lock(td);
  321         if (TD_AWAITING_INTR(td)) {
  322                 msgring_wakeup_sleep[core*4+nt]++;
  323                 TD_CLR_IWAIT(td);
  324                 sched_add(td, SRQ_INTR);
  325         } else
  326                 msgring_wakeup_nosleep[core*4+nt]++;
  327         thread_unlock(td);
  328         return (FILTER_HANDLED);
  329 }
  330 
  331 static void
  332 msgring_process(void *arg)
  333 {
  334         struct msgring_thread *mthd;
  335         struct thread   *td;
  336         int             hwtid, tid, core;
  337         int             nmsgs;
  338 
  339         hwtid = (intptr_t)arg;
  340         core = hwtid / 4;
  341         tid = hwtid % 4;
  342         mthd = &msgring_threads[core];
  343         td = mthd->threads[tid].thread;
  344         KASSERT(curthread == td,
  345             ("Incorrect thread core %d, thread %d", core, hwtid));
  346 
  347         /* First bind this thread to the right CPU */
  348         thread_lock(td);
  349         sched_bind(td, xlr_hwtid_to_cpuid[hwtid]);
  350         thread_unlock(td);
  351 
  352         mtx_lock_spin(&mthd->lock);
  353         ++mthd->nthreads;               /* Active thread count */
  354         mtx_unlock_spin(&mthd->lock);
  355 
  356         /* start processing messages */
  357         for(;;) {
  358                 mtx_lock_spin(&mthd->lock);
  359                 ++mthd->running;
  360                 msgrng_setconfig(mthd->running, mthd->nthreads);
  361                 mtx_unlock_spin(&mthd->lock);
  362 
  363                 atomic_store_rel_int(&mthd->threads[tid].needed, 0);
  364                 nmsgs = xlr_msgring_handler(0xff, 0);
  365                 msgring_nmsgs[hwtid] += nmsgs;
  366 
  367                 mtx_lock_spin(&mthd->lock);
  368                 --mthd->running;
  369                 msgrng_setconfig(mthd->running, mthd->nthreads);
  370                 mtx_unlock_spin(&mthd->lock);
  371 
  372                 /* sleep */
  373                 thread_lock(td);
  374                 if (mthd->threads[tid].needed) {
  375                         thread_unlock(td);
  376                         continue;
  377                 }
  378                 sched_class(td, PRI_ITHD);
  379                 TD_SET_IWAIT(td);
  380                 mi_switch(SW_VOL, NULL);
  381                 thread_unlock(td);
  382         }
  383 }
  384 
  385 static void 
  386 create_msgring_thread(int hwtid)
  387 {
  388         struct msgring_thread *mthd;
  389         struct thread *td;
  390         int     tid, core;
  391         int     error;
  392 
  393         core = hwtid / 4;
  394         tid = hwtid % 4;
  395         mthd = &msgring_threads[core];
  396         if (tid == 0) {
  397                 mtx_init(&mthd->lock, "msgrngcore", NULL, MTX_SPIN);
  398                 mthd->running = mthd->nthreads = 0;
  399         }
  400         error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid,
  401             &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc",
  402             "msgthr%d", hwtid);
  403         if (error)
  404                 panic("kproc_kthread_add() failed with %d", error);
  405         mthd->threads[tid].thread = td;
  406 
  407         thread_lock(td);
  408         sched_class(td, PRI_ITHD);
  409         sched_add(td, SRQ_INTR);
  410         thread_unlock(td);
  411         CTR2(KTR_INTR, "%s: created %s", __func__, td->td_name);
  412 }
  413 
  414 int 
  415 register_msgring_handler(int startb, int endb, msgring_handler action,
  416     void *arg)
  417 {
  418         void    *cookie;
  419         int     i;
  420         static int msgring_int_enabled = 0;
  421 
  422         KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS,
  423             ("Invalid value for for bucket range %d,%d", startb, endb));
  424 
  425         mtx_lock_spin(&msgmap_lock);
  426         for (i = startb; i <= endb; i++) {
  427                 KASSERT(msgmap[i].action == NULL,
  428                    ("Bucket %d already used [action %p]", i, msgmap[i].action));
  429                 msgmap[i].action = action;
  430                 msgmap[i].arg = arg;
  431         }
  432         mtx_unlock_spin(&msgmap_lock);
  433 
  434         if (xlr_test_and_set(&msgring_int_enabled)) {
  435                 create_msgring_thread(0);
  436                 if (msgring_maxthreads > xlr_threads_per_core)
  437                         msgring_maxthreads = xlr_threads_per_core;
  438                 cpu_establish_hardintr("msgring", msgring_process_fast_intr,
  439                         NULL, NULL, IRQ_MSGRING, 
  440                         INTR_TYPE_NET | INTR_FAST, &cookie);
  441         }
  442         return (0);
  443 }
  444 
  445 /*
  446  * Start message ring processing threads on other CPUs, after SMP start
  447  */
  448 static void
  449 start_msgring_threads(void *arg)
  450 {
  451         int     hwt, tid;
  452 
  453         for (hwt = 1; hwt < XLR_MAX_CORES * XLR_NTHREADS; hwt++) {
  454                 if ((xlr_hw_thread_mask & (1 << hwt)) == 0)
  455                         continue;
  456                 tid = hwt % XLR_NTHREADS;
  457                 if (tid >= msgring_maxthreads)
  458                         continue;
  459                 create_msgring_thread(hwt);
  460         }
  461 }
  462 
  463 SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE,
  464     start_msgring_threads, NULL);
  465 
  466 /*
  467  * DEBUG support, XXX: static buffer, not locked 
  468  */
  469 static int
  470 sys_print_debug(SYSCTL_HANDLER_ARGS)
  471 {
  472         int error, nb, i, fs;
  473         static char xprintb[4096], *buf;
  474 
  475         buf = xprintb;
  476         fs = sizeof(xprintb);
  477         nb = snprintf(buf, fs,
  478             "\nID      INTR   ER   WU-SLP   WU-ERR     MSGS\n");
  479         buf += nb;
  480         fs -= nb;
  481         for (i = 0; i < 32; i++) {
  482                 if ((xlr_hw_thread_mask & (1 << i)) == 0)
  483                         continue;
  484                 nb = snprintf(buf, fs,
  485                     "%2d: %8d %4d %8d %8d %8d\n", i,
  486                     msgring_nintr[i/4], msgring_badintr[i/4],
  487                     msgring_wakeup_sleep[i], msgring_wakeup_nosleep[i],
  488                     msgring_nmsgs[i]);
  489                 buf += nb;
  490                 fs -= nb;
  491         } 
  492         error = SYSCTL_OUT(req, xprintb, buf - xprintb);
  493         return (error);
  494 }
  495 
  496 SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0,
  497     sys_print_debug, "A", "msgring debug info");

Cache object: 8169c52ff56b9da100fbd8c178a779c9


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.