cms.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights
    5  * reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions are
    9  * met:
   10  *
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in
   15  *    the documentation and/or other materials provided with the
   16  *    distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   21  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE
   22  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   23  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   24  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   25  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   26  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   27  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
   28  * THE POSSIBILITY OF SUCH DAMAGE.
   29  *
   30  * NETLOGIC_BSD */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD$");
   34 #include <sys/types.h>
   35 #include <sys/systm.h>
   36 #include <sys/param.h>
   37 #include <sys/lock.h>
   38 #include <sys/mutex.h>
   39 #include <sys/proc.h>
   40 #include <sys/limits.h>
   41 #include <sys/bus.h>
   42 #include <sys/sbuf.h>
   43 
   44 #include <sys/ktr.h>
   45 #include <sys/kernel.h>
   46 #include <sys/kthread.h>
   47 #include <sys/proc.h>
   48 #include <sys/resourcevar.h>
   49 #include <sys/sched.h>
   50 #include <sys/unistd.h>
   51 #include <sys/sysctl.h>
   52 #include <sys/malloc.h>
   53 
   54 #include <machine/reg.h>
   55 #include <machine/cpu.h>
   56 #include <machine/hwfunc.h>
   57 #include <machine/mips_opcode.h>
   58 #include <machine/intr_machdep.h>
   59 
   60 #include <mips/nlm/hal/mips-extns.h>
   61 #include <mips/nlm/hal/haldefs.h>
   62 #include <mips/nlm/hal/iomap.h>
   63 #include <mips/nlm/hal/cop2.h>
   64 #include <mips/nlm/hal/fmn.h>
   65 #include <mips/nlm/hal/pic.h>
   66 
   67 #include <mips/nlm/msgring.h>
   68 #include <mips/nlm/interrupt.h>
   69 #include <mips/nlm/xlp.h>
   70 
   71 #define MSGRNG_NSTATIONS        1024
   72 /*
   73  * Keep track of our message ring handler threads, each core has a
   74  * different message station. Ideally we will need to start a few
   75  * message handling threads every core, and wake them up depending on
   76  * load
   77  */
   78 struct msgring_thread {
   79         struct thread   *thread;        /* msgring handler threads */
   80         int     needed;                 /* thread needs to wake up */
   81 };
   82 static struct msgring_thread msgring_threads[XLP_MAX_CORES * XLP_MAX_THREADS];
   83 static struct proc *msgring_proc;       /* all threads are under a proc */
   84 
   85 /*
   86  * The device drivers can register a handler for the messages sent
   87  * from a station (corresponding to the device).
   88  */
   89 struct tx_stn_handler {
   90         msgring_handler action;
   91         void *arg;
   92 };
   93 static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS];
   94 static struct mtx       msgmap_lock;
   95 uint32_t xlp_msg_thread_mask;
   96 static int xlp_msg_threads_per_core = XLP_MAX_THREADS;
   97 
   98 static void create_msgring_thread(int hwtid);
   99 static int msgring_process_fast_intr(void *arg);
  100 
  101 /* Debug counters */
  102 static int msgring_nintr[XLP_MAX_CORES * XLP_MAX_THREADS];
  103 static int msgring_wakeup_sleep[XLP_MAX_CORES * XLP_MAX_THREADS];
  104 static int msgring_wakeup_nosleep[XLP_MAX_CORES * XLP_MAX_THREADS];
  105 static int fmn_msgcount[XLP_MAX_CORES * XLP_MAX_THREADS][4];
  106 static int fmn_loops[XLP_MAX_CORES * XLP_MAX_THREADS];
  107 
  108 /* Whether polled driver implementation */
  109 static int polled = 0;
  110 
  111 /* We do only i/o device credit setup here. CPU credit setup is now
  112  * moved to xlp_msgring_cpu_init() so that the credits get setup
  113  * only if the CPU exists. xlp_msgring_cpu_init() gets called from
  114  * platform_init_ap; and this makes it easy for us to setup CMS
  115  * credits for various types of XLP chips, with varying number of
  116  * cpu's and cores.
  117  */
  118 static void
  119 xlp_cms_credit_setup(int credit)
  120 {
  121         uint64_t cmspcibase, cmsbase, pcibase;
  122         uint32_t devoffset;
  123         int dev, fn, maxqid;
  124         int src, qid, i;
  125 
  126         for (i = 0; i < XLP_MAX_NODES; i++) {
  127                 cmspcibase = nlm_get_cms_pcibase(i);
  128                 if (!nlm_dev_exists(XLP_IO_CMS_OFFSET(i)))
  129                         continue;
  130                 cmsbase = nlm_get_cms_regbase(i);
  131                 maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
  132                 for (dev = 0; dev < 8; dev++) {
  133                         for (fn = 0; fn < 8; fn++) {
  134                                 devoffset = XLP_HDR_OFFSET(i, 0, dev, fn);
  135                                 if (nlm_dev_exists(devoffset) == 0)
  136                                         continue;
  137                                 pcibase = nlm_pcicfg_base(devoffset);
  138                                 src = nlm_qidstart(pcibase);
  139                                 if (src == 0)
  140                                         continue;
  141 #if 0 /* Debug */
  142                                 printf("Setup CMS credits for queues ");
  143                                 printf("[%d to %d] from src %d\n", 0,
  144                                     maxqid, src);
  145 #endif
  146                                 for (qid = 0; qid < maxqid; qid++)
  147                                         nlm_cms_setup_credits(cmsbase, qid,
  148                                             src, credit);
  149                         }
  150                 }
  151         }
  152 }
  153 
  154 void
  155 xlp_msgring_cpu_init(int node, int cpu, int credit)
  156 {
  157         uint64_t cmspcibase = nlm_get_cms_pcibase(node);
  158         uint64_t cmsbase = nlm_get_cms_regbase(node);
  159         int qid, maxqid, src;
  160 
  161         maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
  162 
  163         /* cpu credit setup is done only from thread-0 of each core */
  164         if((cpu % 4) == 0) {
  165                 src = cpu << 2; /* each thread has 4 vc's */
  166                 for (qid = 0; qid < maxqid; qid++)
  167                         nlm_cms_setup_credits(cmsbase, qid, src, credit);
  168         }
  169 }
  170 
  171 /*
  172  * Drain out max_messages for the buckets set in the bucket mask.
  173  * Use max_msgs = 0 to drain out all messages.
  174  */
  175 int
  176 xlp_handle_msg_vc(u_int vcmask, int max_msgs)
  177 {
  178         struct nlm_fmn_msg msg;
  179         int srcid = 0, size = 0, code = 0;
  180         struct tx_stn_handler *he;
  181         uint32_t mflags, status;
  182         int n_msgs = 0, vc, m, hwtid;
  183         u_int msgmask;
  184 
  185         hwtid = nlm_cpuid();
  186         for (;;) {
  187                 /* check if VC empty */
  188                 mflags = nlm_save_flags_cop2();
  189                 status = nlm_read_c2_msgstatus1();
  190                 nlm_restore_flags(mflags);
  191 
  192                 msgmask = ((status >> 24) & 0xf) ^ 0xf;
  193                 msgmask &= vcmask;
  194                 if (msgmask == 0)
  195                             break;
  196                 m = 0;
  197                 for (vc = 0; vc < 4; vc++) {
  198                         if ((msgmask & (1 << vc)) == 0)
  199                                 continue;
  200 
  201                         mflags = nlm_save_flags_cop2();
  202                         status = nlm_fmn_msgrcv(vc, &srcid, &size, &code,
  203                             &msg);
  204                         nlm_restore_flags(mflags);
  205                         if (status != 0)        /*  no msg or error */
  206                                 continue;
  207                         if (srcid < 0 || srcid >= 1024) {
  208                                 printf("[%s]: bad src id %d\n", __func__,
  209                                     srcid);
  210                                 continue;
  211                         }
  212                         he = &msgmap[srcid];
  213                         if(he->action != NULL)
  214                                 (he->action)(vc, size, code, srcid, &msg,
  215                                 he->arg);
  216 #if 0
  217                         else
  218                                 printf("[%s]: No Handler for msg from stn %d,"
  219                                     " vc=%d, size=%d, msg0=%jx, droppinge\n",
  220                                     __func__, srcid, vc, size,
  221                                     (uintmax_t)msg.msg[0]);
  222 #endif
  223                         fmn_msgcount[hwtid][vc] += 1;
  224                         m++;    /* msgs handled in this iter */
  225                 }
  226                 if (m == 0)
  227                         break;  /* nothing done in this iter */
  228                 n_msgs += m;
  229                 if (max_msgs > 0 && n_msgs >= max_msgs)
  230                         break;
  231         }
  232 
  233         return (n_msgs);
  234 }
  235 
  236 static void
  237 xlp_discard_msg_vc(u_int vcmask)
  238 {
  239         struct nlm_fmn_msg msg;
  240         int srcid = 0, size = 0, code = 0, vc;
  241         uint32_t mflags, status;
  242 
  243         for (vc = 0; vc < 4; vc++) {
  244                 for (;;) {
  245                         mflags = nlm_save_flags_cop2();
  246                         status = nlm_fmn_msgrcv(vc, &srcid,
  247                             &size, &code, &msg);
  248                         nlm_restore_flags(mflags);
  249 
  250                         /* break if there is no msg or error */
  251                         if (status != 0)
  252                                 break;
  253                 }
  254         }
  255 }
  256 
  257 void
  258 xlp_cms_enable_intr(int node, int cpu, int type, int watermark)
  259 {
  260         uint64_t cmsbase;
  261         int i, qid;
  262 
  263         cmsbase = nlm_get_cms_regbase(node);
  264 
  265         for (i = 0; i < 4; i++) {
  266                 qid = (i + (cpu * 4)) & 0x7f;
  267                 nlm_cms_per_queue_level_intr(cmsbase, qid, type, watermark);
  268                 nlm_cms_per_queue_timer_intr(cmsbase, qid, 0x1, 0);
  269         }
  270 }
  271 
  272 static int
  273 msgring_process_fast_intr(void *arg)
  274 {
  275         struct msgring_thread *mthd;
  276         struct thread *td;
  277         int     cpu;
  278 
  279         cpu = nlm_cpuid();
  280         mthd = &msgring_threads[cpu];
  281         msgring_nintr[cpu]++;
  282         td = mthd->thread;
  283 
  284         /* clear pending interrupts */
  285         nlm_write_c0_eirr(1ULL << IRQ_MSGRING);
  286 
  287         /* wake up the target thread */
  288         mthd->needed = 1;
  289         thread_lock(td);
  290         if (TD_AWAITING_INTR(td)) {
  291                 msgring_wakeup_sleep[cpu]++;
  292                 TD_CLR_IWAIT(td);
  293                 sched_add(td, SRQ_INTR);
  294         } else
  295                 msgring_wakeup_nosleep[cpu]++;
  296 
  297         thread_unlock(td);
  298 
  299         return (FILTER_HANDLED);
  300 }
  301 
  302 static void
  303 msgring_process(void * arg)
  304 {
  305         volatile struct msgring_thread *mthd;
  306         struct thread *td;
  307         uint32_t mflags, msgstatus1;
  308         int hwtid, nmsgs;
  309 
  310         hwtid = (intptr_t)arg;
  311         mthd = &msgring_threads[hwtid];
  312         td = mthd->thread;
  313         KASSERT(curthread == td,
  314             ("%s:msg_ithread and proc linkage out of sync", __func__));
  315 
  316         /* First bind this thread to the right CPU */
  317         thread_lock(td);
  318         sched_bind(td, xlp_hwtid_to_cpuid[hwtid]);
  319         thread_unlock(td);
  320 
  321         if (hwtid != nlm_cpuid())
  322                 printf("Misscheduled hwtid %d != cpuid %d\n", hwtid,
  323                     nlm_cpuid());
  324 
  325         xlp_discard_msg_vc(0xf);
  326         xlp_msgring_cpu_init(nlm_nodeid(), nlm_cpuid(), CMS_DEFAULT_CREDIT);
  327         if (polled == 0) {
  328                 mflags = nlm_save_flags_cop2();
  329                 nlm_fmn_cpu_init(IRQ_MSGRING, 0, 0, 0, 0, 0);
  330                 nlm_restore_flags(mflags);
  331                 xlp_cms_enable_intr(nlm_nodeid(), nlm_cpuid(), 0x2, 0);
  332                 /* clear pending interrupts.
  333                  *  they will get re-raised if still valid */
  334                 nlm_write_c0_eirr(1ULL << IRQ_MSGRING);
  335         }
  336 
  337         /* start processing messages */
  338         for (;;) {
  339                 atomic_store_rel_int(&mthd->needed, 0);
  340                 nmsgs = xlp_handle_msg_vc(0xf, 0);
  341 
  342                 /* sleep */
  343                 if (polled == 0) {
  344                         /* clear VC-pend bits */
  345                         mflags = nlm_save_flags_cop2();
  346                         msgstatus1 = nlm_read_c2_msgstatus1();
  347                         msgstatus1 |= (0xf << 16);
  348                         nlm_write_c2_msgstatus1(msgstatus1);
  349                         nlm_restore_flags(mflags);
  350 
  351                         thread_lock(td);
  352                         if (mthd->needed) {
  353                                 thread_unlock(td);
  354                                 continue;
  355                         }
  356                         sched_class(td, PRI_ITHD);
  357                         TD_SET_IWAIT(td);
  358                         mi_switch(SW_VOL, NULL);
  359                         thread_unlock(td);
  360                 } else
  361                         pause("wmsg", 1);
  362 
  363                 fmn_loops[hwtid]++;
  364         }
  365 }
  366 
  367 static void
  368 create_msgring_thread(int hwtid)
  369 {
  370         struct msgring_thread *mthd;
  371         struct thread *td;
  372         int     error;
  373 
  374         mthd = &msgring_threads[hwtid];
  375         error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid,
  376             &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc",
  377             "msgthr%d", hwtid);
  378         if (error)
  379                 panic("kproc_kthread_add() failed with %d", error);
  380         mthd->thread = td;
  381 
  382         thread_lock(td);
  383         sched_class(td, PRI_ITHD);
  384         sched_add(td, SRQ_INTR);
  385         thread_unlock(td);
  386 }
  387 
  388 int
  389 register_msgring_handler(int startb, int endb, msgring_handler action,
  390     void *arg)
  391 {
  392         int     i;
  393 
  394         if (bootverbose)
  395                 printf("Register handler %d-%d %p(%p)\n",
  396                     startb, endb, action, arg);
  397         KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS,
  398             ("Invalid value for bucket range %d,%d", startb, endb));
  399 
  400         mtx_lock_spin(&msgmap_lock);
  401         for (i = startb; i <= endb; i++) {
  402                 KASSERT(msgmap[i].action == NULL,
  403                    ("Bucket %d already used [action %p]", i, msgmap[i].action));
  404                 msgmap[i].action = action;
  405                 msgmap[i].arg = arg;
  406         }
  407         mtx_unlock_spin(&msgmap_lock);
  408         return (0);
  409 }
  410 
  411 /*
  412  * Initialize the messaging subsystem.
  413  *
  414  * Message Stations are shared among all threads in a cpu core, this
  415  * has to be called once from every core which is online.
  416  */
  417 static void
  418 xlp_msgring_config(void *arg)
  419 {
  420         void *cookie;
  421         unsigned int thrmask, mask;
  422         int i;
  423 
  424         /* used polled handler for Ax silion */
  425         if (nlm_is_xlp8xx_ax())
  426                 polled = 1;
  427 
  428         /* Don't poll on all threads, if polled */
  429         if (polled)
  430                 xlp_msg_threads_per_core -= 1;
  431 
  432         mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN);
  433         if (xlp_threads_per_core < xlp_msg_threads_per_core)
  434                 xlp_msg_threads_per_core = xlp_threads_per_core;
  435         thrmask = ((1 << xlp_msg_threads_per_core) - 1);
  436         mask = 0;
  437         for (i = 0; i < XLP_MAX_CORES; i++) {
  438                 mask <<= XLP_MAX_THREADS;
  439                 mask |= thrmask;
  440         }
  441         xlp_msg_thread_mask = xlp_hw_thread_mask & mask;
  442 #if 0
  443         printf("CMS Message handler thread mask %#jx\n",
  444             (uintmax_t)xlp_msg_thread_mask);
  445 #endif
  446         xlp_cms_credit_setup(CMS_DEFAULT_CREDIT);
  447         create_msgring_thread(0);
  448         cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL,
  449             NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie);
  450 }
  451 
  452 /*
  453  * Start message ring processing threads on other CPUs, after SMP start
  454  */
  455 static void
  456 start_msgring_threads(void *arg)
  457 {
  458         int     hwt;
  459 
  460         for (hwt = 1; hwt < XLP_MAX_CORES * XLP_MAX_THREADS; hwt++) {
  461                 if ((xlp_msg_thread_mask & (1 << hwt)) == 0)
  462                         continue;
  463                 create_msgring_thread(hwt);
  464         }
  465 }
  466 
  467 SYSINIT(xlp_msgring_config, SI_SUB_DRIVERS, SI_ORDER_FIRST,
  468     xlp_msgring_config, NULL);
  469 SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE,
  470     start_msgring_threads, NULL);
  471 
  472 /*
  473  * DEBUG support, XXX: static buffer, not locked
  474  */
  475 static int
  476 sys_print_debug(SYSCTL_HANDLER_ARGS)
  477 {
  478         struct sbuf sb;
  479         int error, i;
  480 
  481         sbuf_new_for_sysctl(&sb, NULL, 64, req);
  482         sbuf_printf(&sb, 
  483             "\nID     vc0       vc1       vc2     vc3     loops\n");
  484         for (i = 0; i < 32; i++) {
  485                 if ((xlp_hw_thread_mask & (1 << i)) == 0)
  486                         continue;
  487                 sbuf_printf(&sb, "%2d: %8d %8d %8d %8d %8d\n", i,
  488                     fmn_msgcount[i][0], fmn_msgcount[i][1],
  489                     fmn_msgcount[i][2], fmn_msgcount[i][3],
  490                     fmn_loops[i]);
  491         }
  492         error = sbuf_finish(&sb);
  493         sbuf_delete(&sb);
  494         return (error);
  495 }
  496 
  497 SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0,
  498     sys_print_debug, "A", "msgring debug info");
Cache object: b85262ee224e64aace5545e121698cce
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/mips/nlm/cms.c

FreeBSD/Linux Kernel Cross Reference
sys/mips/nlm/cms.c
