The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/include/counter.h

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  *
   26  * $FreeBSD: stable/11/sys/i386/include/counter.h 344230 2019-02-17 10:01:42Z kib $
   27  */
   28 
   29 #ifndef __MACHINE_COUNTER_H__
   30 #define __MACHINE_COUNTER_H__
   31 
   32 #include <sys/pcpu.h>
   33 #ifdef INVARIANTS
   34 #include <sys/proc.h>
   35 #endif
   36 #include <machine/md_var.h>
   37 #include <machine/specialreg.h>
   38 
   39 #define counter_enter() do {                            \
   40         if ((cpu_feature & CPUID_CX8) == 0)             \
   41                 critical_enter();                       \
   42 } while (0)
   43 
   44 #define counter_exit()  do {                            \
   45         if ((cpu_feature & CPUID_CX8) == 0)             \
   46                 critical_exit();                        \
   47 } while (0)
   48 
   49 extern struct pcpu __pcpu[MAXCPU];
   50 
   51 static inline void
   52 counter_64_inc_8b(uint64_t *p, int64_t inc)
   53 {
   54 
   55         __asm __volatile(
   56         "movl   %%fs:(%%esi),%%eax\n\t"
   57         "movl   %%fs:4(%%esi),%%edx\n"
   58 "1:\n\t"
   59         "movl   %%eax,%%ebx\n\t"
   60         "movl   %%edx,%%ecx\n\t"
   61         "addl   (%%edi),%%ebx\n\t"
   62         "adcl   4(%%edi),%%ecx\n\t"
   63         "cmpxchg8b %%fs:(%%esi)\n\t"
   64         "jnz    1b"
   65         :
   66         : "S" ((char *)p - (char *)&__pcpu[0]), "D" (&inc)
   67         : "memory", "cc", "eax", "edx", "ebx", "ecx");
   68 }
   69 
   70 #ifdef IN_SUBR_COUNTER_C
   71 struct counter_u64_fetch_cx8_arg {
   72         uint64_t res;
   73         uint64_t *p;
   74 };
   75 
   76 static uint64_t
   77 counter_u64_read_one_8b(uint64_t *p)
   78 {
   79         uint32_t res_lo, res_high;
   80 
   81         __asm __volatile(
   82         "movl   %%eax,%%ebx\n\t"
   83         "movl   %%edx,%%ecx\n\t"
   84         "cmpxchg8b      (%2)"
   85         : "=a" (res_lo), "=d"(res_high)
   86         : "SD" (p)
   87         : "cc", "ebx", "ecx");
   88         return (res_lo + ((uint64_t)res_high << 32));
   89 }
   90 
   91 static void
   92 counter_u64_fetch_cx8_one(void *arg1)
   93 {
   94         struct counter_u64_fetch_cx8_arg *arg;
   95         uint64_t val;
   96 
   97         arg = arg1;
   98         val = counter_u64_read_one_8b((uint64_t *)((char *)arg->p +
   99             sizeof(struct pcpu) * PCPU_GET(cpuid)));
  100         atomic_add_64(&arg->res, val);
  101 }
  102 
  103 static inline uint64_t
  104 counter_u64_fetch_inline(uint64_t *p)
  105 {
  106         struct counter_u64_fetch_cx8_arg arg;
  107         uint64_t res;
  108         int i;
  109 
  110         res = 0;
  111         if ((cpu_feature & CPUID_CX8) == 0) {
  112                 /*
  113                  * The machines without cmpxchg8b are not SMP.
  114                  * Disabling the preemption provides atomicity of the
  115                  * counter reading, since update is done in the
  116                  * critical section as well.
  117                  */
  118                 critical_enter();
  119                 CPU_FOREACH(i) {
  120                         res += *(uint64_t *)((char *)p +
  121                             sizeof(struct pcpu) * i);
  122                 }
  123                 critical_exit();
  124         } else {
  125                 arg.p = p;
  126                 arg.res = 0;
  127                 smp_rendezvous(NULL, counter_u64_fetch_cx8_one, NULL, &arg);
  128                 res = arg.res;
  129         }
  130         return (res);
  131 }
  132 
  133 static inline void
  134 counter_u64_zero_one_8b(uint64_t *p)
  135 {
  136 
  137         __asm __volatile(
  138         "movl   (%0),%%eax\n\t"
  139         "movl   4(%0),%%edx\n"
  140         "xorl   %%ebx,%%ebx\n\t"
  141         "xorl   %%ecx,%%ecx\n\t"
  142 "1:\n\t"
  143         "cmpxchg8b      (%0)\n\t"
  144         "jnz    1b"
  145         :
  146         : "SD" (p)
  147         : "memory", "cc", "eax", "edx", "ebx", "ecx");
  148 }
  149 
  150 static void
  151 counter_u64_zero_one_cpu(void *arg)
  152 {
  153         uint64_t *p;
  154 
  155         p = (uint64_t *)((char *)arg + sizeof(struct pcpu) * PCPU_GET(cpuid));
  156         counter_u64_zero_one_8b(p);
  157 }
  158 
  159 static inline void
  160 counter_u64_zero_inline(counter_u64_t c)
  161 {
  162         int i;
  163 
  164         if ((cpu_feature & CPUID_CX8) == 0) {
  165                 critical_enter();
  166                 CPU_FOREACH(i)
  167                         *(uint64_t *)((char *)c + sizeof(struct pcpu) * i) = 0;
  168                 critical_exit();
  169         } else {
  170                 smp_rendezvous(smp_no_rendezvous_barrier,
  171                     counter_u64_zero_one_cpu, smp_no_rendezvous_barrier, c);
  172         }
  173 }
  174 #endif
  175 
  176 #define counter_u64_add_protected(c, inc)       do {    \
  177         if ((cpu_feature & CPUID_CX8) == 0) {           \
  178                 CRITICAL_ASSERT(curthread);             \
  179                 *(uint64_t *)zpcpu_get(c) += (inc);     \
  180         } else                                          \
  181                 counter_64_inc_8b((c), (inc));          \
  182 } while (0)
  183 
  184 static inline void
  185 counter_u64_add(counter_u64_t c, int64_t inc)
  186 {
  187 
  188         if ((cpu_feature & CPUID_CX8) == 0) {
  189                 critical_enter();
  190                 *(uint64_t *)zpcpu_get(c) += inc;
  191                 critical_exit();
  192         } else {
  193                 counter_64_inc_8b(c, inc);
  194         }
  195 }
  196 
  197 #endif  /* ! __MACHINE_COUNTER_H__ */

Cache object: cc11d4ee31c44cbc7d287905d6d694ff


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.