The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/include/counter.h

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  *
   28  * $FreeBSD: head/sys/i386/include/counter.h 343723 2019-02-03 21:28:58Z kib $
   29  */
   30 
   31 #ifndef __MACHINE_COUNTER_H__
   32 #define __MACHINE_COUNTER_H__
   33 
   34 #include <sys/pcpu.h>
   35 #ifdef INVARIANTS
   36 #include <sys/proc.h>
   37 #endif
   38 #include <machine/md_var.h>
   39 #include <machine/specialreg.h>
   40 
   41 extern struct pcpu __pcpu[];
   42 
   43 #define EARLY_COUNTER   &__pcpu[0].pc_early_dummy_counter
   44 
   45 #define counter_enter() do {                            \
   46         if ((cpu_feature & CPUID_CX8) == 0)             \
   47                 critical_enter();                       \
   48 } while (0)
   49 
   50 #define counter_exit()  do {                            \
   51         if ((cpu_feature & CPUID_CX8) == 0)             \
   52                 critical_exit();                        \
   53 } while (0)
   54 
   55 static inline void
   56 counter_64_inc_8b(uint64_t *p, int64_t inc)
   57 {
   58 
   59         __asm __volatile(
   60         "movl   %%fs:(%%esi),%%eax\n\t"
   61         "movl   %%fs:4(%%esi),%%edx\n"
   62 "1:\n\t"
   63         "movl   %%eax,%%ebx\n\t"
   64         "movl   %%edx,%%ecx\n\t"
   65         "addl   (%%edi),%%ebx\n\t"
   66         "adcl   4(%%edi),%%ecx\n\t"
   67         "cmpxchg8b %%fs:(%%esi)\n\t"
   68         "jnz    1b"
   69         :
   70         : "S" ((char *)p - (char *)&__pcpu[0]), "D" (&inc)
   71         : "memory", "cc", "eax", "edx", "ebx", "ecx");
   72 }
   73 
   74 #ifdef IN_SUBR_COUNTER_C
   75 struct counter_u64_fetch_cx8_arg {
   76         uint64_t res;
   77         uint64_t *p;
   78 };
   79 
   80 static uint64_t
   81 counter_u64_read_one_8b(uint64_t *p)
   82 {
   83         uint32_t res_lo, res_high;
   84 
   85         __asm __volatile(
   86         "movl   %%eax,%%ebx\n\t"
   87         "movl   %%edx,%%ecx\n\t"
   88         "cmpxchg8b      (%2)"
   89         : "=a" (res_lo), "=d"(res_high)
   90         : "SD" (p)
   91         : "cc", "ebx", "ecx");
   92         return (res_lo + ((uint64_t)res_high << 32));
   93 }
   94 
   95 static void
   96 counter_u64_fetch_cx8_one(void *arg1)
   97 {
   98         struct counter_u64_fetch_cx8_arg *arg;
   99         uint64_t val;
  100 
  101         arg = arg1;
  102         val = counter_u64_read_one_8b((uint64_t *)((char *)arg->p +
  103             UMA_PCPU_ALLOC_SIZE * PCPU_GET(cpuid)));
  104         atomic_add_64(&arg->res, val);
  105 }
  106 
  107 static inline uint64_t
  108 counter_u64_fetch_inline(uint64_t *p)
  109 {
  110         struct counter_u64_fetch_cx8_arg arg;
  111         uint64_t res;
  112         int i;
  113 
  114         res = 0;
  115         if ((cpu_feature & CPUID_CX8) == 0) {
  116                 /*
  117                  * The machines without cmpxchg8b are not SMP.
  118                  * Disabling the preemption provides atomicity of the
  119                  * counter reading, since update is done in the
  120                  * critical section as well.
  121                  */
  122                 critical_enter();
  123                 CPU_FOREACH(i) {
  124                         res += *(uint64_t *)((char *)p +
  125                             UMA_PCPU_ALLOC_SIZE * i);
  126                 }
  127                 critical_exit();
  128         } else {
  129                 arg.p = p;
  130                 arg.res = 0;
  131                 smp_rendezvous(NULL, counter_u64_fetch_cx8_one, NULL, &arg);
  132                 res = arg.res;
  133         }
  134         return (res);
  135 }
  136 
  137 static inline void
  138 counter_u64_zero_one_8b(uint64_t *p)
  139 {
  140 
  141         __asm __volatile(
  142         "movl   (%0),%%eax\n\t"
  143         "movl   4(%0),%%edx\n"
  144         "xorl   %%ebx,%%ebx\n\t"
  145         "xorl   %%ecx,%%ecx\n\t"
  146 "1:\n\t"
  147         "cmpxchg8b      (%0)\n\t"
  148         "jnz    1b"
  149         :
  150         : "SD" (p)
  151         : "memory", "cc", "eax", "edx", "ebx", "ecx");
  152 }
  153 
  154 static void
  155 counter_u64_zero_one_cpu(void *arg)
  156 {
  157         uint64_t *p;
  158 
  159         p = (uint64_t *)((char *)arg + UMA_PCPU_ALLOC_SIZE * PCPU_GET(cpuid));
  160         counter_u64_zero_one_8b(p);
  161 }
  162 
  163 static inline void
  164 counter_u64_zero_inline(counter_u64_t c)
  165 {
  166         int i;
  167 
  168         if ((cpu_feature & CPUID_CX8) == 0) {
  169                 critical_enter();
  170                 CPU_FOREACH(i)
  171                         *(uint64_t *)((char *)c + UMA_PCPU_ALLOC_SIZE * i) = 0;
  172                 critical_exit();
  173         } else {
  174                 smp_rendezvous(smp_no_rendezvous_barrier,
  175                     counter_u64_zero_one_cpu, smp_no_rendezvous_barrier, c);
  176         }
  177 }
  178 #endif
  179 
  180 #define counter_u64_add_protected(c, inc)       do {    \
  181         if ((cpu_feature & CPUID_CX8) == 0) {           \
  182                 CRITICAL_ASSERT(curthread);             \
  183                 *(uint64_t *)zpcpu_get(c) += (inc);     \
  184         } else                                          \
  185                 counter_64_inc_8b((c), (inc));          \
  186 } while (0)
  187 
  188 static inline void
  189 counter_u64_add(counter_u64_t c, int64_t inc)
  190 {
  191 
  192         if ((cpu_feature & CPUID_CX8) == 0) {
  193                 critical_enter();
  194                 *(uint64_t *)zpcpu_get(c) += inc;
  195                 critical_exit();
  196         } else {
  197                 counter_64_inc_8b(c, inc);
  198         }
  199 }
  200 
  201 #endif  /* ! __MACHINE_COUNTER_H__ */

Cache object: b30ea9c7fa6f5bf66f9318892a8e1e27


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.