counter.h

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  *
   28  * $FreeBSD$
   29  */
   30 
   31 #ifndef __MACHINE_COUNTER_H__
   32 #define __MACHINE_COUNTER_H__
   33 
   34 #include <sys/pcpu.h>
   35 #ifdef INVARIANTS
   36 #include <sys/proc.h>
   37 #endif
   38 #include <machine/md_var.h>
   39 #include <machine/specialreg.h>
   40 
   41 #define EARLY_COUNTER   &__pcpu[0].pc_early_dummy_counter
   42 
   43 #define counter_enter() do {                            \
   44         if ((cpu_feature & CPUID_CX8) == 0)             \
   45                 critical_enter();                       \
   46 } while (0)
   47 
   48 #define counter_exit()  do {                            \
   49         if ((cpu_feature & CPUID_CX8) == 0)             \
   50                 critical_exit();                        \
   51 } while (0)
   52 
   53 static inline void
   54 counter_64_inc_8b(uint64_t *p, int64_t inc)
   55 {
   56 
   57         __asm __volatile(
   58         "movl   %%fs:(%%esi),%%eax\n\t"
   59         "movl   %%fs:4(%%esi),%%edx\n"
   60 "1:\n\t"
   61         "movl   %%eax,%%ebx\n\t"
   62         "movl   %%edx,%%ecx\n\t"
   63         "addl   (%%edi),%%ebx\n\t"
   64         "adcl   4(%%edi),%%ecx\n\t"
   65         "cmpxchg8b %%fs:(%%esi)\n\t"
   66         "jnz    1b"
   67         :
   68         : "S" ((char *)p - (char *)&__pcpu[0]), "D" (&inc)
   69         : "memory", "cc", "eax", "edx", "ebx", "ecx");
   70 }
   71 
   72 #ifdef IN_SUBR_COUNTER_C
   73 struct counter_u64_fetch_cx8_arg {
   74         uint64_t res;
   75         uint64_t *p;
   76 };
   77 
   78 static uint64_t
   79 counter_u64_read_one_8b(uint64_t *p)
   80 {
   81         uint32_t res_lo, res_high;
   82 
   83         __asm __volatile(
   84         "movl   %%eax,%%ebx\n\t"
   85         "movl   %%edx,%%ecx\n\t"
   86         "cmpxchg8b      (%2)"
   87         : "=a" (res_lo), "=d"(res_high)
   88         : "SD" (p)
   89         : "cc", "ebx", "ecx");
   90         return (res_lo + ((uint64_t)res_high << 32));
   91 }
   92 
   93 static void
   94 counter_u64_fetch_cx8_one(void *arg1)
   95 {
   96         struct counter_u64_fetch_cx8_arg *arg;
   97         uint64_t val;
   98 
   99         arg = arg1;
  100         val = counter_u64_read_one_8b((uint64_t *)((char *)arg->p +
  101             UMA_PCPU_ALLOC_SIZE * PCPU_GET(cpuid)));
  102         atomic_add_64(&arg->res, val);
  103 }
  104 
  105 static inline uint64_t
  106 counter_u64_fetch_inline(uint64_t *p)
  107 {
  108         struct counter_u64_fetch_cx8_arg arg;
  109         uint64_t res;
  110         int i;
  111 
  112         res = 0;
  113         if ((cpu_feature & CPUID_CX8) == 0) {
  114                 /*
  115                  * The machines without cmpxchg8b are not SMP.
  116                  * Disabling the preemption provides atomicity of the
  117                  * counter reading, since update is done in the
  118                  * critical section as well.
  119                  */
  120                 critical_enter();
  121                 CPU_FOREACH(i) {
  122                         res += *(uint64_t *)((char *)p +
  123                             UMA_PCPU_ALLOC_SIZE * i);
  124                 }
  125                 critical_exit();
  126         } else {
  127                 arg.p = p;
  128                 arg.res = 0;
  129                 smp_rendezvous(NULL, counter_u64_fetch_cx8_one, NULL, &arg);
  130                 res = arg.res;
  131         }
  132         return (res);
  133 }
  134 
  135 static inline void
  136 counter_u64_zero_one_8b(uint64_t *p)
  137 {
  138 
  139         __asm __volatile(
  140         "movl   (%0),%%eax\n\t"
  141         "movl   4(%0),%%edx\n"
  142         "xorl   %%ebx,%%ebx\n\t"
  143         "xorl   %%ecx,%%ecx\n\t"
  144 "1:\n\t"
  145         "cmpxchg8b      (%0)\n\t"
  146         "jnz    1b"
  147         :
  148         : "SD" (p)
  149         : "memory", "cc", "eax", "edx", "ebx", "ecx");
  150 }
  151 
  152 static void
  153 counter_u64_zero_one_cpu(void *arg)
  154 {
  155         uint64_t *p;
  156 
  157         p = (uint64_t *)((char *)arg + UMA_PCPU_ALLOC_SIZE * PCPU_GET(cpuid));
  158         counter_u64_zero_one_8b(p);
  159 }
  160 
  161 static inline void
  162 counter_u64_zero_inline(counter_u64_t c)
  163 {
  164         int i;
  165 
  166         if ((cpu_feature & CPUID_CX8) == 0) {
  167                 critical_enter();
  168                 CPU_FOREACH(i)
  169                         *(uint64_t *)((char *)c + UMA_PCPU_ALLOC_SIZE * i) = 0;
  170                 critical_exit();
  171         } else {
  172                 smp_rendezvous(smp_no_rendezvous_barrier,
  173                     counter_u64_zero_one_cpu, smp_no_rendezvous_barrier, c);
  174         }
  175 }
  176 #endif
  177 
  178 #define counter_u64_add_protected(c, inc)       do {    \
  179         if ((cpu_feature & CPUID_CX8) == 0) {           \
  180                 CRITICAL_ASSERT(curthread);             \
  181                 *(uint64_t *)zpcpu_get(c) += (inc);     \
  182         } else                                          \
  183                 counter_64_inc_8b((c), (inc));          \
  184 } while (0)
  185 
  186 static inline void
  187 counter_u64_add(counter_u64_t c, int64_t inc)
  188 {
  189 
  190         if ((cpu_feature & CPUID_CX8) == 0) {
  191                 critical_enter();
  192                 *(uint64_t *)zpcpu_get(c) += inc;
  193                 critical_exit();
  194         } else {
  195                 counter_64_inc_8b(c, inc);
  196         }
  197 }
  198 
  199 #endif  /* ! __MACHINE_COUNTER_H__ */

Cache object: b30ea9c7fa6f5bf66f9318892a8e1e27

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.

FreeBSD/Linux Kernel Cross Reference sys/i386/include/counter.h

FreeBSD/Linux Kernel Cross Reference
sys/i386/include/counter.h