The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/include/counter.h

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  *
   26  * $FreeBSD: releng/10.3/sys/i386/include/counter.h 252434 2013-07-01 02:48:27Z kib $
   27  */
   28 
   29 #ifndef __MACHINE_COUNTER_H__
   30 #define __MACHINE_COUNTER_H__
   31 
   32 #include <sys/pcpu.h>
   33 #ifdef INVARIANTS
   34 #include <sys/proc.h>
   35 #endif
   36 #include <machine/md_var.h>
   37 #include <machine/specialreg.h>
   38 
   39 #define counter_enter() do {                            \
   40         if ((cpu_feature & CPUID_CX8) == 0)             \
   41                 critical_enter();                       \
   42 } while (0)
   43 
   44 #define counter_exit()  do {                            \
   45         if ((cpu_feature & CPUID_CX8) == 0)             \
   46                 critical_exit();                        \
   47 } while (0)
   48 
   49 extern struct pcpu __pcpu[MAXCPU];
   50 
   51 static inline void
   52 counter_64_inc_8b(uint64_t *p, int64_t inc)
   53 {
   54 
   55         __asm __volatile(
   56         "movl   %%fs:(%%esi),%%eax\n\t"
   57         "movl   %%fs:4(%%esi),%%edx\n"
   58 "1:\n\t"
   59         "movl   %%eax,%%ebx\n\t"
   60         "movl   %%edx,%%ecx\n\t"
   61         "addl   (%%edi),%%ebx\n\t"
   62         "adcl   4(%%edi),%%ecx\n\t"
   63         "cmpxchg8b %%fs:(%%esi)\n\t"
   64         "jnz    1b"
   65         :
   66         : "S" ((char *)p - (char *)&__pcpu[0]), "D" (&inc)
   67         : "memory", "cc", "eax", "edx", "ebx", "ecx");
   68 }
   69 
   70 #ifdef IN_SUBR_COUNTER_C
   71 static inline uint64_t
   72 counter_u64_read_one_8b(uint64_t *p)
   73 {
   74         uint32_t res_lo, res_high;
   75 
   76         __asm __volatile(
   77         "movl   %%eax,%%ebx\n\t"
   78         "movl   %%edx,%%ecx\n\t"
   79         "cmpxchg8b      (%2)"
   80         : "=a" (res_lo), "=d"(res_high)
   81         : "SD" (p)
   82         : "cc", "ebx", "ecx");
   83         return (res_lo + ((uint64_t)res_high << 32));
   84 }
   85 
   86 static inline uint64_t
   87 counter_u64_fetch_inline(uint64_t *p)
   88 {
   89         uint64_t res;
   90         int i;
   91 
   92         res = 0;
   93         if ((cpu_feature & CPUID_CX8) == 0) {
   94                 /*
   95                  * The machines without cmpxchg8b are not SMP.
   96                  * Disabling the preemption provides atomicity of the
   97                  * counter reading, since update is done in the
   98                  * critical section as well.
   99                  */
  100                 critical_enter();
  101                 for (i = 0; i < mp_ncpus; i++) {
  102                         res += *(uint64_t *)((char *)p +
  103                             sizeof(struct pcpu) * i);
  104                 }
  105                 critical_exit();
  106         } else {
  107                 for (i = 0; i < mp_ncpus; i++)
  108                         res += counter_u64_read_one_8b((uint64_t *)((char *)p +
  109                             sizeof(struct pcpu) * i));
  110         }
  111         return (res);
  112 }
  113 
  114 static inline void
  115 counter_u64_zero_one_8b(uint64_t *p)
  116 {
  117 
  118         __asm __volatile(
  119         "movl   (%0),%%eax\n\t"
  120         "movl   4(%0),%%edx\n"
  121         "xorl   %%ebx,%%ebx\n\t"
  122         "xorl   %%ecx,%%ecx\n\t"
  123 "1:\n\t"
  124         "cmpxchg8b      (%0)\n\t"
  125         "jnz    1b"
  126         :
  127         : "SD" (p)
  128         : "memory", "cc", "eax", "edx", "ebx", "ecx");
  129 }
  130 
  131 static void
  132 counter_u64_zero_one_cpu(void *arg)
  133 {
  134         uint64_t *p;
  135 
  136         p = (uint64_t *)((char *)arg + sizeof(struct pcpu) * PCPU_GET(cpuid));
  137         counter_u64_zero_one_8b(p);
  138 }
  139 
  140 static inline void
  141 counter_u64_zero_inline(counter_u64_t c)
  142 {
  143         int i;
  144 
  145         if ((cpu_feature & CPUID_CX8) == 0) {
  146                 critical_enter();
  147                 for (i = 0; i < mp_ncpus; i++)
  148                         *(uint64_t *)((char *)c + sizeof(struct pcpu) * i) = 0;
  149                 critical_exit();
  150         } else {
  151                 smp_rendezvous(smp_no_rendevous_barrier,
  152                     counter_u64_zero_one_cpu, smp_no_rendevous_barrier, c);
  153         }
  154 }
  155 #endif
  156 
  157 #define counter_u64_add_protected(c, inc)       do {    \
  158         if ((cpu_feature & CPUID_CX8) == 0) {           \
  159                 CRITICAL_ASSERT(curthread);             \
  160                 *(uint64_t *)zpcpu_get(c) += (inc);     \
  161         } else                                          \
  162                 counter_64_inc_8b((c), (inc));          \
  163 } while (0)
  164 
  165 static inline void
  166 counter_u64_add(counter_u64_t c, int64_t inc)
  167 {
  168 
  169         if ((cpu_feature & CPUID_CX8) == 0) {
  170                 critical_enter();
  171                 *(uint64_t *)zpcpu_get(c) += inc;
  172                 critical_exit();
  173         } else {
  174                 counter_64_inc_8b(c, inc);
  175         }
  176 }
  177 
  178 #endif  /* ! __MACHINE_COUNTER_H__ */

Cache object: cc5723a63d06c86d523c5d555c175d45


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.