pcpu.h

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-3-Clause
    3  *
    4  * Copyright (c) 2001 Wind River Systems, Inc.
    5  * All rights reserved.
    6  * Written by: John Baldwin <jhb@FreeBSD.org>
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 3. Neither the name of the author nor the names of any co-contributors
   17  *    may be used to endorse or promote products derived from this software
   18  *    without specific prior written permission.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  * $FreeBSD$
   33  */
   34 
   35 #ifndef _SYS_PCPU_H_
   36 #define _SYS_PCPU_H_
   37 
   38 #ifdef LOCORE
   39 #error "no assembler-serviceable parts inside"
   40 #endif
   41 
   42 #include <sys/_cpuset.h>
   43 #include <sys/_lock.h>
   44 #include <sys/_mutex.h>
   45 #include <sys/_sx.h>
   46 #include <sys/queue.h>
   47 #include <sys/_rmlock.h>
   48 #include <sys/resource.h>
   49 #include <machine/pcpu.h>
   50 
   51 #define DPCPU_SETNAME           "set_pcpu"
   52 #define DPCPU_SYMPREFIX         "pcpu_entry_"
   53 
   54 #ifdef _KERNEL
   55 
   56 /*
   57  * Define a set for pcpu data.
   58  */
   59 extern uintptr_t *__start_set_pcpu;
   60 __GLOBL(__start_set_pcpu);
   61 extern uintptr_t *__stop_set_pcpu;
   62 __GLOBL(__stop_set_pcpu);
   63 
   64 /*
   65  * Array of dynamic pcpu base offsets.  Indexed by id.
   66  */
   67 extern uintptr_t dpcpu_off[];
   68 
   69 /*
   70  * Convenience defines.
   71  */
   72 #define DPCPU_START             ((uintptr_t)&__start_set_pcpu)
   73 #define DPCPU_STOP              ((uintptr_t)&__stop_set_pcpu)
   74 #define DPCPU_BYTES             (DPCPU_STOP - DPCPU_START)
   75 #define DPCPU_MODMIN            2048
   76 #define DPCPU_SIZE              roundup2(DPCPU_BYTES, PAGE_SIZE)
   77 #define DPCPU_MODSIZE           (DPCPU_SIZE - (DPCPU_BYTES - DPCPU_MODMIN))
   78 
   79 /*
   80  * Declaration and definition.
   81  */
   82 #define DPCPU_NAME(n)           pcpu_entry_##n
   83 #define DPCPU_DECLARE(t, n)     extern t DPCPU_NAME(n)
   84 /* struct _hack is to stop this from being used with the static keyword. */
   85 #define DPCPU_DEFINE(t, n)      \
   86     struct _hack; t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
   87 #if defined(KLD_MODULE) && (defined(__aarch64__) || defined(__riscv))
   88 /*
   89  * On some architectures the compiler will use PC-relative load to
   90  * find the address of DPCPU data with the static keyword. We then
   91  * use this to find the offset of the data in a per-CPU region.
   92  * This works for in the kernel as we can allocate the space ahead
   93  * of time, however modules need to allocate a sepatate space and
   94  * then use relocations to fix the address of the data. As
   95  * PC-relative data doesn't have a relocation there is nothing for
   96  * the kernel module linker to fix so data is accessed from the
   97  * wrong location.
   98  *
   99  * This is a workaround until a better solution can be found.
  100  *
  101  * VNET_DEFINE_STATIC also has the same workaround.
  102  */
  103 #define DPCPU_DEFINE_STATIC(t, n)       \
  104     t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
  105 #else
  106 #define DPCPU_DEFINE_STATIC(t, n)       \
  107     static t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
  108 #endif
  109 
  110 /*
  111  * Accessors with a given base.
  112  */
  113 #define _DPCPU_PTR(b, n)                                                \
  114     (__typeof(DPCPU_NAME(n))*)((b) + (uintptr_t)&DPCPU_NAME(n))
  115 #define _DPCPU_GET(b, n)        (*_DPCPU_PTR(b, n))
  116 #define _DPCPU_SET(b, n, v)     (*_DPCPU_PTR(b, n) = v)
  117 
  118 /*
  119  * Accessors for the current cpu.
  120  */
  121 #define DPCPU_PTR(n)            _DPCPU_PTR(PCPU_GET(dynamic), n)
  122 #define DPCPU_GET(n)            (*DPCPU_PTR(n))
  123 #define DPCPU_SET(n, v)         (*DPCPU_PTR(n) = v)
  124 
  125 /*
  126  * Accessors for remote cpus.
  127  */
  128 #define DPCPU_ID_PTR(i, n)      _DPCPU_PTR(dpcpu_off[(i)], n)
  129 #define DPCPU_ID_GET(i, n)      (*DPCPU_ID_PTR(i, n))
  130 #define DPCPU_ID_SET(i, n, v)   (*DPCPU_ID_PTR(i, n) = v)
  131 
  132 /*
  133  * Utility macros.
  134  */
  135 #define DPCPU_SUM(n) __extension__                                      \
  136 ({                                                                      \
  137         u_int _i;                                                       \
  138         __typeof(*DPCPU_PTR(n)) sum;                                    \
  139                                                                         \
  140         sum = 0;                                                        \
  141         CPU_FOREACH(_i) {                                               \
  142                 sum += *DPCPU_ID_PTR(_i, n);                            \
  143         }                                                               \
  144         sum;                                                            \
  145 })
  146 
  147 #define DPCPU_VARSUM(n, var) __extension__                              \
  148 ({                                                                      \
  149         u_int _i;                                                       \
  150         __typeof((DPCPU_PTR(n))->var) sum;                              \
  151                                                                         \
  152         sum = 0;                                                        \
  153         CPU_FOREACH(_i) {                                               \
  154                 sum += (DPCPU_ID_PTR(_i, n))->var;                      \
  155         }                                                               \
  156         sum;                                                            \
  157 })
  158 
  159 #define DPCPU_ZERO(n) do {                                              \
  160         u_int _i;                                                       \
  161                                                                         \
  162         CPU_FOREACH(_i) {                                               \
  163                 bzero(DPCPU_ID_PTR(_i, n), sizeof(*DPCPU_PTR(n)));      \
  164         }                                                               \
  165 } while(0)
  166 
  167 #endif /* _KERNEL */
  168 
  169 /*
  170  * This structure maps out the global data that needs to be kept on a
  171  * per-cpu basis.  The members are accessed via the PCPU_GET/SET/PTR
  172  * macros defined in <machine/pcpu.h>.  Machine dependent fields are
  173  * defined in the PCPU_MD_FIELDS macro defined in <machine/pcpu.h>.
  174  */
  175 struct pcpu {
  176         struct thread   *pc_curthread;          /* Current thread */
  177         struct thread   *pc_idlethread;         /* Idle thread */
  178         struct thread   *pc_fpcurthread;        /* Fp state owner */
  179         struct thread   *pc_deadthread;         /* Zombie thread or NULL */
  180         struct pcb      *pc_curpcb;             /* Current pcb */
  181         uint64_t        pc_switchtime;          /* cpu_ticks() at last csw */
  182         int             pc_switchticks;         /* `ticks' at last csw */
  183         u_int           pc_cpuid;               /* This cpu number */
  184         STAILQ_ENTRY(pcpu) pc_allcpu;
  185         struct lock_list_entry *pc_spinlocks;
  186         long            pc_cp_time[CPUSTATES];  /* statclock ticks */
  187         struct device   *pc_device;
  188         void            *pc_netisr;             /* netisr SWI cookie */
  189         int             pc_unused1;             /* unused field */
  190         int             pc_domain;              /* Memory domain. */
  191         struct rm_queue pc_rm_queue;            /* rmlock list of trackers */
  192         uintptr_t       pc_dynamic;             /* Dynamic per-cpu data area */
  193         uint64_t        pc_early_dummy_counter; /* Startup time counter(9) */
  194 
  195         /*
  196          * Keep MD fields last, so that CPU-specific variations on a
  197          * single architecture don't result in offset variations of
  198          * the machine-independent fields of the pcpu.  Even though
  199          * the pcpu structure is private to the kernel, some ports
  200          * (e.g., lsof, part of gtop) define _KERNEL and include this
  201          * header.  While strictly speaking this is wrong, there's no
  202          * reason not to keep the offsets of the MI fields constant
  203          * if only to make kernel debugging easier.
  204          */
  205         PCPU_MD_FIELDS;
  206 } __aligned(CACHE_LINE_SIZE);
  207 
  208 #ifdef _KERNEL
  209 
  210 STAILQ_HEAD(cpuhead, pcpu);
  211 
  212 extern struct cpuhead cpuhead;
  213 extern struct pcpu *cpuid_to_pcpu[];
  214 
  215 #define curcpu          PCPU_GET(cpuid)
  216 #define curvidata       PCPU_GET(vidata)
  217 
  218 #define UMA_PCPU_ALLOC_SIZE             PAGE_SIZE
  219 
  220 #include <machine/pcpu_aux.h>
  221 
  222 #ifndef curthread
  223 #define curthread       PCPU_GET(curthread)
  224 #endif
  225 #define curproc         (curthread->td_proc)
  226 
  227 /* Accessor to elements allocated via UMA_ZONE_PCPU zone. */
  228 static inline void *
  229 zpcpu_get(void *base)
  230 {
  231 
  232         return ((char *)(base) + UMA_PCPU_ALLOC_SIZE * curcpu);
  233 }
  234 
  235 static inline void *
  236 zpcpu_get_cpu(void *base, int cpu)
  237 {
  238 
  239         return ((char *)(base) + UMA_PCPU_ALLOC_SIZE * cpu);
  240 }
  241 
  242 /*
  243  * Machine dependent callouts.  cpu_pcpu_init() is responsible for
  244  * initializing machine dependent fields of struct pcpu, and
  245  * db_show_mdpcpu() is responsible for handling machine dependent
  246  * fields for the DDB 'show pcpu' command.
  247  */
  248 void    cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size);
  249 void    db_show_mdpcpu(struct pcpu *pcpu);
  250 
  251 void    *dpcpu_alloc(int size);
  252 void    dpcpu_copy(void *s, int size);
  253 void    dpcpu_free(void *s, int size);
  254 void    dpcpu_init(void *dpcpu, int cpuid);
  255 void    pcpu_destroy(struct pcpu *pcpu);
  256 struct  pcpu *pcpu_find(u_int cpuid);
  257 void    pcpu_init(struct pcpu *pcpu, int cpuid, size_t size);
  258 
  259 #endif /* _KERNEL */
  260 
  261 #endif /* !_SYS_PCPU_H_ */
Cache object: 77791d70d371e0b5b51fc7b798f07472
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/sys/pcpu.h

FreeBSD/Linux Kernel Cross Reference
sys/sys/pcpu.h
