The Design and Implementation of the FreeBSD Operating System, Second Edition

Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/arm64/arm64/identcpu.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10

    1 /*-
    2  * Copyright (c) 2014 Andrew Turner
    3  * Copyright (c) 2014 The FreeBSD Foundation
    4  * All rights reserved.
    5  *
    6  * Portions of this software were developed by Semihalf
    7  * under sponsorship of the FreeBSD Foundation.
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD$");
   34 
   35 #include <sys/param.h>
   36 #include <sys/kernel.h>
   37 #include <sys/pcpu.h>
   38 #include <sys/sbuf.h>
   39 #include <sys/smp.h>
   40 #include <sys/sysctl.h>
   41 #include <sys/systm.h>
   42 
   43 #include <machine/atomic.h>
   44 #include <machine/cpu.h>
   45 #include <machine/cpufunc.h>
   46 #include <machine/elf.h>
   47 #include <machine/md_var.h>
   48 #include <machine/undefined.h>
   49 
   50 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
   51 static void print_cpu_features(u_int cpu);
   52 static void print_cpu_caches(struct sbuf *sb, u_int);
   53 #ifdef COMPAT_FREEBSD32
   54 static u_long parse_cpu_features_hwcap32(void);
   55 #endif
   56 
   57 char machine[] = "arm64";
   58 
   59 #ifdef SCTL_MASK32
   60 extern int adaptive_machine_arch;
   61 #endif
   62 
   63 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
   64     "Cache management tuning");
   65 
   66 static int allow_dic = 1;
   67 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
   68     "Allow optimizations based on the DIC cache bit");
   69 
   70 static int allow_idc = 1;
   71 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
   72     "Allow optimizations based on the IDC cache bit");
   73 
   74 static void check_cpu_regs(u_int cpu);
   75 
   76 /*
   77  * The default implementation of I-cache sync assumes we have an
   78  * aliasing cache until we know otherwise.
   79  */
   80 void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
   81     &arm64_aliasing_icache_sync_range;
   82 
   83 static int
   84 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
   85 {
   86 #ifdef SCTL_MASK32
   87         static const char machine32[] = "arm";
   88 #endif
   89         int error;
   90 
   91 #ifdef SCTL_MASK32
   92         if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
   93                 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
   94         else
   95 #endif
   96                 error = SYSCTL_OUT(req, machine, sizeof(machine));
   97         return (error);
   98 }
   99 
  100 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
  101         CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A", "Machine class");
  102 
  103 static char cpu_model[64];
  104 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
  105         cpu_model, sizeof(cpu_model), "Machine model");
  106 
  107 #define MAX_CACHES      8       /* Maximum number of caches supported
  108                                    architecturally. */
  109 /*
  110  * Per-CPU affinity as provided in MPIDR_EL1
  111  * Indexed by CPU number in logical order selected by the system.
  112  * Relevant fields can be extracted using CPU_AFFn macros,
  113  * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
  114  *
  115  * Fields used by us:
  116  * Aff1 - Cluster number
  117  * Aff0 - CPU number in Aff1 cluster
  118  */
  119 uint64_t __cpu_affinity[MAXCPU];
  120 static u_int cpu_aff_levels;
  121 
  122 struct cpu_desc {
  123         uint64_t        mpidr;
  124         uint64_t        id_aa64afr0;
  125         uint64_t        id_aa64afr1;
  126         uint64_t        id_aa64dfr0;
  127         uint64_t        id_aa64dfr1;
  128         uint64_t        id_aa64isar0;
  129         uint64_t        id_aa64isar1;
  130         uint64_t        id_aa64isar2;
  131         uint64_t        id_aa64mmfr0;
  132         uint64_t        id_aa64mmfr1;
  133         uint64_t        id_aa64mmfr2;
  134         uint64_t        id_aa64pfr0;
  135         uint64_t        id_aa64pfr1;
  136         uint64_t        id_aa64zfr0;
  137         uint64_t        ctr;
  138 #ifdef COMPAT_FREEBSD32
  139         uint64_t        id_isar5;
  140         uint64_t        mvfr0;
  141         uint64_t        mvfr1;
  142 #endif
  143         uint64_t        clidr;
  144         uint32_t        ccsidr[MAX_CACHES][2]; /* 2 possible types. */
  145         bool            have_sve;
  146 };
  147 
  148 static struct cpu_desc cpu_desc[MAXCPU];
  149 static struct cpu_desc kern_cpu_desc;
  150 static struct cpu_desc user_cpu_desc;
  151 
  152 struct cpu_parts {
  153         u_int           part_id;
  154         const char      *part_name;
  155 };
  156 #define CPU_PART_NONE   { 0, NULL }
  157 
  158 struct cpu_implementers {
  159         u_int                   impl_id;
  160         const char              *impl_name;
  161         /*
  162          * Part number is implementation defined
  163          * so each vendor will have its own set of values and names.
  164          */
  165         const struct cpu_parts  *cpu_parts;
  166 };
  167 #define CPU_IMPLEMENTER_NONE    { 0, NULL, NULL }
  168 
  169 /*
  170  * Per-implementer table of (PartNum, CPU Name) pairs.
  171  */
  172 /* ARM Ltd. */
  173 static const struct cpu_parts cpu_parts_arm[] = {
  174         { CPU_PART_AEM_V8, "AEMv8" },
  175         { CPU_PART_FOUNDATION, "Foundation-Model" },
  176         { CPU_PART_CORTEX_A34, "Cortex-A34" },
  177         { CPU_PART_CORTEX_A35, "Cortex-A35" },
  178         { CPU_PART_CORTEX_A53, "Cortex-A53" },
  179         { CPU_PART_CORTEX_A55, "Cortex-A55" },
  180         { CPU_PART_CORTEX_A57, "Cortex-A57" },
  181         { CPU_PART_CORTEX_A65, "Cortex-A65" },
  182         { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
  183         { CPU_PART_CORTEX_A72, "Cortex-A72" },
  184         { CPU_PART_CORTEX_A73, "Cortex-A73" },
  185         { CPU_PART_CORTEX_A75, "Cortex-A75" },
  186         { CPU_PART_CORTEX_A76, "Cortex-A76" },
  187         { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
  188         { CPU_PART_CORTEX_A77, "Cortex-A77" },
  189         { CPU_PART_CORTEX_A78, "Cortex-A78" },
  190         { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
  191         { CPU_PART_CORTEX_A510, "Cortex-A510" },
  192         { CPU_PART_CORTEX_A710, "Cortex-A710" },
  193         { CPU_PART_CORTEX_A715, "Cortex-A715" },
  194         { CPU_PART_CORTEX_X1, "Cortex-X1" },
  195         { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
  196         { CPU_PART_CORTEX_X2, "Cortex-X2" },
  197         { CPU_PART_CORTEX_X3, "Cortex-X3" },
  198         { CPU_PART_NEOVERSE_E1, "Neoverse-E1" },
  199         { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
  200         { CPU_PART_NEOVERSE_N2, "Neoverse-N2" },
  201         { CPU_PART_NEOVERSE_V1, "Neoverse-V1" },
  202         { CPU_PART_NEOVERSE_V2, "Neoverse-V2" },
  203         CPU_PART_NONE,
  204 };
  205 
  206 /* Cavium */
  207 static const struct cpu_parts cpu_parts_cavium[] = {
  208         { CPU_PART_THUNDERX, "ThunderX" },
  209         { CPU_PART_THUNDERX2, "ThunderX2" },
  210         CPU_PART_NONE,
  211 };
  212 
  213 /* APM / Ampere */
  214 static const struct cpu_parts cpu_parts_apm[] = {
  215         { CPU_PART_EMAG8180, "eMAG 8180" },
  216         CPU_PART_NONE,
  217 };
  218 
  219 /* Qualcomm */
  220 static const struct cpu_parts cpu_parts_qcom[] = {
  221         { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
  222         { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
  223         CPU_PART_NONE,
  224 };
  225 
  226 /* Unknown */
  227 static const struct cpu_parts cpu_parts_none[] = {
  228         CPU_PART_NONE,
  229 };
  230 
  231 /*
  232  * Implementers table.
  233  */
  234 const struct cpu_implementers cpu_implementers[] = {
  235         { CPU_IMPL_AMPERE,      "Ampere",       cpu_parts_none },
  236         { CPU_IMPL_APPLE,       "Apple",        cpu_parts_none },
  237         { CPU_IMPL_APM,         "APM",          cpu_parts_apm },
  238         { CPU_IMPL_ARM,         "ARM",          cpu_parts_arm },
  239         { CPU_IMPL_BROADCOM,    "Broadcom",     cpu_parts_none },
  240         { CPU_IMPL_CAVIUM,      "Cavium",       cpu_parts_cavium },
  241         { CPU_IMPL_DEC,         "DEC",          cpu_parts_none },
  242         { CPU_IMPL_FREESCALE,   "Freescale",    cpu_parts_none },
  243         { CPU_IMPL_FUJITSU,     "Fujitsu",      cpu_parts_none },
  244         { CPU_IMPL_INFINEON,    "IFX",          cpu_parts_none },
  245         { CPU_IMPL_INTEL,       "Intel",        cpu_parts_none },
  246         { CPU_IMPL_MARVELL,     "Marvell",      cpu_parts_none },
  247         { CPU_IMPL_NVIDIA,      "NVIDIA",       cpu_parts_none },
  248         { CPU_IMPL_QUALCOMM,    "Qualcomm",     cpu_parts_qcom },
  249         CPU_IMPLEMENTER_NONE,
  250 };
  251 
  252 #define MRS_TYPE_MASK           0xf
  253 #define MRS_INVALID             0
  254 #define MRS_EXACT               1
  255 #define MRS_EXACT_VAL(x)        (MRS_EXACT | ((x) << 4))
  256 #define MRS_EXACT_FIELD(x)      ((x) >> 4)
  257 #define MRS_LOWER               2
  258 
  259 struct mrs_field_value {
  260         uint64_t        value;
  261         const char      *desc;
  262 };
  263 
  264 #define MRS_FIELD_VALUE(_value, _desc)                                  \
  265         {                                                               \
  266                 .value = (_value),                                      \
  267                 .desc = (_desc),                                        \
  268         }
  269 
  270 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl)           \
  271         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""),         \
  272         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
  273 
  274 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc)                      \
  275         MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
  276         MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
  277         MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
  278         MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
  279         MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
  280         MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
  281         MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
  282         MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
  283         MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
  284         MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
  285         MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
  286         MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
  287         MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
  288         MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
  289         MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
  290         MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
  291 
  292 #define MRS_FIELD_VALUE_END     { .desc = NULL }
  293 
  294 struct mrs_field_hwcap {
  295         u_long          *hwcap;
  296         uint64_t        min;
  297         u_long          hwcap_val;
  298 };
  299 
  300 #define MRS_HWCAP(_hwcap, _val, _min)                           \
  301 {                                                               \
  302         .hwcap = (_hwcap),                                      \
  303         .hwcap_val = (_val),                                    \
  304         .min = (_min),                                          \
  305 }
  306 
  307 #define MRS_HWCAP_END           { .hwcap = NULL }
  308 
  309 struct mrs_field {
  310         const char      *name;
  311         struct mrs_field_value *values;
  312         struct mrs_field_hwcap *hwcaps;
  313         uint64_t        mask;
  314         bool            sign;
  315         u_int           type;
  316         u_int           shift;
  317 };
  318 
  319 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, _hwcap) \
  320         {                                                               \
  321                 .name = #_name,                                         \
  322                 .sign = (_sign),                                        \
  323                 .type = (_type),                                        \
  324                 .shift = _register ## _ ## _name ## _SHIFT,             \
  325                 .mask = _register ## _ ## _name ## _MASK,               \
  326                 .values = (_values),                                    \
  327                 .hwcaps = (_hwcap),                                     \
  328         }
  329 
  330 #define MRS_FIELD(_register, _name, _sign, _type, _values)              \
  331         MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, NULL)
  332 
  333 #define MRS_FIELD_END   { .type = MRS_INVALID, }
  334 
  335 /* ID_AA64AFR0_EL1 */
  336 static struct mrs_field id_aa64afr0_fields[] = {
  337         MRS_FIELD_END,
  338 };
  339 
  340 
  341 /* ID_AA64AFR1_EL1 */
  342 static struct mrs_field id_aa64afr1_fields[] = {
  343         MRS_FIELD_END,
  344 };
  345 
  346 
  347 /* ID_AA64DFR0_EL1 */
  348 static struct mrs_field_value id_aa64dfr0_tracefilt[] = {
  349         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
  350         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
  351         MRS_FIELD_VALUE_END,
  352 };
  353 
  354 static struct mrs_field_value id_aa64dfr0_doublelock[] = {
  355         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
  356         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
  357         MRS_FIELD_VALUE_END,
  358 };
  359 
  360 static struct mrs_field_value id_aa64dfr0_pmsver[] = {
  361         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
  362         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
  363         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_8_3, "SPE v8.3"),
  364         MRS_FIELD_VALUE_END,
  365 };
  366 
  367 static struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
  368         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
  369         MRS_FIELD_VALUE_END,
  370 };
  371 
  372 static struct mrs_field_value id_aa64dfr0_wrps[] = {
  373         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
  374         MRS_FIELD_VALUE_END,
  375 };
  376 
  377 static struct mrs_field_value id_aa64dfr0_brps[] = {
  378         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
  379         MRS_FIELD_VALUE_END,
  380 };
  381 
  382 static struct mrs_field_value id_aa64dfr0_pmuver[] = {
  383         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
  384         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
  385         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3 v8.1"),
  386         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3 v8.4"),
  387         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3 v8.5"),
  388         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
  389         MRS_FIELD_VALUE_END,
  390 };
  391 
  392 static struct mrs_field_value id_aa64dfr0_tracever[] = {
  393         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
  394         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
  395         MRS_FIELD_VALUE_END,
  396 };
  397 
  398 static struct mrs_field_value id_aa64dfr0_debugver[] = {
  399         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
  400         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
  401         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8.2"),
  402         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8.4"),
  403         MRS_FIELD_VALUE_END,
  404 };
  405 
  406 static struct mrs_field id_aa64dfr0_fields[] = {
  407         MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_EXACT,
  408             id_aa64dfr0_tracefilt),
  409         MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_EXACT,
  410             id_aa64dfr0_doublelock),
  411         MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver),
  412         MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT,
  413             id_aa64dfr0_ctx_cmps),
  414         MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, id_aa64dfr0_wrps),
  415         MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps),
  416         MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver),
  417         MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT,
  418             id_aa64dfr0_tracever),
  419         MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6),
  420             id_aa64dfr0_debugver),
  421         MRS_FIELD_END,
  422 };
  423 
  424 
  425 /* ID_AA64DFR1_EL1 */
  426 static struct mrs_field id_aa64dfr1_fields[] = {
  427         MRS_FIELD_END,
  428 };
  429 
  430 
  431 /* ID_AA64ISAR0_EL1 */
  432 static struct mrs_field_value id_aa64isar0_rndr[] = {
  433         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
  434         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
  435         MRS_FIELD_VALUE_END,
  436 };
  437 
  438 static struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
  439         MRS_HWCAP(&elf_hwcap2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
  440         MRS_HWCAP_END
  441 };
  442 
  443 static struct mrs_field_value id_aa64isar0_tlb[] = {
  444         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
  445         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
  446         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
  447         MRS_FIELD_VALUE_END,
  448 };
  449 
  450 static struct mrs_field_value id_aa64isar0_ts[] = {
  451         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
  452         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
  453         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
  454         MRS_FIELD_VALUE_END,
  455 };
  456 
  457 static struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
  458         MRS_HWCAP(&elf_hwcap, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
  459         MRS_HWCAP(&elf_hwcap2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
  460         MRS_HWCAP_END
  461 };
  462 
  463 static struct mrs_field_value id_aa64isar0_fhm[] = {
  464         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
  465         MRS_FIELD_VALUE_END,
  466 };
  467 
  468 static struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
  469         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
  470         MRS_HWCAP_END
  471 };
  472 
  473 static struct mrs_field_value id_aa64isar0_dp[] = {
  474         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
  475         MRS_FIELD_VALUE_END,
  476 };
  477 
  478 static struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
  479         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
  480         MRS_HWCAP_END
  481 };
  482 
  483 static struct mrs_field_value id_aa64isar0_sm4[] = {
  484         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
  485         MRS_FIELD_VALUE_END,
  486 };
  487 
  488 static struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
  489         MRS_HWCAP(&elf_hwcap, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
  490         MRS_HWCAP_END
  491 };
  492 
  493 static struct mrs_field_value id_aa64isar0_sm3[] = {
  494         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
  495         MRS_FIELD_VALUE_END,
  496 };
  497 
  498 static struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
  499         MRS_HWCAP(&elf_hwcap, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
  500         MRS_HWCAP_END
  501 };
  502 
  503 static struct mrs_field_value id_aa64isar0_sha3[] = {
  504         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
  505         MRS_FIELD_VALUE_END,
  506 };
  507 
  508 static struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
  509         MRS_HWCAP(&elf_hwcap, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
  510         MRS_HWCAP_END
  511 };
  512 
  513 static struct mrs_field_value id_aa64isar0_rdm[] = {
  514         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
  515         MRS_FIELD_VALUE_END,
  516 };
  517 
  518 static struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
  519         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
  520         MRS_HWCAP_END
  521 };
  522 
  523 static struct mrs_field_value id_aa64isar0_atomic[] = {
  524         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
  525         MRS_FIELD_VALUE_END,
  526 };
  527 
  528 static struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
  529         MRS_HWCAP(&elf_hwcap, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
  530         MRS_HWCAP_END
  531 };
  532 
  533 static struct mrs_field_value id_aa64isar0_crc32[] = {
  534         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
  535         MRS_FIELD_VALUE_END,
  536 };
  537 
  538 static struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
  539         MRS_HWCAP(&elf_hwcap, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
  540         MRS_HWCAP_END
  541 };
  542 
  543 static struct mrs_field_value id_aa64isar0_sha2[] = {
  544         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
  545         MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
  546         MRS_FIELD_VALUE_END,
  547 };
  548 
  549 static struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
  550         MRS_HWCAP(&elf_hwcap, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
  551         MRS_HWCAP(&elf_hwcap, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
  552         MRS_HWCAP_END
  553 };
  554 
  555 static struct mrs_field_value id_aa64isar0_sha1[] = {
  556         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
  557         MRS_FIELD_VALUE_END,
  558 };
  559 
  560 static struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
  561         MRS_HWCAP(&elf_hwcap, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
  562         MRS_HWCAP_END
  563 };
  564 
  565 static struct mrs_field_value id_aa64isar0_aes[] = {
  566         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
  567         MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
  568         MRS_FIELD_VALUE_END,
  569 };
  570 
  571 static struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
  572         MRS_HWCAP(&elf_hwcap, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
  573         MRS_HWCAP(&elf_hwcap, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
  574         MRS_HWCAP_END
  575 };
  576 
  577 static struct mrs_field id_aa64isar0_fields[] = {
  578         MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER,
  579             id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
  580         MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_EXACT, id_aa64isar0_tlb),
  581         MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, id_aa64isar0_ts,
  582             id_aa64isar0_ts_caps),
  583         MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, id_aa64isar0_fhm,
  584             id_aa64isar0_fhm_caps),
  585         MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp,
  586             id_aa64isar0_dp_caps),
  587         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4,
  588             id_aa64isar0_sm4_caps),
  589         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3,
  590             id_aa64isar0_sm3_caps),
  591         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3,
  592             id_aa64isar0_sha3_caps),
  593         MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm,
  594             id_aa64isar0_rdm_caps),
  595         MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER,
  596             id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
  597         MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER,
  598             id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
  599         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2,
  600             id_aa64isar0_sha2_caps),
  601         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER,
  602             id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
  603         MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes,
  604             id_aa64isar0_aes_caps),
  605         MRS_FIELD_END,
  606 };
  607 
  608 
  609 /* ID_AA64ISAR1_EL1 */
  610 static struct mrs_field_value id_aa64isar1_i8mm[] = {
  611         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
  612         MRS_FIELD_VALUE_END,
  613 };
  614 
  615 static struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
  616         MRS_HWCAP(&elf_hwcap2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
  617         MRS_HWCAP_END
  618 };
  619 
  620 static struct mrs_field_value id_aa64isar1_dgh[] = {
  621         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
  622         MRS_FIELD_VALUE_END,
  623 };
  624 
  625 static struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
  626         MRS_HWCAP(&elf_hwcap2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
  627         MRS_HWCAP_END
  628 };
  629 
  630 static struct mrs_field_value id_aa64isar1_bf16[] = {
  631         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
  632         MRS_FIELD_VALUE_END,
  633 };
  634 
  635 static struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
  636         MRS_HWCAP(&elf_hwcap2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
  637         MRS_HWCAP_END
  638 };
  639 
  640 static struct mrs_field_value id_aa64isar1_specres[] = {
  641         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
  642         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_IMPL, "PredInv"),
  643         MRS_FIELD_VALUE_END,
  644 };
  645 
  646 static struct mrs_field_value id_aa64isar1_sb[] = {
  647         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
  648         MRS_FIELD_VALUE_END,
  649 };
  650 
  651 static struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
  652         MRS_HWCAP(&elf_hwcap, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
  653         MRS_HWCAP_END
  654 };
  655 
  656 static struct mrs_field_value id_aa64isar1_frintts[] = {
  657         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
  658         MRS_FIELD_VALUE_END,
  659 };
  660 
  661 static struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
  662         MRS_HWCAP(&elf_hwcap2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
  663         MRS_HWCAP_END
  664 };
  665 
  666 static struct mrs_field_value id_aa64isar1_gpi[] = {
  667         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
  668         MRS_FIELD_VALUE_END,
  669 };
  670 
  671 static struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = {
  672         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL),
  673         MRS_HWCAP_END
  674 };
  675 
  676 static struct mrs_field_value id_aa64isar1_gpa[] = {
  677         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
  678         MRS_FIELD_VALUE_END,
  679 };
  680 
  681 static struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = {
  682         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL),
  683         MRS_HWCAP_END
  684 };
  685 
  686 static struct mrs_field_value id_aa64isar1_lrcpc[] = {
  687         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
  688         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
  689         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
  690         MRS_FIELD_VALUE_END,
  691 };
  692 
  693 static struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
  694         MRS_HWCAP(&elf_hwcap, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
  695         MRS_HWCAP(&elf_hwcap, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
  696         MRS_HWCAP_END
  697 };
  698 
  699 static struct mrs_field_value id_aa64isar1_fcma[] = {
  700         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
  701         MRS_FIELD_VALUE_END,
  702 };
  703 
  704 static struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
  705         MRS_HWCAP(&elf_hwcap, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
  706         MRS_HWCAP_END
  707 };
  708 
  709 static struct mrs_field_value id_aa64isar1_jscvt[] = {
  710         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
  711         MRS_FIELD_VALUE_END,
  712 };
  713 
  714 static struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
  715         MRS_HWCAP(&elf_hwcap, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
  716         MRS_HWCAP_END
  717 };
  718 
  719 static struct mrs_field_value id_aa64isar1_api[] = {
  720         MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
  721         MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
  722         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
  723         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"),
  724         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"),
  725         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED,
  726             "Impl PAuth+FPAC+Combined"),
  727         MRS_FIELD_VALUE_END,
  728 };
  729 
  730 static struct mrs_field_hwcap id_aa64isar1_api_caps[] = {
  731         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR1_API_PAC),
  732         MRS_HWCAP_END
  733 };
  734 
  735 static struct mrs_field_value id_aa64isar1_apa[] = {
  736         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
  737         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
  738         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
  739         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"),
  740         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"),
  741         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED,
  742             "APA FPAC+Combined"),
  743         MRS_FIELD_VALUE_END,
  744 };
  745 
  746 static struct mrs_field_hwcap id_aa64isar1_apa_caps[] = {
  747         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR1_APA_PAC),
  748         MRS_HWCAP_END
  749 };
  750 
  751 static struct mrs_field_value id_aa64isar1_dpb[] = {
  752         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
  753         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
  754         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
  755         MRS_FIELD_VALUE_END,
  756 };
  757 
  758 static struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
  759         MRS_HWCAP(&elf_hwcap, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
  760         MRS_HWCAP(&elf_hwcap2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
  761         MRS_HWCAP_END
  762 };
  763 
  764 static struct mrs_field id_aa64isar1_fields[] = {
  765         MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER,
  766             id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
  767         MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, id_aa64isar1_dgh,
  768             id_aa64isar1_dgh_caps),
  769         MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER,
  770             id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
  771         MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_EXACT,
  772             id_aa64isar1_specres),
  773         MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, id_aa64isar1_sb,
  774             id_aa64isar1_sb_caps),
  775         MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER,
  776             id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
  777         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi,
  778             id_aa64isar1_gpi_caps),
  779         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa,
  780             id_aa64isar1_gpa_caps),
  781         MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER,
  782             id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
  783         MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER,
  784             id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
  785         MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER,
  786             id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
  787         MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api,
  788             id_aa64isar1_api_caps),
  789         MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa,
  790             id_aa64isar1_apa_caps),
  791         MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb,
  792             id_aa64isar1_dpb_caps),
  793         MRS_FIELD_END,
  794 };
  795 
  796 
  797 /* ID_AA64ISAR2_EL1 */
  798 static struct mrs_field_value id_aa64isar2_pac_frac[] = {
  799         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL),
  800         MRS_FIELD_VALUE_END,
  801 };
  802 
  803 static struct mrs_field_value id_aa64isar2_bc[] = {
  804         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL),
  805         MRS_FIELD_VALUE_END,
  806 };
  807 
  808 static struct mrs_field_value id_aa64isar2_mops[] = {
  809         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL),
  810         MRS_FIELD_VALUE_END,
  811 };
  812 
  813 static struct mrs_field_value id_aa64isar2_apa3[] = {
  814         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""),
  815         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"),
  816         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"),
  817         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"),
  818         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"),
  819         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED,
  820             "APA3 FPAC+Combined"),
  821         MRS_FIELD_VALUE_END,
  822 };
  823 
  824 static struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = {
  825         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC),
  826         MRS_HWCAP_END
  827 };
  828 
  829 static struct mrs_field_value id_aa64isar2_gpa3[] = {
  830         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL),
  831         MRS_FIELD_VALUE_END,
  832 };
  833 
  834 static struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = {
  835         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL),
  836         MRS_HWCAP_END
  837 };
  838 
  839 static struct mrs_field_value id_aa64isar2_rpres[] = {
  840         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL),
  841         MRS_FIELD_VALUE_END,
  842 };
  843 
  844 static struct mrs_field_value id_aa64isar2_wfxt[] = {
  845         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL),
  846         MRS_FIELD_VALUE_END,
  847 };
  848 
  849 static struct mrs_field id_aa64isar2_fields[] = {
  850         MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_EXACT,
  851             id_aa64isar2_pac_frac),
  852         MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_EXACT, id_aa64isar2_bc),
  853         MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_EXACT, id_aa64isar2_mops),
  854         MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_EXACT,
  855             id_aa64isar2_apa3, id_aa64isar2_apa3_caps),
  856         MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_EXACT,
  857             id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps),
  858         MRS_FIELD(ID_AA64ISAR2, RPRES, false, MRS_EXACT, id_aa64isar2_rpres),
  859         MRS_FIELD(ID_AA64ISAR2, WFxT, false, MRS_EXACT, id_aa64isar2_wfxt),
  860         MRS_FIELD_END,
  861 };
  862 
  863 
  864 /* ID_AA64MMFR0_EL1 */
  865 static struct mrs_field_value id_aa64mmfr0_exs[] = {
  866         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
  867         MRS_FIELD_VALUE_END,
  868 };
  869 
  870 static struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
  871         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
  872         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
  873         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
  874         MRS_FIELD_VALUE_END,
  875 };
  876 
  877 static struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
  878         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
  879         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
  880         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
  881         MRS_FIELD_VALUE_END,
  882 };
  883 
  884 static struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
  885         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
  886         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
  887         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
  888         MRS_FIELD_VALUE_END,
  889 };
  890 
  891 static struct mrs_field_value id_aa64mmfr0_tgran4[] = {
  892         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4,NONE, IMPL),
  893         MRS_FIELD_VALUE_END,
  894 };
  895 
  896 static struct mrs_field_value id_aa64mmfr0_tgran64[] = {
  897         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
  898         MRS_FIELD_VALUE_END,
  899 };
  900 
  901 static struct mrs_field_value id_aa64mmfr0_tgran16[] = {
  902         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
  903         MRS_FIELD_VALUE_END,
  904 };
  905 
  906 static struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
  907         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
  908         MRS_FIELD_VALUE_END,
  909 };
  910 
  911 static struct mrs_field_value id_aa64mmfr0_snsmem[] = {
  912         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
  913         MRS_FIELD_VALUE_END,
  914 };
  915 
  916 static struct mrs_field_value id_aa64mmfr0_bigend[] = {
  917         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
  918         MRS_FIELD_VALUE_END,
  919 };
  920 
  921 static struct mrs_field_value id_aa64mmfr0_asidbits[] = {
  922         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
  923         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
  924         MRS_FIELD_VALUE_END,
  925 };
  926 
  927 static struct mrs_field_value id_aa64mmfr0_parange[] = {
  928         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
  929         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
  930         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
  931         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
  932         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
  933         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
  934         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
  935         MRS_FIELD_VALUE_END,
  936 };
  937 
  938 static struct mrs_field id_aa64mmfr0_fields[] = {
  939         MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_EXACT, id_aa64mmfr0_exs),
  940         MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_EXACT,
  941             id_aa64mmfr0_tgran4_2),
  942         MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_EXACT,
  943             id_aa64mmfr0_tgran64_2),
  944         MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_EXACT,
  945             id_aa64mmfr0_tgran16_2),
  946         MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4),
  947         MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT,
  948             id_aa64mmfr0_tgran64),
  949         MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT,
  950             id_aa64mmfr0_tgran16),
  951         MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT,
  952             id_aa64mmfr0_bigendel0),
  953         MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem),
  954         MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend),
  955         MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT,
  956             id_aa64mmfr0_asidbits),
  957         MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT,
  958             id_aa64mmfr0_parange),
  959         MRS_FIELD_END,
  960 };
  961 
  962 
  963 /* ID_AA64MMFR1_EL1 */
  964 static struct mrs_field_value id_aa64mmfr1_xnx[] = {
  965         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
  966         MRS_FIELD_VALUE_END,
  967 };
  968 
  969 static struct mrs_field_value id_aa64mmfr1_specsei[] = {
  970         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
  971         MRS_FIELD_VALUE_END,
  972 };
  973 
  974 static struct mrs_field_value id_aa64mmfr1_pan[] = {
  975         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
  976         MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
  977         MRS_FIELD_VALUE_END,
  978 };
  979 
  980 static struct mrs_field_value id_aa64mmfr1_lo[] = {
  981         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
  982         MRS_FIELD_VALUE_END,
  983 };
  984 
  985 static struct mrs_field_value id_aa64mmfr1_hpds[] = {
  986         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
  987         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
  988         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
  989         MRS_FIELD_VALUE_END,
  990 };
  991 
  992 static struct mrs_field_value id_aa64mmfr1_vh[] = {
  993         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
  994         MRS_FIELD_VALUE_END,
  995 };
  996 
  997 static struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
  998         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
  999         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
 1000         MRS_FIELD_VALUE_END,
 1001 };
 1002 
 1003 static struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
 1004         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
 1005         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
 1006         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
 1007         MRS_FIELD_VALUE_END,
 1008 };
 1009 
 1010 static struct mrs_field id_aa64mmfr1_fields[] = {
 1011         MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx),
 1012         MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT,
 1013             id_aa64mmfr1_specsei),
 1014         MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan),
 1015         MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo),
 1016         MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds),
 1017         MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh),
 1018         MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT,
 1019             id_aa64mmfr1_vmidbits),
 1020         MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs),
 1021         MRS_FIELD_END,
 1022 };
 1023 
 1024 
 1025 /* ID_AA64MMFR2_EL1 */
 1026 static struct mrs_field_value id_aa64mmfr2_e0pd[] = {
 1027         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
 1028         MRS_FIELD_VALUE_END,
 1029 };
 1030 
 1031 static struct mrs_field_value id_aa64mmfr2_evt[] = {
 1032         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
 1033         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
 1034         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
 1035         MRS_FIELD_VALUE_END,
 1036 };
 1037 
 1038 static struct mrs_field_value id_aa64mmfr2_bbm[] = {
 1039         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
 1040         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
 1041         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
 1042         MRS_FIELD_VALUE_END,
 1043 };
 1044 
 1045 static struct mrs_field_value id_aa64mmfr2_ttl[] = {
 1046         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
 1047         MRS_FIELD_VALUE_END,
 1048 };
 1049 
 1050 static struct mrs_field_value id_aa64mmfr2_fwb[] = {
 1051         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
 1052         MRS_FIELD_VALUE_END,
 1053 };
 1054 
 1055 static struct mrs_field_value id_aa64mmfr2_ids[] = {
 1056         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
 1057         MRS_FIELD_VALUE_END,
 1058 };
 1059 
 1060 static struct mrs_field_value id_aa64mmfr2_at[] = {
 1061         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
 1062         MRS_FIELD_VALUE_END,
 1063 };
 1064 
 1065 static struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
 1066         MRS_HWCAP(&elf_hwcap, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
 1067         MRS_HWCAP_END
 1068 };
 1069 
 1070 static struct mrs_field_value id_aa64mmfr2_st[] = {
 1071         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
 1072         MRS_FIELD_VALUE_END,
 1073 };
 1074 
 1075 static struct mrs_field_value id_aa64mmfr2_nv[] = {
 1076         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
 1077         MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
 1078         MRS_FIELD_VALUE_END,
 1079 };
 1080 
 1081 static struct mrs_field_value id_aa64mmfr2_ccidx[] = {
 1082         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
 1083         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
 1084         MRS_FIELD_VALUE_END,
 1085 };
 1086 
 1087 static struct mrs_field_value id_aa64mmfr2_varange[] = {
 1088         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
 1089         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
 1090         MRS_FIELD_VALUE_END,
 1091 };
 1092 
 1093 static struct mrs_field_value id_aa64mmfr2_iesb[] = {
 1094         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
 1095         MRS_FIELD_VALUE_END,
 1096 };
 1097 
 1098 static struct mrs_field_value id_aa64mmfr2_lsm[] = {
 1099         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
 1100         MRS_FIELD_VALUE_END,
 1101 };
 1102 
 1103 static struct mrs_field_value id_aa64mmfr2_uao[] = {
 1104         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
 1105         MRS_FIELD_VALUE_END,
 1106 };
 1107 
 1108 static struct mrs_field_value id_aa64mmfr2_cnp[] = {
 1109         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
 1110         MRS_FIELD_VALUE_END,
 1111 };
 1112 
 1113 static struct mrs_field id_aa64mmfr2_fields[] = {
 1114         MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_EXACT, id_aa64mmfr2_e0pd),
 1115         MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_EXACT, id_aa64mmfr2_evt),
 1116         MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_EXACT, id_aa64mmfr2_bbm),
 1117         MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_EXACT, id_aa64mmfr2_ttl),
 1118         MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_EXACT, id_aa64mmfr2_fwb),
 1119         MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_EXACT, id_aa64mmfr2_ids),
 1120         MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, id_aa64mmfr2_at,
 1121             id_aa64mmfr2_at_caps),
 1122         MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_EXACT, id_aa64mmfr2_st),
 1123         MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv),
 1124         MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx),
 1125         MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT,
 1126             id_aa64mmfr2_varange),
 1127         MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb),
 1128         MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm),
 1129         MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao),
 1130         MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp),
 1131         MRS_FIELD_END,
 1132 };
 1133 
 1134 
 1135 /* ID_AA64PFR0_EL1 */
 1136 static struct mrs_field_value id_aa64pfr0_csv3[] = {
 1137         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
 1138         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
 1139         MRS_FIELD_VALUE_END,
 1140 };
 1141 
 1142 static struct mrs_field_value id_aa64pfr0_csv2[] = {
 1143         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
 1144         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
 1145         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "SCXTNUM"),
 1146         MRS_FIELD_VALUE_END,
 1147 };
 1148 
 1149 static struct mrs_field_value id_aa64pfr0_dit[] = {
 1150         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
 1151         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
 1152         MRS_FIELD_VALUE_END,
 1153 };
 1154 
 1155 static struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
 1156         MRS_HWCAP(&elf_hwcap, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
 1157         MRS_HWCAP_END
 1158 };
 1159 
 1160 static struct mrs_field_value id_aa64pfr0_amu[] = {
 1161         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
 1162         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
 1163         MRS_FIELD_VALUE_END,
 1164 };
 1165 
 1166 static struct mrs_field_value id_aa64pfr0_mpam[] = {
 1167         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
 1168         MRS_FIELD_VALUE_END,
 1169 };
 1170 
 1171 static struct mrs_field_value id_aa64pfr0_sel2[] = {
 1172         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
 1173         MRS_FIELD_VALUE_END,
 1174 };
 1175 
 1176 static struct mrs_field_value id_aa64pfr0_sve[] = {
 1177         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
 1178         MRS_FIELD_VALUE_END,
 1179 };
 1180 
 1181 #if 0
 1182 /* Enable when we add SVE support */
 1183 static struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
 1184         MRS_HWCAP(&elf_hwcap, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
 1185         MRS_HWCAP_END
 1186 };
 1187 #endif
 1188 
 1189 static struct mrs_field_value id_aa64pfr0_ras[] = {
 1190         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
 1191         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
 1192         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
 1193         MRS_FIELD_VALUE_END,
 1194 };
 1195 
 1196 static struct mrs_field_value id_aa64pfr0_gic[] = {
 1197         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
 1198         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""),
 1199         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"),
 1200         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"),
 1201         MRS_FIELD_VALUE_END,
 1202 };
 1203 
 1204 static struct mrs_field_value id_aa64pfr0_advsimd[] = {
 1205         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
 1206         MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
 1207         MRS_FIELD_VALUE_END,
 1208 };
 1209 
 1210 static struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
 1211         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
 1212         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
 1213         MRS_HWCAP_END
 1214 };
 1215 
 1216 static struct mrs_field_value id_aa64pfr0_fp[] = {
 1217         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
 1218         MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
 1219         MRS_FIELD_VALUE_END,
 1220 };
 1221 
 1222 static struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
 1223         MRS_HWCAP(&elf_hwcap, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
 1224         MRS_HWCAP(&elf_hwcap, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
 1225         MRS_HWCAP_END
 1226 };
 1227 
 1228 static struct mrs_field_value id_aa64pfr0_el3[] = {
 1229         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
 1230         MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
 1231         MRS_FIELD_VALUE_END,
 1232 };
 1233 
 1234 static struct mrs_field_value id_aa64pfr0_el2[] = {
 1235         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
 1236         MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
 1237         MRS_FIELD_VALUE_END,
 1238 };
 1239 
 1240 static struct mrs_field_value id_aa64pfr0_el1[] = {
 1241         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
 1242         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
 1243         MRS_FIELD_VALUE_END,
 1244 };
 1245 
 1246 static struct mrs_field_value id_aa64pfr0_el0[] = {
 1247         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
 1248         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
 1249         MRS_FIELD_VALUE_END,
 1250 };
 1251 
 1252 static struct mrs_field id_aa64pfr0_fields[] = {
 1253         MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3),
 1254         MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2),
 1255         MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, id_aa64pfr0_dit,
 1256             id_aa64pfr0_dit_caps),
 1257         MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu),
 1258         MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam),
 1259         MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2),
 1260         MRS_FIELD(ID_AA64PFR0, SVE, false, MRS_EXACT, id_aa64pfr0_sve),
 1261         MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras),
 1262         MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic),
 1263         MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER,
 1264             id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
 1265         MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true,  MRS_LOWER, id_aa64pfr0_fp,
 1266             id_aa64pfr0_fp_caps),
 1267         MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3),
 1268         MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2),
 1269         MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1),
 1270         MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0),
 1271         MRS_FIELD_END,
 1272 };
 1273 
 1274 
 1275 /* ID_AA64PFR1_EL1 */
 1276 static struct mrs_field_value id_aa64pfr1_mte[] = {
 1277         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
 1278         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL_EL0, "MTE EL0"),
 1279         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL, "MTE"),
 1280         MRS_FIELD_VALUE_END,
 1281 };
 1282 
 1283 static struct mrs_field_value id_aa64pfr1_ssbs[] = {
 1284         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
 1285         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
 1286         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
 1287         MRS_FIELD_VALUE_END,
 1288 };
 1289 
 1290 static struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
 1291         MRS_HWCAP(&elf_hwcap, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
 1292         MRS_HWCAP_END
 1293 };
 1294 
 1295 static struct mrs_field_value id_aa64pfr1_bt[] = {
 1296         MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
 1297         MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
 1298         MRS_FIELD_VALUE_END,
 1299 };
 1300 
 1301 #if 0
 1302 /* Enable when we add BTI support */
 1303 static struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
 1304         MRS_HWCAP(&elf_hwcap2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
 1305         MRS_HWCAP_END
 1306 };
 1307 #endif
 1308 
 1309 static struct mrs_field id_aa64pfr1_fields[] = {
 1310         MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte),
 1311         MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, id_aa64pfr1_ssbs,
 1312             id_aa64pfr1_ssbs_caps),
 1313         MRS_FIELD(ID_AA64PFR1, BT, false, MRS_EXACT, id_aa64pfr1_bt),
 1314         MRS_FIELD_END,
 1315 };
 1316 
 1317 
 1318 /* ID_AA64ZFR0_EL1 */
 1319 static struct mrs_field_value id_aa64zfr0_f64mm[] = {
 1320         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL),
 1321         MRS_FIELD_VALUE_END,
 1322 };
 1323 
 1324 static struct mrs_field_value id_aa64zfr0_f32mm[] = {
 1325         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL),
 1326         MRS_FIELD_VALUE_END,
 1327 };
 1328 
 1329 static struct mrs_field_value id_aa64zfr0_i8mm[] = {
 1330         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL),
 1331         MRS_FIELD_VALUE_END,
 1332 };
 1333 
 1334 static struct mrs_field_value id_aa64zfr0_sm4[] = {
 1335         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL),
 1336         MRS_FIELD_VALUE_END,
 1337 };
 1338 
 1339 static struct mrs_field_value id_aa64zfr0_sha3[] = {
 1340         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL),
 1341         MRS_FIELD_VALUE_END,
 1342 };
 1343 
 1344 static struct mrs_field_value id_aa64zfr0_bf16[] = {
 1345         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE),
 1346         MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"),
 1347         MRS_FIELD_VALUE_END,
 1348 };
 1349 
 1350 static struct mrs_field_value id_aa64zfr0_bitperm[] = {
 1351         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL),
 1352         MRS_FIELD_VALUE_END,
 1353 };
 1354 
 1355 static struct mrs_field_value id_aa64zfr0_aes[] = {
 1356         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE),
 1357         MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"),
 1358         MRS_FIELD_VALUE_END,
 1359 };
 1360 
 1361 static struct mrs_field_value id_aa64zfr0_svever[] = {
 1362         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"),
 1363         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"),
 1364         MRS_FIELD_VALUE_END,
 1365 };
 1366 
 1367 static struct mrs_field id_aa64zfr0_fields[] = {
 1368         MRS_FIELD(ID_AA64ZFR0, F64MM, false, MRS_EXACT, id_aa64zfr0_f64mm),
 1369         MRS_FIELD(ID_AA64ZFR0, F32MM, false, MRS_EXACT, id_aa64zfr0_f32mm),
 1370         MRS_FIELD(ID_AA64ZFR0, I8MM, false, MRS_EXACT, id_aa64zfr0_i8mm),
 1371         MRS_FIELD(ID_AA64ZFR0, SM4, false, MRS_EXACT, id_aa64zfr0_sm4),
 1372         MRS_FIELD(ID_AA64ZFR0, SHA3, false, MRS_EXACT, id_aa64zfr0_sha3),
 1373         MRS_FIELD(ID_AA64ZFR0, BF16, false, MRS_EXACT, id_aa64zfr0_bf16),
 1374         MRS_FIELD(ID_AA64ZFR0, BitPerm, false, MRS_EXACT, id_aa64zfr0_bitperm),
 1375         MRS_FIELD(ID_AA64ZFR0, AES, false, MRS_EXACT, id_aa64zfr0_aes),
 1376         MRS_FIELD(ID_AA64ZFR0, SVEver, false, MRS_EXACT, id_aa64zfr0_svever),
 1377         MRS_FIELD_END,
 1378 };
 1379 
 1380 
 1381 #ifdef COMPAT_FREEBSD32
 1382 /* ID_ISAR5_EL1 */
 1383 static struct mrs_field_value id_isar5_vcma[] = {
 1384         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
 1385         MRS_FIELD_VALUE_END,
 1386 };
 1387 
 1388 static struct mrs_field_value id_isar5_rdm[] = {
 1389         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
 1390         MRS_FIELD_VALUE_END,
 1391 };
 1392 
 1393 static struct mrs_field_value id_isar5_crc32[] = {
 1394         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
 1395         MRS_FIELD_VALUE_END,
 1396 };
 1397 
 1398 static struct mrs_field_hwcap id_isar5_crc32_caps[] = {
 1399         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
 1400         MRS_HWCAP_END
 1401 };
 1402 
 1403 static struct mrs_field_value id_isar5_sha2[] = {
 1404         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
 1405         MRS_FIELD_VALUE_END,
 1406 };
 1407 
 1408 static struct mrs_field_hwcap id_isar5_sha2_caps[] = {
 1409         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
 1410         MRS_HWCAP_END
 1411 };
 1412 
 1413 static struct mrs_field_value id_isar5_sha1[] = {
 1414         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
 1415         MRS_FIELD_VALUE_END,
 1416 };
 1417 
 1418 static struct mrs_field_hwcap id_isar5_sha1_caps[] = {
 1419         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
 1420         MRS_HWCAP_END
 1421 };
 1422 
 1423 static struct mrs_field_value id_isar5_aes[] = {
 1424         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
 1425         MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
 1426         MRS_FIELD_VALUE_END,
 1427 };
 1428 
 1429 static struct mrs_field_hwcap id_isar5_aes_caps[] = {
 1430         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
 1431         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
 1432         MRS_HWCAP_END
 1433 };
 1434 
 1435 static struct mrs_field_value id_isar5_sevl[] = {
 1436         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
 1437         MRS_FIELD_VALUE_END,
 1438 };
 1439 
 1440 static struct mrs_field id_isar5_fields[] = {
 1441         MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER, id_isar5_vcma),
 1442         MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, id_isar5_rdm),
 1443         MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, id_isar5_crc32,
 1444             id_isar5_crc32_caps),
 1445         MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, id_isar5_sha2,
 1446             id_isar5_sha2_caps),
 1447         MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, id_isar5_sha1,
 1448             id_isar5_sha1_caps),
 1449         MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, id_isar5_aes,
 1450             id_isar5_aes_caps),
 1451         MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, id_isar5_sevl),
 1452         MRS_FIELD_END,
 1453 };
 1454 
 1455 /* MVFR0 */
 1456 static struct mrs_field_value mvfr0_fpround[] = {
 1457         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
 1458         MRS_FIELD_VALUE_END,
 1459 };
 1460 
 1461 static struct mrs_field_value mvfr0_fpsqrt[] = {
 1462         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
 1463         MRS_FIELD_VALUE_END,
 1464 };
 1465 
 1466 static struct mrs_field_value mvfr0_fpdivide[] = {
 1467         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
 1468         MRS_FIELD_VALUE_END,
 1469 };
 1470 
 1471 static struct mrs_field_value mvfr0_fptrap[] = {
 1472         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
 1473         MRS_FIELD_VALUE_END,
 1474 };
 1475 
 1476 static struct mrs_field_value mvfr0_fpdp[] = {
 1477         MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
 1478         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
 1479         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
 1480         MRS_FIELD_VALUE_END,
 1481 };
 1482 
 1483 static struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
 1484         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
 1485         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
 1486 };
 1487 
 1488 static struct mrs_field_value mvfr0_fpsp[] = {
 1489         MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
 1490         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
 1491         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
 1492         MRS_FIELD_VALUE_END,
 1493 };
 1494 
 1495 static struct mrs_field_value mvfr0_simdreg[] = {
 1496         MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
 1497         MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
 1498         MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
 1499         MRS_FIELD_VALUE_END,
 1500 };
 1501 
 1502 static struct mrs_field mvfr0_fields[] = {
 1503         MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, mvfr0_fpround),
 1504         MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, mvfr0_fpsqrt),
 1505         MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, mvfr0_fpdivide),
 1506         MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, mvfr0_fptrap),
 1507         MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, mvfr0_fpdp,
 1508             mvfr0_fpdp_caps),
 1509         MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, mvfr0_fpsp),
 1510         MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, mvfr0_simdreg),
 1511         MRS_FIELD_END,
 1512 };
 1513 
 1514 /* MVFR1 */
 1515 static struct mrs_field_value mvfr1_simdfmac[] = {
 1516         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
 1517         MRS_FIELD_VALUE_END,
 1518 };
 1519 
 1520 static struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
 1521         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
 1522         MRS_HWCAP_END
 1523 };
 1524 
 1525 static struct mrs_field_value mvfr1_fphp[] = {
 1526         MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
 1527         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
 1528         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
 1529         MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
 1530         MRS_FIELD_VALUE_END,
 1531 };
 1532 
 1533 static struct mrs_field_value mvfr1_simdhp[] = {
 1534         MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
 1535         MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
 1536         MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
 1537         MRS_FIELD_VALUE_END,
 1538 };
 1539 
 1540 static struct mrs_field_value mvfr1_simdsp[] = {
 1541         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
 1542         MRS_FIELD_VALUE_END,
 1543 };
 1544 
 1545 static struct mrs_field_value mvfr1_simdint[] = {
 1546         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
 1547         MRS_FIELD_VALUE_END,
 1548 };
 1549 
 1550 static struct mrs_field_value mvfr1_simdls[] = {
 1551         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
 1552         MRS_FIELD_VALUE_END,
 1553 };
 1554 
 1555 static struct mrs_field_hwcap mvfr1_simdls_caps[] = {
 1556         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
 1557         MRS_HWCAP_END
 1558 };
 1559 
 1560 static struct mrs_field_value mvfr1_fpdnan[] = {
 1561         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
 1562         MRS_FIELD_VALUE_END,
 1563 };
 1564 
 1565 static struct mrs_field_value mvfr1_fpftz[] = {
 1566         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
 1567         MRS_FIELD_VALUE_END,
 1568 };
 1569 
 1570 static struct mrs_field mvfr1_fields[] = {
 1571         MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, mvfr1_simdfmac,
 1572             mvfr1_simdfmac_caps),
 1573         MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, mvfr1_fphp),
 1574         MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, mvfr1_simdhp),
 1575         MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, mvfr1_simdsp),
 1576         MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, mvfr1_simdint),
 1577         MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, mvfr1_simdls,
 1578             mvfr1_simdls_caps),
 1579         MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, mvfr1_fpdnan),
 1580         MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, mvfr1_fpftz),
 1581         MRS_FIELD_END,
 1582 };
 1583 #endif /* COMPAT_FREEBSD32 */
 1584 
 1585 struct mrs_user_reg {
 1586         u_int           reg;
 1587         u_int           CRm;
 1588         u_int           Op2;
 1589         size_t          offset;
 1590         struct mrs_field *fields;
 1591 };
 1592 
 1593 #define USER_REG(name, field_name)                                      \
 1594         {                                                               \
 1595                 .reg = name,                                            \
 1596                 .CRm = name##_CRm,                                      \
 1597                 .Op2 = name##_op2,                                      \
 1598                 .offset = __offsetof(struct cpu_desc, field_name),      \
 1599                 .fields = field_name##_fields,                          \
 1600         }
 1601 static struct mrs_user_reg user_regs[] = {
 1602         USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0),
 1603 
 1604         USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0),
 1605         USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1),
 1606 
 1607         USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0),
 1608         USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1),
 1609         USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2),
 1610 
 1611         USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0),
 1612         USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1),
 1613 #ifdef COMPAT_FREEBSD32
 1614         USER_REG(ID_ISAR5_EL1, id_isar5),
 1615 
 1616         USER_REG(MVFR0_EL1, mvfr0),
 1617         USER_REG(MVFR1_EL1, mvfr1),
 1618 #endif /* COMPAT_FREEBSD32 */
 1619 };
 1620 
 1621 #define CPU_DESC_FIELD(desc, idx)                                       \
 1622     *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
 1623 
 1624 static int
 1625 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
 1626     uint32_t esr)
 1627 {
 1628         uint64_t value;
 1629         int CRm, Op2, i, reg;
 1630 
 1631         if ((insn & MRS_MASK) != MRS_VALUE)
 1632                 return (0);
 1633 
 1634         /*
 1635          * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
 1636          * These are in the EL1 CPU identification space.
 1637          * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
 1638          * CRm == {4-7} holds the ID_AA64 registers.
 1639          *
 1640          * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
 1641          * Table D9-2 System instruction encodings for non-Debug System
 1642          * register accesses.
 1643          */
 1644         if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
 1645                 return (0);
 1646 
 1647         CRm = mrs_CRm(insn);
 1648         if (CRm > 7 || (CRm < 4 && CRm != 0))
 1649                 return (0);
 1650 
 1651         Op2 = mrs_Op2(insn);
 1652         value = 0;
 1653 
 1654         for (i = 0; i < nitems(user_regs); i++) {
 1655                 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
 1656                         value = CPU_DESC_FIELD(user_cpu_desc, i);
 1657                         break;
 1658                 }
 1659         }
 1660 
 1661         if (CRm == 0) {
 1662                 switch (Op2) {
 1663                 case 0:
 1664                         value = READ_SPECIALREG(midr_el1);
 1665                         break;
 1666                 case 5:
 1667                         value = READ_SPECIALREG(mpidr_el1);
 1668                         break;
 1669                 case 6:
 1670                         value = READ_SPECIALREG(revidr_el1);
 1671                         break;
 1672                 default:
 1673                         return (0);
 1674                 }
 1675         }
 1676 
 1677         /*
 1678          * We will handle this instruction, move to the next so we
 1679          * don't trap here again.
 1680          */
 1681         frame->tf_elr += INSN_SIZE;
 1682 
 1683         reg = MRS_REGISTER(insn);
 1684         /* If reg is 31 then write to xzr, i.e. do nothing */
 1685         if (reg == 31)
 1686                 return (1);
 1687 
 1688         if (reg < nitems(frame->tf_x))
 1689                 frame->tf_x[reg] = value;
 1690         else if (reg == 30)
 1691                 frame->tf_lr = value;
 1692 
 1693         return (1);
 1694 }
 1695 
 1696 bool
 1697 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
 1698 {
 1699         uint64_t value;
 1700         int i;
 1701 
 1702         for (i = 0; i < nitems(user_regs); i++) {
 1703                 if (user_regs[i].reg == reg) {
 1704                         value = CPU_DESC_FIELD(user_cpu_desc, i);
 1705                         *val = value >> field_shift;
 1706                         return (true);
 1707                 }
 1708         }
 1709 
 1710         return (false);
 1711 }
 1712 
 1713 bool
 1714 get_kernel_reg(u_int reg, uint64_t *val)
 1715 {
 1716         int i;
 1717 
 1718         for (i = 0; i < nitems(user_regs); i++) {
 1719                 if (user_regs[i].reg == reg) {
 1720                         *val = CPU_DESC_FIELD(kern_cpu_desc, i);
 1721                         return (true);
 1722                 }
 1723         }
 1724 
 1725         return (false);
 1726 }
 1727 
 1728 /*
 1729  * Compares two field values that may be signed or unsigned.
 1730  * Returns:
 1731  *  < 0 when a is less than b
 1732  *  = 0 when a equals b
 1733  *  > 0 when a is greater than b
 1734  */
 1735 static int
 1736 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
 1737 {
 1738         uint64_t mask;
 1739 
 1740         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
 1741             width));
 1742 
 1743         mask = (1ul << width) - 1;
 1744         /* Move the field to the lower bits */
 1745         a = (a >> shift) & mask;
 1746         b = (b >> shift) & mask;
 1747 
 1748         if (sign) {
 1749                 /*
 1750                  * The field is signed. Toggle the upper bit so the comparison
 1751                  * works on unsigned values as this makes positive numbers,
 1752                  * i.e. those with a 0 bit, larger than negative numbers,
 1753                  * i.e. those with a 1 bit, in an unsigned comparison.
 1754                  */
 1755                 a ^= 1ul << (width - 1);
 1756                 b ^= 1ul << (width - 1);
 1757         }
 1758 
 1759         return (a - b);
 1760 }
 1761 
 1762 static uint64_t
 1763 update_lower_register(uint64_t val, uint64_t new_val, u_int shift,
 1764     int width, bool sign)
 1765 {
 1766         uint64_t mask;
 1767 
 1768         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
 1769             width));
 1770 
 1771         /*
 1772          * If the new value is less than the existing value update it.
 1773          */
 1774         if (mrs_field_cmp(new_val, val, shift, width, sign) < 0) {
 1775                 mask = (1ul << width) - 1;
 1776                 val &= ~(mask << shift);
 1777                 val |= new_val & (mask << shift);
 1778         }
 1779 
 1780         return (val);
 1781 }
 1782 
 1783 void
 1784 update_special_regs(u_int cpu)
 1785 {
 1786         struct mrs_field *fields;
 1787         uint64_t user_reg, kern_reg, value;
 1788         int i, j;
 1789 
 1790         if (cpu == 0) {
 1791                 /* Create a user visible cpu description with safe values */
 1792                 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
 1793                 /* Safe values for these registers */
 1794                 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
 1795                     ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
 1796                     ID_AA64PFR0_EL0_64;
 1797                 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
 1798         }
 1799 
 1800         for (i = 0; i < nitems(user_regs); i++) {
 1801                 value = CPU_DESC_FIELD(cpu_desc[cpu], i);
 1802                 if (cpu == 0) {
 1803                         kern_reg = value;
 1804                         user_reg = value;
 1805                 } else {
 1806                         kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i);
 1807                         user_reg = CPU_DESC_FIELD(user_cpu_desc, i);
 1808                 }
 1809 
 1810                 fields = user_regs[i].fields;
 1811                 for (j = 0; fields[j].type != 0; j++) {
 1812                         switch (fields[j].type & MRS_TYPE_MASK) {
 1813                         case MRS_EXACT:
 1814                                 user_reg &= ~(0xful << fields[j].shift);
 1815                                 user_reg |=
 1816                                     (uint64_t)MRS_EXACT_FIELD(fields[j].type) <<
 1817                                     fields[j].shift;
 1818                                 break;
 1819                         case MRS_LOWER:
 1820                                 user_reg = update_lower_register(user_reg,
 1821                                     value, fields[j].shift, 4, fields[j].sign);
 1822                                 break;
 1823                         default:
 1824                                 panic("Invalid field type: %d", fields[j].type);
 1825                         }
 1826                         kern_reg = update_lower_register(kern_reg, value,
 1827                             fields[j].shift, 4, fields[j].sign);
 1828                 }
 1829 
 1830                 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg;
 1831                 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg;
 1832         }
 1833 }
 1834 
 1835 /* HWCAP */
 1836 bool __read_frequently lse_supported = false;
 1837 
 1838 bool __read_frequently icache_aliasing = false;
 1839 bool __read_frequently icache_vmid = false;
 1840 
 1841 int64_t dcache_line_size;       /* The minimum D cache line size */
 1842 int64_t icache_line_size;       /* The minimum I cache line size */
 1843 int64_t idcache_line_size;      /* The minimum cache line size */
 1844 
 1845 /*
 1846  * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
 1847  */
 1848 static void
 1849 parse_cpu_features(void)
 1850 {
 1851         struct mrs_field_hwcap *hwcaps;
 1852         struct mrs_field *fields;
 1853         uint64_t min, reg;
 1854         int i, j, k;
 1855 
 1856         for (i = 0; i < nitems(user_regs); i++) {
 1857                 reg = CPU_DESC_FIELD(user_cpu_desc, i);
 1858                 fields = user_regs[i].fields;
 1859                 for (j = 0; fields[j].type != 0; j++) {
 1860                         hwcaps = fields[j].hwcaps;
 1861                         if (hwcaps == NULL)
 1862                                 continue;
 1863 
 1864                         for (k = 0; hwcaps[k].hwcap != NULL; k++) {
 1865                                 min = hwcaps[k].min;
 1866 
 1867                                 /*
 1868                                  * If the field is greater than the minimum
 1869                                  * value we can set the hwcap;
 1870                                  */
 1871                                 if (mrs_field_cmp(reg, min, fields[j].shift,
 1872                                     4, fields[j].sign) >= 0) {
 1873                                         *hwcaps[k].hwcap |= hwcaps[k].hwcap_val;
 1874                                 }
 1875                         }
 1876                 }
 1877         }
 1878 }
 1879 
 1880 static void
 1881 identify_cpu_sysinit(void *dummy __unused)
 1882 {
 1883         int cpu;
 1884         bool dic, idc;
 1885 
 1886         dic = (allow_dic != 0);
 1887         idc = (allow_idc != 0);
 1888 
 1889         CPU_FOREACH(cpu) {
 1890                 check_cpu_regs(cpu);
 1891                 if (cpu != 0)
 1892                         update_special_regs(cpu);
 1893 
 1894                 if (CTR_DIC_VAL(cpu_desc[cpu].ctr) == 0)
 1895                         dic = false;
 1896                 if (CTR_IDC_VAL(cpu_desc[cpu].ctr) == 0)
 1897                         idc = false;
 1898         }
 1899 
 1900         /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
 1901         parse_cpu_features();
 1902 
 1903 #ifdef COMPAT_FREEBSD32
 1904         /* Set the default caps and any that need to check multiple fields */
 1905         elf32_hwcap |= parse_cpu_features_hwcap32();
 1906 #endif
 1907 
 1908         if (dic && idc) {
 1909                 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
 1910                 if (bootverbose)
 1911                         printf("Enabling DIC & IDC ICache sync\n");
 1912         } else if (idc) {
 1913                 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range;
 1914                 if (bootverbose)
 1915                         printf("Enabling IDC ICache sync\n");
 1916         }
 1917 
 1918         if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
 1919                 lse_supported = true;
 1920                 if (bootverbose)
 1921                         printf("Enabling LSE atomics in the kernel\n");
 1922         }
 1923 #ifdef LSE_ATOMICS
 1924         if (!lse_supported)
 1925                 panic("CPU does not support LSE atomic instructions");
 1926 #endif
 1927 
 1928         install_undef_handler(true, user_mrs_handler);
 1929 }
 1930 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
 1931 
 1932 static void
 1933 cpu_features_sysinit(void *dummy __unused)
 1934 {
 1935         struct sbuf sb;
 1936         u_int cpu;
 1937 
 1938         CPU_FOREACH(cpu)
 1939                 print_cpu_features(cpu);
 1940 
 1941         /* Fill in cpu_model for the hw.model sysctl */
 1942         sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
 1943         print_cpu_midr(&sb, 0);
 1944 
 1945         sbuf_finish(&sb);
 1946         sbuf_delete(&sb);
 1947 }
 1948 /* Log features before APs are released and start printing to the dmesg. */
 1949 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
 1950 
 1951 #ifdef COMPAT_FREEBSD32
 1952 static u_long
 1953 parse_cpu_features_hwcap32(void)
 1954 {
 1955         u_long hwcap = HWCAP32_DEFAULT;
 1956 
 1957         if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
 1958              MVFR1_SIMDLS_IMPL) &&
 1959             (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
 1960              MVFR1_SIMDInt_IMPL) &&
 1961             (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
 1962              MVFR1_SIMDSP_IMPL))
 1963                 hwcap |= HWCAP32_NEON;
 1964 
 1965         return (hwcap);
 1966 }
 1967 #endif /* COMPAT_FREEBSD32 */
 1968 
 1969 static void
 1970 print_ctr_fields(struct sbuf *sb, uint64_t reg, void *arg)
 1971 {
 1972 
 1973         sbuf_printf(sb, "%u byte D-cacheline,", CTR_DLINE_SIZE(reg));
 1974         sbuf_printf(sb, "%u byte I-cacheline,", CTR_ILINE_SIZE(reg));
 1975         reg &= ~(CTR_DLINE_MASK | CTR_ILINE_MASK);
 1976 
 1977         switch(CTR_L1IP_VAL(reg)) {
 1978         case CTR_L1IP_VPIPT:
 1979                 sbuf_printf(sb, "VPIPT");
 1980                 break;
 1981         case CTR_L1IP_AIVIVT:
 1982                 sbuf_printf(sb, "AIVIVT");
 1983                 break;
 1984         case CTR_L1IP_VIPT:
 1985                 sbuf_printf(sb, "VIPT");
 1986                 break;
 1987         case CTR_L1IP_PIPT:
 1988                 sbuf_printf(sb, "PIPT");
 1989                 break;
 1990         }
 1991         sbuf_printf(sb, " ICache,");
 1992         reg &= ~CTR_L1IP_MASK;
 1993 
 1994         sbuf_printf(sb, "%d byte ERG,", CTR_ERG_SIZE(reg));
 1995         sbuf_printf(sb, "%d byte CWG", CTR_CWG_SIZE(reg));
 1996         reg &= ~(CTR_ERG_MASK | CTR_CWG_MASK);
 1997 
 1998         if (CTR_IDC_VAL(reg) != 0)
 1999                 sbuf_printf(sb, ",IDC");
 2000         if (CTR_DIC_VAL(reg) != 0)
 2001                 sbuf_printf(sb, ",DIC");
 2002         reg &= ~(CTR_IDC_MASK | CTR_DIC_MASK);
 2003         reg &= ~CTR_RES1;
 2004 
 2005         if (reg != 0)
 2006                 sbuf_printf(sb, ",%lx", reg);
 2007 }
 2008 
 2009 static void
 2010 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
 2011     void (*print_fields)(struct sbuf *, uint64_t, void *), void *arg)
 2012 {
 2013 
 2014         sbuf_printf(sb, "%29s = <", reg_name);
 2015 
 2016         print_fields(sb, reg, arg);
 2017 
 2018         sbuf_finish(sb);
 2019         printf("%s>\n", sbuf_data(sb));
 2020         sbuf_clear(sb);
 2021 }
 2022 
 2023 static void
 2024 print_id_fields(struct sbuf *sb, uint64_t reg, void *arg)
 2025 {
 2026         struct mrs_field *fields = arg;
 2027         struct mrs_field_value *fv;
 2028         int field, i, j, printed;
 2029 
 2030 #define SEP_STR ((printed++) == 0) ? "" : ","
 2031         printed = 0;
 2032         for (i = 0; fields[i].type != 0; i++) {
 2033                 fv = fields[i].values;
 2034 
 2035                 /* TODO: Handle with an unknown message */
 2036                 if (fv == NULL)
 2037                         continue;
 2038 
 2039                 field = (reg & fields[i].mask) >> fields[i].shift;
 2040                 for (j = 0; fv[j].desc != NULL; j++) {
 2041                         if ((fv[j].value >> fields[i].shift) != field)
 2042                                 continue;
 2043 
 2044                         if (fv[j].desc[0] != '\0')
 2045                                 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
 2046                         break;
 2047                 }
 2048                 if (fv[j].desc == NULL)
 2049                         sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
 2050                             fields[i].name, field);
 2051 
 2052                 reg &= ~(0xful << fields[i].shift);
 2053         }
 2054 
 2055         if (reg != 0)
 2056                 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
 2057 #undef SEP_STR
 2058 }
 2059 
 2060 static void
 2061 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
 2062     struct mrs_field *fields)
 2063 {
 2064 
 2065         print_register(sb, reg_name, reg, print_id_fields, fields);
 2066 }
 2067 
 2068 static void
 2069 print_cpu_midr(struct sbuf *sb, u_int cpu)
 2070 {
 2071         const struct cpu_parts *cpu_partsp;
 2072         const char *cpu_impl_name;
 2073         const char *cpu_part_name;
 2074         u_int midr;
 2075         u_int impl_id;
 2076         u_int part_id;
 2077 
 2078         midr = pcpu_find(cpu)->pc_midr;
 2079 
 2080         cpu_impl_name = NULL;
 2081         cpu_partsp = NULL;
 2082         impl_id = CPU_IMPL(midr);
 2083         for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
 2084                 if (impl_id == cpu_implementers[i].impl_id) {
 2085                         cpu_impl_name = cpu_implementers[i].impl_name;
 2086                         cpu_partsp = cpu_implementers[i].cpu_parts;
 2087                         break;
 2088                 }
 2089         }
 2090         /* Unknown implementer, so unknown part */
 2091         if (cpu_impl_name == NULL) {
 2092                 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
 2093                 return;
 2094         }
 2095 
 2096         KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
 2097             __func__, cpu_impl_name));
 2098 
 2099         cpu_part_name = NULL;
 2100         part_id = CPU_PART(midr);
 2101         for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
 2102                 if (part_id == cpu_partsp[i].part_id) {
 2103                         cpu_part_name = cpu_partsp[i].part_name;
 2104                         break;
 2105                 }
 2106         }
 2107         /* Known Implementer, Unknown part */
 2108         if (cpu_part_name == NULL) {
 2109                 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
 2110                     cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
 2111                 return;
 2112         }
 2113 
 2114         sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
 2115             cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
 2116 }
 2117 
 2118 static void
 2119 print_cpu_cache(u_int cpu, struct sbuf *sb, uint64_t ccs, bool icache,
 2120     bool unified)
 2121 {
 2122         size_t cache_size;
 2123         size_t line_size;
 2124 
 2125         /* LineSize is Log2(S) - 4. */
 2126         line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4);
 2127         /*
 2128          * Calculate cache size (sets * ways * line size).  There are different
 2129          * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature
 2130          * register.
 2131          */
 2132         if ((cpu_desc[cpu].id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64))
 2133                 cache_size = (CCSIDR_NSETS_64(ccs) + 1) *
 2134                     (CCSIDR_ASSOC_64(ccs) + 1);
 2135         else
 2136                 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1);
 2137 
 2138         cache_size *= line_size;
 2139         sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024,
 2140             icache ? "instruction" : unified ? "unified" : "data");
 2141 }
 2142 
 2143 static void
 2144 print_cpu_caches(struct sbuf *sb, u_int cpu)
 2145 {
 2146         /* Print out each cache combination */
 2147         uint64_t clidr;
 2148         int i = 1;
 2149         clidr = cpu_desc[cpu].clidr;
 2150 
 2151         for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
 2152                 int j = 0;
 2153                 int ctype_m = (clidr & CLIDR_CTYPE_MASK);
 2154 
 2155                 sbuf_printf(sb, " L%d cache: ", i + 1);
 2156                 if ((clidr & CLIDR_CTYPE_IO)) {
 2157                         print_cpu_cache(cpu, sb, cpu_desc[cpu].ccsidr[i][j++],
 2158                             true, false);
 2159                         /* If there's more, add to the line. */
 2160                         if ((ctype_m & ~CLIDR_CTYPE_IO) != 0)
 2161                                 sbuf_printf(sb, ", ");
 2162                 }
 2163                 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) {
 2164                         print_cpu_cache(cpu, sb, cpu_desc[cpu].ccsidr[i][j],
 2165                             false, (clidr & CLIDR_CTYPE_UNIFIED));
 2166                 }
 2167                 sbuf_printf(sb, "\n");
 2168 
 2169         }
 2170         sbuf_finish(sb);
 2171         printf("%s", sbuf_data(sb));
 2172 }
 2173 
 2174 static void
 2175 print_cpu_features(u_int cpu)
 2176 {
 2177         struct sbuf *sb;
 2178 
 2179         sb = sbuf_new_auto();
 2180         sbuf_printf(sb, "CPU%3u: ", cpu);
 2181         print_cpu_midr(sb, cpu);
 2182 
 2183         sbuf_cat(sb, " affinity:");
 2184         switch(cpu_aff_levels) {
 2185         default:
 2186         case 4:
 2187                 sbuf_printf(sb, " %2d", CPU_AFF3(cpu_desc[cpu].mpidr));
 2188                 /* FALLTHROUGH */
 2189         case 3:
 2190                 sbuf_printf(sb, " %2d", CPU_AFF2(cpu_desc[cpu].mpidr));
 2191                 /* FALLTHROUGH */
 2192         case 2:
 2193                 sbuf_printf(sb, " %2d", CPU_AFF1(cpu_desc[cpu].mpidr));
 2194                 /* FALLTHROUGH */
 2195         case 1:
 2196         case 0: /* On UP this will be zero */
 2197                 sbuf_printf(sb, " %2d", CPU_AFF0(cpu_desc[cpu].mpidr));
 2198                 break;
 2199         }
 2200         sbuf_finish(sb);
 2201         printf("%s\n", sbuf_data(sb));
 2202         sbuf_clear(sb);
 2203 
 2204         /*
 2205          * There is a hardware errata where, if one CPU is performing a TLB
 2206          * invalidation while another is performing a store-exclusive the
 2207          * store-exclusive may return the wrong status. A workaround seems
 2208          * to be to use an IPI to invalidate on each CPU, however given the
 2209          * limited number of affected units (pass 1.1 is the evaluation
 2210          * hardware revision), and the lack of information from Cavium
 2211          * this has not been implemented.
 2212          *
 2213          * At the time of writing this the only information is from:
 2214          * https://lkml.org/lkml/2016/8/4/722
 2215          */
 2216         /*
 2217          * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
 2218          * triggers on pass 2.0+.
 2219          */
 2220         if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
 2221             CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
 2222                 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
 2223                     "hardware bugs that may cause the incorrect operation of "
 2224                     "atomic operations.\n");
 2225 
 2226 #define SHOULD_PRINT_REG(_reg)                                          \
 2227     (cpu == 0 || cpu_desc[cpu]._reg != cpu_desc[cpu - 1]._reg)
 2228 
 2229         /* Cache Type Register */
 2230         if (SHOULD_PRINT_REG(ctr)) {
 2231                 print_register(sb, "Cache Type",
 2232                     cpu_desc[cpu].ctr, print_ctr_fields, NULL);
 2233         }
 2234 
 2235         /* AArch64 Instruction Set Attribute Register 0 */
 2236         if (SHOULD_PRINT_REG(id_aa64isar0))
 2237                 print_id_register(sb, "Instruction Set Attributes 0",
 2238                     cpu_desc[cpu].id_aa64isar0, id_aa64isar0_fields);
 2239 
 2240         /* AArch64 Instruction Set Attribute Register 1 */
 2241         if (SHOULD_PRINT_REG(id_aa64isar1))
 2242                 print_id_register(sb, "Instruction Set Attributes 1",
 2243                     cpu_desc[cpu].id_aa64isar1, id_aa64isar1_fields);
 2244 
 2245         /* AArch64 Instruction Set Attribute Register 2 */
 2246         if (SHOULD_PRINT_REG(id_aa64isar2))
 2247                 print_id_register(sb, "Instruction Set Attributes 2",
 2248                     cpu_desc[cpu].id_aa64isar2, id_aa64isar2_fields);
 2249 
 2250         /* AArch64 Processor Feature Register 0 */
 2251         if (SHOULD_PRINT_REG(id_aa64pfr0))
 2252                 print_id_register(sb, "Processor Features 0",
 2253                     cpu_desc[cpu].id_aa64pfr0, id_aa64pfr0_fields);
 2254 
 2255         /* AArch64 Processor Feature Register 1 */
 2256         if (SHOULD_PRINT_REG(id_aa64pfr1))
 2257                 print_id_register(sb, "Processor Features 1",
 2258                     cpu_desc[cpu].id_aa64pfr1, id_aa64pfr1_fields);
 2259 
 2260         /* AArch64 Memory Model Feature Register 0 */
 2261         if (SHOULD_PRINT_REG(id_aa64mmfr0))
 2262                 print_id_register(sb, "Memory Model Features 0",
 2263                     cpu_desc[cpu].id_aa64mmfr0, id_aa64mmfr0_fields);
 2264 
 2265         /* AArch64 Memory Model Feature Register 1 */
 2266         if (SHOULD_PRINT_REG(id_aa64mmfr1))
 2267                 print_id_register(sb, "Memory Model Features 1",
 2268                     cpu_desc[cpu].id_aa64mmfr1, id_aa64mmfr1_fields);
 2269 
 2270         /* AArch64 Memory Model Feature Register 2 */
 2271         if (SHOULD_PRINT_REG(id_aa64mmfr2))
 2272                 print_id_register(sb, "Memory Model Features 2",
 2273                     cpu_desc[cpu].id_aa64mmfr2, id_aa64mmfr2_fields);
 2274 
 2275         /* AArch64 Debug Feature Register 0 */
 2276         if (SHOULD_PRINT_REG(id_aa64dfr0))
 2277                 print_id_register(sb, "Debug Features 0",
 2278                     cpu_desc[cpu].id_aa64dfr0, id_aa64dfr0_fields);
 2279 
 2280         /* AArch64 Memory Model Feature Register 1 */
 2281         if (SHOULD_PRINT_REG(id_aa64dfr1))
 2282                 print_id_register(sb, "Debug Features 1",
 2283                     cpu_desc[cpu].id_aa64dfr1, id_aa64dfr1_fields);
 2284 
 2285         /* AArch64 Auxiliary Feature Register 0 */
 2286         if (SHOULD_PRINT_REG(id_aa64afr0))
 2287                 print_id_register(sb, "Auxiliary Features 0",
 2288                     cpu_desc[cpu].id_aa64afr0, id_aa64afr0_fields);
 2289 
 2290         /* AArch64 Auxiliary Feature Register 1 */
 2291         if (SHOULD_PRINT_REG(id_aa64afr1))
 2292                 print_id_register(sb, "Auxiliary Features 1",
 2293                     cpu_desc[cpu].id_aa64afr1, id_aa64afr1_fields);
 2294 
 2295         /* AArch64 SVE Feature Register 0 */
 2296         if (cpu_desc[cpu].have_sve) {
 2297                 if (SHOULD_PRINT_REG(id_aa64zfr0) ||
 2298                     !cpu_desc[cpu - 1].have_sve) {
 2299                         print_id_register(sb, "SVE Features 0",
 2300                             cpu_desc[cpu].id_aa64zfr0, id_aa64zfr0_fields);
 2301                 }
 2302         }
 2303 
 2304 #ifdef COMPAT_FREEBSD32
 2305         /* AArch32 Instruction Set Attribute Register 5 */
 2306         if (SHOULD_PRINT_REG(id_isar5))
 2307                 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
 2308                      cpu_desc[cpu].id_isar5, id_isar5_fields);
 2309 
 2310         /* AArch32 Media and VFP Feature Register 0 */
 2311         if (SHOULD_PRINT_REG(mvfr0))
 2312                 print_id_register(sb, "AArch32 Media and VFP Features 0",
 2313                      cpu_desc[cpu].mvfr0, mvfr0_fields);
 2314 
 2315         /* AArch32 Media and VFP Feature Register 1 */
 2316         if (SHOULD_PRINT_REG(mvfr1))
 2317                 print_id_register(sb, "AArch32 Media and VFP Features 1",
 2318                      cpu_desc[cpu].mvfr1, mvfr1_fields);
 2319 #endif
 2320         if (bootverbose)
 2321                 print_cpu_caches(sb, cpu);
 2322 
 2323         sbuf_delete(sb);
 2324         sb = NULL;
 2325 #undef SHOULD_PRINT_REG
 2326 #undef SEP_STR
 2327 }
 2328 
 2329 void
 2330 identify_cache(uint64_t ctr)
 2331 {
 2332 
 2333         /* Identify the L1 cache type */
 2334         switch (CTR_L1IP_VAL(ctr)) {
 2335         case CTR_L1IP_PIPT:
 2336                 break;
 2337         case CTR_L1IP_VPIPT:
 2338                 icache_vmid = true;
 2339                 break;
 2340         default:
 2341         case CTR_L1IP_VIPT:
 2342                 icache_aliasing = true;
 2343                 break;
 2344         }
 2345 
 2346         if (dcache_line_size == 0) {
 2347                 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
 2348                     __func__, icache_line_size));
 2349 
 2350                 /* Get the D cache line size */
 2351                 dcache_line_size = CTR_DLINE_SIZE(ctr);
 2352                 /* And the same for the I cache */
 2353                 icache_line_size = CTR_ILINE_SIZE(ctr);
 2354 
 2355                 idcache_line_size = MIN(dcache_line_size, icache_line_size);
 2356         }
 2357 
 2358         if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
 2359                 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
 2360                     dcache_line_size, CTR_DLINE_SIZE(ctr));
 2361         }
 2362 
 2363         if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
 2364                 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
 2365                     icache_line_size, CTR_ILINE_SIZE(ctr));
 2366         }
 2367 }
 2368 
 2369 void
 2370 identify_cpu(u_int cpu)
 2371 {
 2372         uint64_t clidr;
 2373 
 2374         /* Save affinity for current CPU */
 2375         cpu_desc[cpu].mpidr = get_mpidr();
 2376         CPU_AFFINITY(cpu) = cpu_desc[cpu].mpidr & CPU_AFF_MASK;
 2377 
 2378         cpu_desc[cpu].ctr = READ_SPECIALREG(ctr_el0);
 2379         cpu_desc[cpu].id_aa64dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
 2380         cpu_desc[cpu].id_aa64dfr1 = READ_SPECIALREG(id_aa64dfr1_el1);
 2381         cpu_desc[cpu].id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
 2382         cpu_desc[cpu].id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
 2383         cpu_desc[cpu].id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1);
 2384         cpu_desc[cpu].id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
 2385         cpu_desc[cpu].id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
 2386         cpu_desc[cpu].id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
 2387         cpu_desc[cpu].id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
 2388         cpu_desc[cpu].id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
 2389 
 2390         /*
 2391          * ID_AA64ZFR0_EL1 is only valid when at least one of:
 2392          *  - ID_AA64PFR0_EL1.SVE is non-zero
 2393          *  - ID_AA64PFR1_EL1.SME is non-zero
 2394          * In other cases it is zero, but still safe to read
 2395          */
 2396         cpu_desc[cpu].have_sve =
 2397             (ID_AA64PFR0_SVE_VAL(cpu_desc[cpu].id_aa64pfr0) != 0);
 2398         cpu_desc[cpu].id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG);
 2399 
 2400         cpu_desc[cpu].clidr = READ_SPECIALREG(clidr_el1);
 2401 
 2402         clidr = cpu_desc[cpu].clidr;
 2403 
 2404         for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
 2405                 int j = 0;
 2406                 if ((clidr & CLIDR_CTYPE_IO)) {
 2407                         WRITE_SPECIALREG(csselr_el1,
 2408                             CSSELR_Level(i) | CSSELR_InD);
 2409                         cpu_desc[cpu].ccsidr[i][j++] =
 2410                             READ_SPECIALREG(ccsidr_el1);
 2411                 }
 2412                 if ((clidr & ~CLIDR_CTYPE_IO) == 0)
 2413                         continue;
 2414                 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i));
 2415                 cpu_desc[cpu].ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1);
 2416         }
 2417 
 2418 #ifdef COMPAT_FREEBSD32
 2419         /* Only read aarch32 SRs if EL0-32 is available */
 2420         if (ID_AA64PFR0_EL0_VAL(cpu_desc[cpu].id_aa64pfr0) ==
 2421             ID_AA64PFR0_EL0_64_32) {
 2422                 cpu_desc[cpu].id_isar5 = READ_SPECIALREG(id_isar5_el1);
 2423                 cpu_desc[cpu].mvfr0 = READ_SPECIALREG(mvfr0_el1);
 2424                 cpu_desc[cpu].mvfr1 = READ_SPECIALREG(mvfr1_el1);
 2425         }
 2426 #endif
 2427 }
 2428 
 2429 static void
 2430 check_cpu_regs(u_int cpu)
 2431 {
 2432 
 2433         switch (cpu_aff_levels) {
 2434         case 0:
 2435                 if (CPU_AFF0(cpu_desc[cpu].mpidr) !=
 2436                     CPU_AFF0(cpu_desc[0].mpidr))
 2437                         cpu_aff_levels = 1;
 2438                 /* FALLTHROUGH */
 2439         case 1:
 2440                 if (CPU_AFF1(cpu_desc[cpu].mpidr) !=
 2441                     CPU_AFF1(cpu_desc[0].mpidr))
 2442                         cpu_aff_levels = 2;
 2443                 /* FALLTHROUGH */
 2444         case 2:
 2445                 if (CPU_AFF2(cpu_desc[cpu].mpidr) !=
 2446                     CPU_AFF2(cpu_desc[0].mpidr))
 2447                         cpu_aff_levels = 3;
 2448                 /* FALLTHROUGH */
 2449         case 3:
 2450                 if (CPU_AFF3(cpu_desc[cpu].mpidr) !=
 2451                     CPU_AFF3(cpu_desc[0].mpidr))
 2452                         cpu_aff_levels = 4;
 2453                 break;
 2454         }
 2455 
 2456         if (cpu_desc[cpu].ctr != cpu_desc[0].ctr) {
 2457                 /*
 2458                  * If the cache type register is different we may
 2459                  * have a different l1 cache type.
 2460                  */
 2461                 identify_cache(cpu_desc[cpu].ctr);
 2462         }
 2463 }

Cache object: 5b82370047df8cb67ff844e7e053012f

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.