1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2003-2008 Joseph Koshy
5 * Copyright (c) 2007 The FreeBSD Foundation
6 * All rights reserved.
7 *
8 * Portions of this software were developed by A. Joseph Koshy under
9 * sponsorship from the FreeBSD Foundation and Google, Inc.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35
36 /* Support for the AMD K7 and later processors */
37
38 #include <sys/param.h>
39 #include <sys/lock.h>
40 #include <sys/malloc.h>
41 #include <sys/mutex.h>
42 #include <sys/pcpu.h>
43 #include <sys/pmc.h>
44 #include <sys/pmckern.h>
45 #include <sys/smp.h>
46 #include <sys/systm.h>
47
48 #include <machine/cpu.h>
49 #include <machine/cpufunc.h>
50 #include <machine/md_var.h>
51 #include <machine/specialreg.h>
52
53 #ifdef HWPMC_DEBUG
54 enum pmc_class amd_pmc_class;
55 #endif
56
57 #define OVERFLOW_WAIT_COUNT 50
58
59 DPCPU_DEFINE_STATIC(uint32_t, nmi_counter);
60
61 /* AMD K7 & K8 PMCs */
62 struct amd_descr {
63 struct pmc_descr pm_descr; /* "base class" */
64 uint32_t pm_evsel; /* address of EVSEL register */
65 uint32_t pm_perfctr; /* address of PERFCTR register */
66 };
67
68 static struct amd_descr amd_pmcdesc[AMD_NPMCS] =
69 {
70 {
71 .pm_descr =
72 {
73 .pd_name = "",
74 .pd_class = -1,
75 .pd_caps = AMD_PMC_CAPS,
76 .pd_width = 48
77 },
78 .pm_evsel = AMD_PMC_EVSEL_0,
79 .pm_perfctr = AMD_PMC_PERFCTR_0
80 },
81 {
82 .pm_descr =
83 {
84 .pd_name = "",
85 .pd_class = -1,
86 .pd_caps = AMD_PMC_CAPS,
87 .pd_width = 48
88 },
89 .pm_evsel = AMD_PMC_EVSEL_1,
90 .pm_perfctr = AMD_PMC_PERFCTR_1
91 },
92 {
93 .pm_descr =
94 {
95 .pd_name = "",
96 .pd_class = -1,
97 .pd_caps = AMD_PMC_CAPS,
98 .pd_width = 48
99 },
100 .pm_evsel = AMD_PMC_EVSEL_2,
101 .pm_perfctr = AMD_PMC_PERFCTR_2
102 },
103 {
104 .pm_descr =
105 {
106 .pd_name = "",
107 .pd_class = -1,
108 .pd_caps = AMD_PMC_CAPS,
109 .pd_width = 48
110 },
111 .pm_evsel = AMD_PMC_EVSEL_3,
112 .pm_perfctr = AMD_PMC_PERFCTR_3
113 },
114 {
115 .pm_descr =
116 {
117 .pd_name = "",
118 .pd_class = -1,
119 .pd_caps = AMD_PMC_CAPS,
120 .pd_width = 48
121 },
122 .pm_evsel = AMD_PMC_EVSEL_4,
123 .pm_perfctr = AMD_PMC_PERFCTR_4
124 },
125 {
126 .pm_descr =
127 {
128 .pd_name = "",
129 .pd_class = -1,
130 .pd_caps = AMD_PMC_CAPS,
131 .pd_width = 48
132 },
133 .pm_evsel = AMD_PMC_EVSEL_5,
134 .pm_perfctr = AMD_PMC_PERFCTR_5
135 },
136 {
137 .pm_descr =
138 {
139 .pd_name = "",
140 .pd_class = -1,
141 .pd_caps = AMD_PMC_CAPS,
142 .pd_width = 48
143 },
144 .pm_evsel = AMD_PMC_EVSEL_EP_L3_0,
145 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_0
146 },
147 {
148 .pm_descr =
149 {
150 .pd_name = "",
151 .pd_class = -1,
152 .pd_caps = AMD_PMC_CAPS,
153 .pd_width = 48
154 },
155 .pm_evsel = AMD_PMC_EVSEL_EP_L3_1,
156 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_1
157 },
158 {
159 .pm_descr =
160 {
161 .pd_name = "",
162 .pd_class = -1,
163 .pd_caps = AMD_PMC_CAPS,
164 .pd_width = 48
165 },
166 .pm_evsel = AMD_PMC_EVSEL_EP_L3_2,
167 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_2
168 },
169 {
170 .pm_descr =
171 {
172 .pd_name = "",
173 .pd_class = -1,
174 .pd_caps = AMD_PMC_CAPS,
175 .pd_width = 48
176 },
177 .pm_evsel = AMD_PMC_EVSEL_EP_L3_3,
178 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_3
179 },
180 {
181 .pm_descr =
182 {
183 .pd_name = "",
184 .pd_class = -1,
185 .pd_caps = AMD_PMC_CAPS,
186 .pd_width = 48
187 },
188 .pm_evsel = AMD_PMC_EVSEL_EP_L3_4,
189 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_4
190 },
191 {
192 .pm_descr =
193 {
194 .pd_name = "",
195 .pd_class = -1,
196 .pd_caps = AMD_PMC_CAPS,
197 .pd_width = 48
198 },
199 .pm_evsel = AMD_PMC_EVSEL_EP_L3_5,
200 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_5
201 },
202 {
203 .pm_descr =
204 {
205 .pd_name = "",
206 .pd_class = -1,
207 .pd_caps = AMD_PMC_CAPS,
208 .pd_width = 48
209 },
210 .pm_evsel = AMD_PMC_EVSEL_EP_DF_0,
211 .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_0
212 },
213 {
214 .pm_descr =
215 {
216 .pd_name = "",
217 .pd_class = -1,
218 .pd_caps = AMD_PMC_CAPS,
219 .pd_width = 48
220 },
221 .pm_evsel = AMD_PMC_EVSEL_EP_DF_1,
222 .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_1
223 },
224 {
225 .pm_descr =
226 {
227 .pd_name = "",
228 .pd_class = -1,
229 .pd_caps = AMD_PMC_CAPS,
230 .pd_width = 48
231 },
232 .pm_evsel = AMD_PMC_EVSEL_EP_DF_2,
233 .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_2
234 },
235 {
236 .pm_descr =
237 {
238 .pd_name = "",
239 .pd_class = -1,
240 .pd_caps = AMD_PMC_CAPS,
241 .pd_width = 48
242 },
243 .pm_evsel = AMD_PMC_EVSEL_EP_DF_3,
244 .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_3
245 }
246 };
247
248 struct amd_event_code_map {
249 enum pmc_event pe_ev; /* enum value */
250 uint16_t pe_code; /* encoded event mask */
251 uint8_t pe_mask; /* bits allowed in unit mask */
252 };
253
254 const struct amd_event_code_map amd_event_codes[] = {
255 #if defined(__i386__) /* 32 bit Athlon (K7) only */
256 { PMC_EV_K7_DC_ACCESSES, 0x40, 0 },
257 { PMC_EV_K7_DC_MISSES, 0x41, 0 },
258 { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI },
259 { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI },
260 { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI },
261 { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 },
262 { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 },
263 { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 },
264
265 { PMC_EV_K7_IC_FETCHES, 0x80, 0 },
266 { PMC_EV_K7_IC_MISSES, 0x81, 0 },
267
268 { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 },
269 { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 },
270
271 { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 },
272 { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 },
273 { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 },
274 { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 },
275 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 },
276 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 },
277 { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 },
278 { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 },
279 { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 },
280 { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 },
281 { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 },
282 #endif
283
284 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F },
285 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 },
286 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 },
287
288 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F },
289 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE,
290 0x21, 0x00 },
291 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 },
292 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 },
293 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 },
294 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 },
295 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 },
296 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 },
297
298 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 },
299 { PMC_EV_K8_DC_MISS, 0x41, 0x00 },
300 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F },
301 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F },
302 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F },
303 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 },
304 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 },
305 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 },
306 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 },
307 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 },
308 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 },
309 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 },
310 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 },
311
312 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 },
313 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F },
314 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 },
315 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 },
316
317 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 },
318 { PMC_EV_K8_IC_MISS, 0x81, 0x00 },
319 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 },
320 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 },
321 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 },
322 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 },
323 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 },
324 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 },
325 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 },
326 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 },
327
328 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 },
329 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 },
330 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 },
331 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 },
332 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 },
333 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 },
334 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 },
335 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 },
336 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 },
337 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 },
338 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE,
339 0xCA, 0x00 },
340 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F },
341 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS,
342 0xCC, 0x07 },
343 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 },
344 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 },
345 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 },
346
347 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 },
348 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 },
349 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE,
350 0xD2, 0x00 },
351 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 },
352 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 },
353 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL,
354 0xD5, 0x00 },
355 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL,
356 0xD6, 0x00 },
357 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 },
358 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 },
359 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET,
360 0xD9, 0x00 },
361 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING,
362 0xDA, 0x00 },
363 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F },
364 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 },
365 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 },
366 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 },
367 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 },
368
369 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 },
370 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 },
371 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED,
372 0xE2, 0x00 },
373 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 },
374 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F },
375 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F },
376 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F },
377 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F },
378 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F },
379 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F }
380
381 };
382
383 const int amd_event_codes_size = nitems(amd_event_codes);
384
385 /*
386 * Per-processor information
387 */
388
389 struct amd_cpu {
390 struct pmc_hw pc_amdpmcs[AMD_NPMCS];
391 };
392
393 static struct amd_cpu **amd_pcpu;
394
395 /*
396 * read a pmc register
397 */
398
399 static int
400 amd_read_pmc(int cpu, int ri, pmc_value_t *v)
401 {
402 enum pmc_mode mode;
403 const struct amd_descr *pd;
404 struct pmc *pm;
405 pmc_value_t tmp;
406
407 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
408 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
409 KASSERT(ri >= 0 && ri < AMD_NPMCS,
410 ("[amd,%d] illegal row-index %d", __LINE__, ri));
411 KASSERT(amd_pcpu[cpu],
412 ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu));
413
414 pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
415 pd = &amd_pmcdesc[ri];
416
417 KASSERT(pm != NULL,
418 ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
419 cpu, ri));
420
421 mode = PMC_TO_MODE(pm);
422
423 PMCDBG2(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class);
424
425 #ifdef HWPMC_DEBUG
426 KASSERT(pd->pm_descr.pd_class == amd_pmc_class,
427 ("[amd,%d] unknown PMC class (%d)", __LINE__,
428 pd->pm_descr.pd_class));
429 #endif
430
431 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */
432 PMCDBG2(MDP,REA,2,"amd-read (pre-munge) id=%d -> %jd", ri, tmp);
433 if (PMC_IS_SAMPLING_MODE(mode)) {
434 /*
435 * Clamp value to 0 if the counter just overflowed,
436 * otherwise the returned reload count would wrap to a
437 * huge value.
438 */
439 if ((tmp & (1ULL << 47)) == 0)
440 tmp = 0;
441 else {
442 /* Sign extend 48 bit value to 64 bits. */
443 tmp = (pmc_value_t) ((int64_t)(tmp << 16) >> 16);
444 tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
445 }
446 }
447 *v = tmp;
448
449 PMCDBG2(MDP,REA,2,"amd-read (post-munge) id=%d -> %jd", ri, *v);
450
451 return 0;
452 }
453
454 /*
455 * Write a PMC MSR.
456 */
457
458 static int
459 amd_write_pmc(int cpu, int ri, pmc_value_t v)
460 {
461 const struct amd_descr *pd;
462 enum pmc_mode mode;
463 struct pmc *pm;
464
465 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
466 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
467 KASSERT(ri >= 0 && ri < AMD_NPMCS,
468 ("[amd,%d] illegal row-index %d", __LINE__, ri));
469
470 pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
471 pd = &amd_pmcdesc[ri];
472
473 KASSERT(pm != NULL,
474 ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__,
475 cpu, ri));
476
477 mode = PMC_TO_MODE(pm);
478
479 #ifdef HWPMC_DEBUG
480 KASSERT(pd->pm_descr.pd_class == amd_pmc_class,
481 ("[amd,%d] unknown PMC class (%d)", __LINE__,
482 pd->pm_descr.pd_class));
483 #endif
484
485 /* use 2's complement of the count for sampling mode PMCs */
486 if (PMC_IS_SAMPLING_MODE(mode))
487 v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
488
489 PMCDBG3(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v);
490
491 /* write the PMC value */
492 wrmsr(pd->pm_perfctr, v);
493 return 0;
494 }
495
496 /*
497 * configure hardware pmc according to the configuration recorded in
498 * pmc 'pm'.
499 */
500
501 static int
502 amd_config_pmc(int cpu, int ri, struct pmc *pm)
503 {
504 struct pmc_hw *phw;
505
506 PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
507
508 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
509 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
510 KASSERT(ri >= 0 && ri < AMD_NPMCS,
511 ("[amd,%d] illegal row-index %d", __LINE__, ri));
512
513 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
514
515 KASSERT(pm == NULL || phw->phw_pmc == NULL,
516 ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
517 __LINE__, pm, phw->phw_pmc));
518
519 phw->phw_pmc = pm;
520 return 0;
521 }
522
523 /*
524 * Retrieve a configured PMC pointer from hardware state.
525 */
526
527 static int
528 amd_get_config(int cpu, int ri, struct pmc **ppm)
529 {
530 *ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
531
532 return 0;
533 }
534
535 /*
536 * Machine dependent actions taken during the context switch in of a
537 * thread.
538 */
539
540 static int
541 amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
542 {
543 (void) pc;
544
545 PMCDBG3(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
546 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0);
547
548 /* enable the RDPMC instruction if needed */
549 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
550 load_cr4(rcr4() | CR4_PCE);
551
552 return 0;
553 }
554
555 /*
556 * Machine dependent actions taken during the context switch out of a
557 * thread.
558 */
559
560 static int
561 amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
562 {
563 (void) pc;
564 (void) pp; /* can be NULL */
565
566 PMCDBG3(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ?
567 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0);
568
569 /* always turn off the RDPMC instruction */
570 load_cr4(rcr4() & ~CR4_PCE);
571
572 return 0;
573 }
574
575 /*
576 * Check if a given allocation is feasible.
577 */
578
579 static int
580 amd_allocate_pmc(int cpu, int ri, struct pmc *pm,
581 const struct pmc_op_pmcallocate *a)
582 {
583 int i;
584 uint64_t allowed_unitmask, caps, config, unitmask;
585 enum pmc_event pe;
586 const struct pmc_descr *pd;
587
588 (void) cpu;
589
590 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
591 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
592 KASSERT(ri >= 0 && ri < AMD_NPMCS,
593 ("[amd,%d] illegal row index %d", __LINE__, ri));
594
595 pd = &amd_pmcdesc[ri].pm_descr;
596
597 /* check class match */
598 if (pd->pd_class != a->pm_class)
599 return EINVAL;
600
601 caps = pm->pm_caps;
602
603 PMCDBG2(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps);
604
605 if((ri >= 0 && ri < 6) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_CORE))
606 return EINVAL;
607 if((ri >= 6 && ri < 12) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_L3_CACHE))
608 return EINVAL;
609 if((ri >= 12 && ri < 16) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_DATA_FABRIC))
610 return EINVAL;
611
612 if (strlen(pmc_cpuid) != 0) {
613 pm->pm_md.pm_amd.pm_amd_evsel =
614 a->pm_md.pm_amd.pm_amd_config;
615 PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, a->pm_md.pm_amd.pm_amd_config);
616 return (0);
617 }
618
619 pe = a->pm_ev;
620
621 /* map ev to the correct event mask code */
622 config = allowed_unitmask = 0;
623 for (i = 0; i < amd_event_codes_size; i++)
624 if (amd_event_codes[i].pe_ev == pe) {
625 config =
626 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code);
627 allowed_unitmask =
628 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask);
629 break;
630 }
631 if (i == amd_event_codes_size)
632 return EINVAL;
633
634 unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK;
635 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
636 return EINVAL;
637
638 if (unitmask && (caps & PMC_CAP_QUALIFIER))
639 config |= unitmask;
640
641 if (caps & PMC_CAP_THRESHOLD)
642 config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK;
643
644 /* set at least one of the 'usr' or 'os' caps */
645 if (caps & PMC_CAP_USER)
646 config |= AMD_PMC_USR;
647 if (caps & PMC_CAP_SYSTEM)
648 config |= AMD_PMC_OS;
649 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0)
650 config |= (AMD_PMC_USR|AMD_PMC_OS);
651
652 if (caps & PMC_CAP_EDGE)
653 config |= AMD_PMC_EDGE;
654 if (caps & PMC_CAP_INVERT)
655 config |= AMD_PMC_INVERT;
656 if (caps & PMC_CAP_INTERRUPT)
657 config |= AMD_PMC_INT;
658
659 pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */
660
661 PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config);
662
663 return 0;
664 }
665
666 /*
667 * Release machine dependent state associated with a PMC. This is a
668 * no-op on this architecture.
669 *
670 */
671
672 /* ARGSUSED0 */
673 static int
674 amd_release_pmc(int cpu, int ri, struct pmc *pmc)
675 {
676 #ifdef HWPMC_DEBUG
677 const struct amd_descr *pd;
678 #endif
679 struct pmc_hw *phw __diagused;
680
681 (void) pmc;
682
683 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
684 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
685 KASSERT(ri >= 0 && ri < AMD_NPMCS,
686 ("[amd,%d] illegal row-index %d", __LINE__, ri));
687
688 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
689
690 KASSERT(phw->phw_pmc == NULL,
691 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
692
693 #ifdef HWPMC_DEBUG
694 pd = &amd_pmcdesc[ri];
695 if (pd->pm_descr.pd_class == amd_pmc_class)
696 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel),
697 ("[amd,%d] PMC %d released while active", __LINE__, ri));
698 #endif
699
700 return 0;
701 }
702
703 /*
704 * start a PMC.
705 */
706
707 static int
708 amd_start_pmc(int cpu, int ri)
709 {
710 uint64_t config;
711 struct pmc *pm;
712 struct pmc_hw *phw;
713 const struct amd_descr *pd;
714
715 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
716 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
717 KASSERT(ri >= 0 && ri < AMD_NPMCS,
718 ("[amd,%d] illegal row-index %d", __LINE__, ri));
719
720 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
721 pm = phw->phw_pmc;
722 pd = &amd_pmcdesc[ri];
723
724 KASSERT(pm != NULL,
725 ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__,
726 cpu, ri));
727
728 PMCDBG2(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri);
729
730 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel),
731 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__,
732 ri, cpu, pd->pm_descr.pd_name));
733
734 /* turn on the PMC ENABLE bit */
735 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE;
736
737 PMCDBG1(MDP,STA,2,"amd-start config=0x%x", config);
738
739 wrmsr(pd->pm_evsel, config);
740 return 0;
741 }
742
743 /*
744 * Stop a PMC.
745 */
746
747 static int
748 amd_stop_pmc(int cpu, int ri)
749 {
750 struct pmc *pm;
751 struct pmc_hw *phw;
752 const struct amd_descr *pd;
753 uint64_t config;
754 int i;
755
756 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
757 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
758 KASSERT(ri >= 0 && ri < AMD_NPMCS,
759 ("[amd,%d] illegal row-index %d", __LINE__, ri));
760
761 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
762 pm = phw->phw_pmc;
763 pd = &amd_pmcdesc[ri];
764
765 KASSERT(pm != NULL,
766 ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__,
767 cpu, ri));
768 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel),
769 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped",
770 __LINE__, ri, cpu, pd->pm_descr.pd_name));
771
772 PMCDBG1(MDP,STO,1,"amd-stop ri=%d", ri);
773
774 /* turn off the PMC ENABLE bit */
775 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE;
776 wrmsr(pd->pm_evsel, config);
777
778 /*
779 * Due to NMI latency on newer AMD processors
780 * NMI interrupts are ignored, which leads to
781 * panic or messages based on kernel configuration
782 */
783
784 /* Wait for the count to be reset */
785 for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) {
786 if (rdmsr(pd->pm_perfctr) & (1 << (pd->pm_descr.pd_width - 1)))
787 break;
788
789 DELAY(1);
790 }
791
792 return 0;
793 }
794
795 /*
796 * Interrupt handler. This function needs to return '1' if the
797 * interrupt was this CPU's PMCs or '' otherwise. It is not allowed
798 * to sleep or do anything a 'fast' interrupt handler is not allowed
799 * to do.
800 */
801
802 static int
803 amd_intr(struct trapframe *tf)
804 {
805 int i, error, retval, cpu;
806 uint64_t config, evsel, perfctr;
807 struct pmc *pm;
808 struct amd_cpu *pac;
809 pmc_value_t v;
810 uint32_t active = 0, count = 0;
811
812 cpu = curcpu;
813 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
814 ("[amd,%d] out of range CPU %d", __LINE__, cpu));
815
816 PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
817 TRAPF_USERMODE(tf));
818
819 retval = 0;
820
821 pac = amd_pcpu[cpu];
822
823 /*
824 * look for all PMCs that have interrupted:
825 * - look for a running, sampling PMC which has overflowed
826 * and which has a valid 'struct pmc' association
827 *
828 * If found, we call a helper to process the interrupt.
829 *
830 * PMCs interrupting at the same time are collapsed into
831 * a single interrupt. Check all the valid pmcs for
832 * overflow.
833 */
834
835 for (i = 0; i < AMD_CORE_NPMCS; i++) {
836
837 if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL ||
838 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
839 continue;
840 }
841
842 /* Consider pmc with valid handle as active */
843 active++;
844
845 if (!AMD_PMC_HAS_OVERFLOWED(i))
846 continue;
847
848 retval = 1; /* Found an interrupting PMC. */
849
850 if (pm->pm_state != PMC_STATE_RUNNING)
851 continue;
852
853 /* Stop the PMC, reload count. */
854 evsel = amd_pmcdesc[i].pm_evsel;
855 perfctr = amd_pmcdesc[i].pm_perfctr;
856 v = pm->pm_sc.pm_reloadcount;
857 config = rdmsr(evsel);
858
859 KASSERT((config & ~AMD_PMC_ENABLE) ==
860 (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE),
861 ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__,
862 (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel));
863
864 wrmsr(evsel, config & ~AMD_PMC_ENABLE);
865 wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
866
867 /* Restart the counter if logging succeeded. */
868 error = pmc_process_interrupt(PMC_HR, pm, tf);
869 if (error == 0)
870 wrmsr(evsel, config);
871 }
872
873 /*
874 * Due to NMI latency, there can be a scenario in which
875 * multiple pmcs gets serviced in an earlier NMI and we
876 * do not find an overflow in the subsequent NMI.
877 *
878 * For such cases we keep a per-cpu count of active NMIs
879 * and compare it with min(active pmcs, 2) to determine
880 * if this NMI was for a pmc overflow which was serviced
881 * in an earlier request or should be ignored.
882 */
883
884 if (retval) {
885 DPCPU_SET(nmi_counter, min(2, active));
886 } else {
887 if ((count = DPCPU_GET(nmi_counter))) {
888 retval = 1;
889 DPCPU_SET(nmi_counter, --count);
890 }
891 }
892
893 if (retval)
894 counter_u64_add(pmc_stats.pm_intr_processed, 1);
895 else
896 counter_u64_add(pmc_stats.pm_intr_ignored, 1);
897
898 PMCDBG1(MDP,INT,2, "retval=%d", retval);
899 return (retval);
900 }
901
902 /*
903 * describe a PMC
904 */
905 static int
906 amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
907 {
908 int error;
909 size_t copied;
910 const struct amd_descr *pd;
911 struct pmc_hw *phw;
912
913 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
914 ("[amd,%d] illegal CPU %d", __LINE__, cpu));
915 KASSERT(ri >= 0 && ri < AMD_NPMCS,
916 ("[amd,%d] row-index %d out of range", __LINE__, ri));
917
918 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
919 pd = &amd_pmcdesc[ri];
920
921 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
922 PMC_NAME_MAX, &copied)) != 0)
923 return error;
924
925 pi->pm_class = pd->pm_descr.pd_class;
926
927 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
928 pi->pm_enabled = TRUE;
929 *ppmc = phw->phw_pmc;
930 } else {
931 pi->pm_enabled = FALSE;
932 *ppmc = NULL;
933 }
934
935 return 0;
936 }
937
938 /*
939 * i386 specific entry points
940 */
941
942 /*
943 * return the MSR address of the given PMC.
944 */
945
946 static int
947 amd_get_msr(int ri, uint32_t *msr)
948 {
949 KASSERT(ri >= 0 && ri < AMD_NPMCS,
950 ("[amd,%d] ri %d out of range", __LINE__, ri));
951
952 *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0;
953
954 return (0);
955 }
956
957 /*
958 * processor dependent initialization.
959 */
960
961 static int
962 amd_pcpu_init(struct pmc_mdep *md, int cpu)
963 {
964 int classindex, first_ri, n;
965 struct pmc_cpu *pc;
966 struct amd_cpu *pac;
967 struct pmc_hw *phw;
968
969 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
970 ("[amd,%d] insane cpu number %d", __LINE__, cpu));
971
972 PMCDBG1(MDP,INI,1,"amd-init cpu=%d", cpu);
973
974 amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC,
975 M_WAITOK|M_ZERO);
976
977 /*
978 * Set the content of the hardware descriptors to a known
979 * state and initialize pointers in the MI per-cpu descriptor.
980 */
981 pc = pmc_pcpu[cpu];
982 #if defined(__amd64__)
983 classindex = PMC_MDEP_CLASS_INDEX_K8;
984 #elif defined(__i386__)
985 classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ?
986 PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7;
987 #endif
988 first_ri = md->pmd_classdep[classindex].pcd_ri;
989
990 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__));
991
992 for (n = 0, phw = pac->pc_amdpmcs; n < AMD_NPMCS; n++, phw++) {
993 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
994 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
995 phw->phw_pmc = NULL;
996 pc->pc_hwpmcs[n + first_ri] = phw;
997 }
998
999 return (0);
1000 }
1001
1002
1003 /*
1004 * processor dependent cleanup prior to the KLD
1005 * being unloaded
1006 */
1007
1008 static int
1009 amd_pcpu_fini(struct pmc_mdep *md, int cpu)
1010 {
1011 int classindex, first_ri, i;
1012 uint32_t evsel;
1013 struct pmc_cpu *pc;
1014 struct amd_cpu *pac;
1015
1016 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1017 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu));
1018
1019 PMCDBG1(MDP,INI,1,"amd-cleanup cpu=%d", cpu);
1020
1021 /*
1022 * First, turn off all PMCs on this CPU.
1023 */
1024 for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */
1025 evsel = rdmsr(AMD_PMC_EVSEL_0 + i);
1026 evsel &= ~AMD_PMC_ENABLE;
1027 wrmsr(AMD_PMC_EVSEL_0 + i, evsel);
1028 }
1029
1030 /*
1031 * Next, free up allocated space.
1032 */
1033 if ((pac = amd_pcpu[cpu]) == NULL)
1034 return (0);
1035
1036 amd_pcpu[cpu] = NULL;
1037
1038 #ifdef HWPMC_DEBUG
1039 for (i = 0; i < AMD_NPMCS; i++) {
1040 KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL,
1041 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i));
1042 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i),
1043 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i));
1044 }
1045 #endif
1046
1047 pc = pmc_pcpu[cpu];
1048 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__));
1049
1050 #if defined(__amd64__)
1051 classindex = PMC_MDEP_CLASS_INDEX_K8;
1052 #elif defined(__i386__)
1053 classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 :
1054 PMC_MDEP_CLASS_INDEX_K7;
1055 #endif
1056 first_ri = md->pmd_classdep[classindex].pcd_ri;
1057
1058 /*
1059 * Reset pointers in the MI 'per-cpu' state.
1060 */
1061 for (i = 0; i < AMD_NPMCS; i++) {
1062 pc->pc_hwpmcs[i + first_ri] = NULL;
1063 }
1064
1065
1066 free(pac, M_PMC);
1067
1068 return (0);
1069 }
1070
1071 /*
1072 * Initialize ourselves.
1073 */
1074
1075 struct pmc_mdep *
1076 pmc_amd_initialize(void)
1077 {
1078 int classindex, error, i, ncpus;
1079 struct pmc_classdep *pcd;
1080 enum pmc_cputype cputype;
1081 struct pmc_mdep *pmc_mdep;
1082 enum pmc_class class;
1083 int family, model, stepping;
1084 char *name;
1085
1086 /*
1087 * The presence of hardware performance counters on the AMD
1088 * Athlon, Duron or later processors, is _not_ indicated by
1089 * any of the processor feature flags set by the 'CPUID'
1090 * instruction, so we only check the 'instruction family'
1091 * field returned by CPUID for instruction family >= 6.
1092 */
1093
1094 name = NULL;
1095 family = CPUID_TO_FAMILY(cpu_id);
1096 model = CPUID_TO_MODEL(cpu_id);
1097 stepping = CPUID_TO_STEPPING(cpu_id);
1098
1099 if (family == 0x18)
1100 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X",
1101 family, model, stepping);
1102 else
1103 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "AuthenticAMD-%d-%02X-%X",
1104 family, model, stepping);
1105
1106 switch (cpu_id & 0xF00) {
1107 #if defined(__i386__)
1108 case 0x600: /* Athlon(tm) processor */
1109 classindex = PMC_MDEP_CLASS_INDEX_K7;
1110 cputype = PMC_CPU_AMD_K7;
1111 class = PMC_CLASS_K7;
1112 name = "K7";
1113 break;
1114 #endif
1115 case 0xF00: /* Athlon64/Opteron processor */
1116 classindex = PMC_MDEP_CLASS_INDEX_K8;
1117 cputype = PMC_CPU_AMD_K8;
1118 class = PMC_CLASS_K8;
1119 name = "K8";
1120 break;
1121
1122 default:
1123 (void) printf("pmc: Unknown AMD CPU %x %d-%d.\n", cpu_id, (cpu_id & 0xF00) >> 8, model);
1124 return NULL;
1125 }
1126
1127 #ifdef HWPMC_DEBUG
1128 amd_pmc_class = class;
1129 #endif
1130
1131 /*
1132 * Allocate space for pointers to PMC HW descriptors and for
1133 * the MDEP structure used by MI code.
1134 */
1135 amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC,
1136 M_WAITOK|M_ZERO);
1137
1138 /*
1139 * These processors have two classes of PMCs: the TSC and
1140 * programmable PMCs.
1141 */
1142 pmc_mdep = pmc_mdep_alloc(2);
1143
1144 pmc_mdep->pmd_cputype = cputype;
1145
1146 ncpus = pmc_cpu_max();
1147
1148 /* Initialize the TSC. */
1149 error = pmc_tsc_initialize(pmc_mdep, ncpus);
1150 if (error)
1151 goto error;
1152
1153 /* Initialize AMD K7 and K8 PMC handling. */
1154 pcd = &pmc_mdep->pmd_classdep[classindex];
1155
1156 pcd->pcd_caps = AMD_PMC_CAPS;
1157 pcd->pcd_class = class;
1158 pcd->pcd_num = AMD_NPMCS;
1159 pcd->pcd_ri = pmc_mdep->pmd_npmc;
1160 pcd->pcd_width = 48;
1161
1162 /* fill in the correct pmc name and class */
1163 for (i = 0; i < AMD_NPMCS; i++) {
1164 (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name,
1165 sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d",
1166 name, i);
1167 amd_pmcdesc[i].pm_descr.pd_class = class;
1168 }
1169
1170 pcd->pcd_allocate_pmc = amd_allocate_pmc;
1171 pcd->pcd_config_pmc = amd_config_pmc;
1172 pcd->pcd_describe = amd_describe;
1173 pcd->pcd_get_config = amd_get_config;
1174 pcd->pcd_get_msr = amd_get_msr;
1175 pcd->pcd_pcpu_fini = amd_pcpu_fini;
1176 pcd->pcd_pcpu_init = amd_pcpu_init;
1177 pcd->pcd_read_pmc = amd_read_pmc;
1178 pcd->pcd_release_pmc = amd_release_pmc;
1179 pcd->pcd_start_pmc = amd_start_pmc;
1180 pcd->pcd_stop_pmc = amd_stop_pmc;
1181 pcd->pcd_write_pmc = amd_write_pmc;
1182
1183 pmc_mdep->pmd_pcpu_init = NULL;
1184 pmc_mdep->pmd_pcpu_fini = NULL;
1185 pmc_mdep->pmd_intr = amd_intr;
1186 pmc_mdep->pmd_switch_in = amd_switch_in;
1187 pmc_mdep->pmd_switch_out = amd_switch_out;
1188
1189 pmc_mdep->pmd_npmc += AMD_NPMCS;
1190
1191 PMCDBG0(MDP,INI,0,"amd-initialize");
1192
1193 return (pmc_mdep);
1194
1195 error:
1196 if (error) {
1197 free(pmc_mdep, M_PMC);
1198 pmc_mdep = NULL;
1199 }
1200
1201 return (NULL);
1202 }
1203
1204 /*
1205 * Finalization code for AMD CPUs.
1206 */
1207
1208 void
1209 pmc_amd_finalize(struct pmc_mdep *md)
1210 {
1211 #if defined(INVARIANTS)
1212 int classindex, i, ncpus, pmcclass;
1213 #endif
1214
1215 pmc_tsc_finalize(md);
1216
1217 KASSERT(amd_pcpu != NULL, ("[amd,%d] NULL per-cpu array pointer",
1218 __LINE__));
1219
1220 #if defined(INVARIANTS)
1221 switch (md->pmd_cputype) {
1222 #if defined(__i386__)
1223 case PMC_CPU_AMD_K7:
1224 classindex = PMC_MDEP_CLASS_INDEX_K7;
1225 pmcclass = PMC_CLASS_K7;
1226 break;
1227 #endif
1228 default:
1229 classindex = PMC_MDEP_CLASS_INDEX_K8;
1230 pmcclass = PMC_CLASS_K8;
1231 }
1232
1233 KASSERT(md->pmd_classdep[classindex].pcd_class == pmcclass,
1234 ("[amd,%d] pmc class mismatch", __LINE__));
1235
1236 ncpus = pmc_cpu_max();
1237
1238 for (i = 0; i < ncpus; i++)
1239 KASSERT(amd_pcpu[i] == NULL, ("[amd,%d] non-null pcpu",
1240 __LINE__));
1241 #endif
1242
1243 free(amd_pcpu, M_PMC);
1244 amd_pcpu = NULL;
1245 }
Cache object: 5885e93bbed0fc5ccffedd17d5e57038
|