1 /*-
2 * Copyright (c) 2003-2005,2008 Joseph Koshy
3 * Copyright (c) 2007 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by A. Joseph Koshy under
7 * sponsorship from the FreeBSD Foundation and Google, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD: releng/7.3/sys/dev/hwpmc/hwpmc_ppro.c 196616 2009-08-28 14:22:01Z jhb $");
33
34 #include <sys/param.h>
35 #include <sys/bus.h>
36 #include <sys/lock.h>
37 #include <sys/mutex.h>
38 #include <sys/pmc.h>
39 #include <sys/pmckern.h>
40 #include <sys/smp.h>
41 #include <sys/systm.h>
42
43 #include <machine/intr_machdep.h>
44 #include <machine/apicvar.h>
45 #include <machine/cpu.h>
46 #include <machine/cpufunc.h>
47 #include <machine/cputypes.h>
48 #include <machine/md_var.h>
49 #include <machine/pmc_mdep.h>
50 #include <machine/specialreg.h>
51
52 /*
53 * PENTIUM PRO SUPPORT
54 *
55 * Quirks:
56 *
57 * - Both PMCs are enabled by a single bit P6_EVSEL_EN in performance
58 * counter ''. This bit needs to be '1' if any of the two
59 * performance counters are in use. Perf counters can also be
60 * switched off by writing zeros to their EVSEL register.
61 *
62 * - While the width of these counters is 40 bits, we do not appear to
63 * have a way of writing 40 bits to the counter MSRs. A WRMSR
64 * instruction will sign extend bit 31 of the value being written to
65 * the perf counter -- a value of 0x80000000 written to an perf
66 * counter register will be sign extended to 0xFF80000000.
67 *
68 * This quirk primarily affects thread-mode PMCs in counting mode, as
69 * these PMCs read and write PMC registers at every context switch.
70 */
71
72 struct p6pmc_descr {
73 struct pmc_descr pm_descr; /* common information */
74 uint32_t pm_pmc_msr;
75 uint32_t pm_evsel_msr;
76 };
77
78 static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = {
79
80 #define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
81 PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
82 PMC_CAP_INVERT | PMC_CAP_QUALIFIER)
83
84 /* PMC 0 */
85 {
86 .pm_descr =
87 {
88 .pd_name ="P6-0",
89 .pd_class = PMC_CLASS_P6,
90 .pd_caps = P6_PMC_CAPS,
91 .pd_width = 40
92 },
93 .pm_pmc_msr = P6_MSR_PERFCTR0,
94 .pm_evsel_msr = P6_MSR_EVSEL0
95 },
96
97 /* PMC 1 */
98 {
99 .pm_descr =
100 {
101 .pd_name ="P6-1",
102 .pd_class = PMC_CLASS_P6,
103 .pd_caps = P6_PMC_CAPS,
104 .pd_width = 40
105 },
106 .pm_pmc_msr = P6_MSR_PERFCTR1,
107 .pm_evsel_msr = P6_MSR_EVSEL1
108 }
109 };
110
111 static enum pmc_cputype p6_cputype;
112
113 /*
114 * P6 Event descriptor
115 *
116 * The 'pm_flags' field has the following structure:
117 * - The upper 4 bits are used to track which counter an event is valid on.
118 * - The lower bits form a bitmask of flags indicating support for the event
119 * on a given CPU.
120 */
121
122 struct p6_event_descr {
123 const enum pmc_event pm_event;
124 uint32_t pm_evsel;
125 uint32_t pm_flags;
126 uint32_t pm_unitmask;
127 };
128
129 #define P6F_CTR(C) (1 << (28 + (C)))
130 #define P6F_CTR0 P6F_CTR(0)
131 #define P6F_CTR1 P6F_CTR(1)
132 #define P6F(CPU) (1 << ((CPU) - PMC_CPU_INTEL_P6))
133 #define _P6F(C) P6F(PMC_CPU_INTEL_##C)
134 #define P6F_P6 _P6F(P6)
135 #define P6F_CL _P6F(CL)
136 #define P6F_PII _P6F(PII)
137 #define P6F_PIII _P6F(PIII)
138 #define P6F_PM _P6F(PM)
139 #define P6F_ALL_CPUS (P6F_P6 | P6F_PII | P6F_CL | P6F_PIII | P6F_PM)
140 #define P6F_ALL_CTRS (P6F_CTR0 | P6F_CTR1)
141 #define P6F_ALL (P6F_ALL_CPUS | P6F_ALL_CTRS)
142
143 #define P6_EVENT_VALID_FOR_CPU(P,CPU) ((P)->pm_flags & P6F(CPU))
144 #define P6_EVENT_VALID_FOR_CTR(P,CTR) ((P)->pm_flags & P6F_CTR(CTR))
145
146 static const struct p6_event_descr p6_events[] = {
147
148 #define P6_EVDESCR(NAME, EVSEL, FLAGS, UMASK) \
149 { \
150 .pm_event = PMC_EV_P6_##NAME, \
151 .pm_evsel = (EVSEL), \
152 .pm_flags = (FLAGS), \
153 .pm_unitmask = (UMASK) \
154 }
155
156 P6_EVDESCR(DATA_MEM_REFS, 0x43, P6F_ALL, 0x00),
157 P6_EVDESCR(DCU_LINES_IN, 0x45, P6F_ALL, 0x00),
158 P6_EVDESCR(DCU_M_LINES_IN, 0x46, P6F_ALL, 0x00),
159 P6_EVDESCR(DCU_M_LINES_OUT, 0x47, P6F_ALL, 0x00),
160 P6_EVDESCR(DCU_MISS_OUTSTANDING, 0x47, P6F_ALL, 0x00),
161 P6_EVDESCR(IFU_FETCH, 0x80, P6F_ALL, 0x00),
162 P6_EVDESCR(IFU_FETCH_MISS, 0x81, P6F_ALL, 0x00),
163 P6_EVDESCR(ITLB_MISS, 0x85, P6F_ALL, 0x00),
164 P6_EVDESCR(IFU_MEM_STALL, 0x86, P6F_ALL, 0x00),
165 P6_EVDESCR(ILD_STALL, 0x87, P6F_ALL, 0x00),
166 P6_EVDESCR(L2_IFETCH, 0x28, P6F_ALL, 0x0F),
167 P6_EVDESCR(L2_LD, 0x29, P6F_ALL, 0x0F),
168 P6_EVDESCR(L2_ST, 0x2A, P6F_ALL, 0x0F),
169 P6_EVDESCR(L2_LINES_IN, 0x24, P6F_ALL, 0x0F),
170 P6_EVDESCR(L2_LINES_OUT, 0x26, P6F_ALL, 0x0F),
171 P6_EVDESCR(L2_M_LINES_INM, 0x25, P6F_ALL, 0x00),
172 P6_EVDESCR(L2_M_LINES_OUTM, 0x27, P6F_ALL, 0x0F),
173 P6_EVDESCR(L2_RQSTS, 0x2E, P6F_ALL, 0x0F),
174 P6_EVDESCR(L2_ADS, 0x21, P6F_ALL, 0x00),
175 P6_EVDESCR(L2_DBUS_BUSY, 0x22, P6F_ALL, 0x00),
176 P6_EVDESCR(L2_DBUS_BUSY_RD, 0x23, P6F_ALL, 0x00),
177 P6_EVDESCR(BUS_DRDY_CLOCKS, 0x62, P6F_ALL, 0x20),
178 P6_EVDESCR(BUS_LOCK_CLOCKS, 0x63, P6F_ALL, 0x20),
179 P6_EVDESCR(BUS_REQ_OUTSTANDING, 0x60, P6F_ALL, 0x00),
180 P6_EVDESCR(BUS_TRAN_BRD, 0x65, P6F_ALL, 0x20),
181 P6_EVDESCR(BUS_TRAN_RFO, 0x66, P6F_ALL, 0x20),
182 P6_EVDESCR(BUS_TRANS_WB, 0x67, P6F_ALL, 0x20),
183 P6_EVDESCR(BUS_TRAN_IFETCH, 0x68, P6F_ALL, 0x20),
184 P6_EVDESCR(BUS_TRAN_INVAL, 0x69, P6F_ALL, 0x20),
185 P6_EVDESCR(BUS_TRAN_PWR, 0x6A, P6F_ALL, 0x20),
186 P6_EVDESCR(BUS_TRANS_P, 0x6B, P6F_ALL, 0x20),
187 P6_EVDESCR(BUS_TRANS_IO, 0x6C, P6F_ALL, 0x20),
188 P6_EVDESCR(BUS_TRAN_DEF, 0x6D, P6F_ALL, 0x20),
189 P6_EVDESCR(BUS_TRAN_BURST, 0x6E, P6F_ALL, 0x20),
190 P6_EVDESCR(BUS_TRAN_ANY, 0x70, P6F_ALL, 0x20),
191 P6_EVDESCR(BUS_TRAN_MEM, 0x6F, P6F_ALL, 0x20),
192 P6_EVDESCR(BUS_DATA_RCV, 0x64, P6F_ALL, 0x00),
193 P6_EVDESCR(BUS_BNR_DRV, 0x61, P6F_ALL, 0x00),
194 P6_EVDESCR(BUS_HIT_DRV, 0x7A, P6F_ALL, 0x00),
195 P6_EVDESCR(BUS_HITM_DRV, 0x7B, P6F_ALL, 0x00),
196 P6_EVDESCR(BUS_SNOOP_STALL, 0x7E, P6F_ALL, 0x00),
197 P6_EVDESCR(FLOPS, 0xC1, P6F_ALL_CPUS | P6F_CTR0, 0x00),
198 P6_EVDESCR(FP_COMPS_OPS_EXE, 0x10, P6F_ALL_CPUS | P6F_CTR0, 0x00),
199 P6_EVDESCR(FP_ASSIST, 0x11, P6F_ALL_CPUS | P6F_CTR1, 0x00),
200 P6_EVDESCR(MUL, 0x12, P6F_ALL_CPUS | P6F_CTR1, 0x00),
201 P6_EVDESCR(DIV, 0x13, P6F_ALL_CPUS | P6F_CTR1, 0x00),
202 P6_EVDESCR(CYCLES_DIV_BUSY, 0x14, P6F_ALL_CPUS | P6F_CTR0, 0x00),
203 P6_EVDESCR(LD_BLOCKS, 0x03, P6F_ALL, 0x00),
204 P6_EVDESCR(SB_DRAINS, 0x04, P6F_ALL, 0x00),
205 P6_EVDESCR(MISALIGN_MEM_REF, 0x05, P6F_ALL, 0x00),
206 P6_EVDESCR(EMON_KNI_PREF_DISPATCHED, 0x07, P6F_PIII | P6F_ALL_CTRS, 0x03),
207 P6_EVDESCR(EMON_KNI_PREF_MISS, 0x4B, P6F_PIII | P6F_ALL_CTRS, 0x03),
208 P6_EVDESCR(INST_RETIRED, 0xC0, P6F_ALL, 0x00),
209 P6_EVDESCR(UOPS_RETIRED, 0xC2, P6F_ALL, 0x00),
210 P6_EVDESCR(INST_DECODED, 0xD0, P6F_ALL, 0x00),
211 P6_EVDESCR(EMON_KNI_INST_RETIRED, 0xD8, P6F_PIII | P6F_ALL_CTRS, 0x01),
212 P6_EVDESCR(EMON_KNI_COMP_INST_RET, 0xD9, P6F_PIII | P6F_ALL_CTRS, 0x01),
213 P6_EVDESCR(HW_INT_RX, 0xC8, P6F_ALL, 0x00),
214 P6_EVDESCR(CYCLES_INT_MASKED, 0xC6, P6F_ALL, 0x00),
215 P6_EVDESCR(CYCLES_INT_PENDING_AND_MASKED, 0xC7, P6F_ALL, 0x00),
216 P6_EVDESCR(BR_INST_RETIRED, 0xC4, P6F_ALL, 0x00),
217 P6_EVDESCR(BR_MISS_PRED_RETIRED, 0xC5, P6F_ALL, 0x00),
218 P6_EVDESCR(BR_TAKEN_RETIRED, 0xC9, P6F_ALL, 0x00),
219 P6_EVDESCR(BR_MISS_PRED_TAKEN_RET, 0xCA, P6F_ALL, 0x00),
220 P6_EVDESCR(BR_INST_DECODED, 0xE0, P6F_ALL, 0x00),
221 P6_EVDESCR(BTB_MISSES, 0xE2, P6F_ALL, 0x00),
222 P6_EVDESCR(BR_BOGUS, 0xE4, P6F_ALL, 0x00),
223 P6_EVDESCR(BACLEARS, 0xE6, P6F_ALL, 0x00),
224 P6_EVDESCR(RESOURCE_STALLS, 0xA2, P6F_ALL, 0x00),
225 P6_EVDESCR(PARTIAL_RAT_STALLS, 0xD2, P6F_ALL, 0x00),
226 P6_EVDESCR(SEGMENT_REG_LOADS, 0x06, P6F_ALL, 0x00),
227 P6_EVDESCR(CPU_CLK_UNHALTED, 0x79, P6F_ALL, 0x00),
228 P6_EVDESCR(MMX_INSTR_EXEC, 0xB0,
229 P6F_ALL_CTRS | P6F_CL | P6F_PII, 0x00),
230 P6_EVDESCR(MMX_SAT_INSTR_EXEC, 0xB1,
231 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
232 P6_EVDESCR(MMX_UOPS_EXEC, 0xB2,
233 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
234 P6_EVDESCR(MMX_INSTR_TYPE_EXEC, 0xB3,
235 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x3F),
236 P6_EVDESCR(FP_MMX_TRANS, 0xCC,
237 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x01),
238 P6_EVDESCR(MMX_ASSIST, 0xCD,
239 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
240 P6_EVDESCR(MMX_INSTR_RET, 0xCE, P6F_ALL_CTRS | P6F_PII, 0x00),
241 P6_EVDESCR(SEG_RENAME_STALLS, 0xD4,
242 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
243 P6_EVDESCR(SEG_REG_RENAMES, 0xD5,
244 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
245 P6_EVDESCR(RET_SEG_RENAMES, 0xD6,
246 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
247 P6_EVDESCR(EMON_EST_TRANS, 0x58, P6F_ALL_CTRS | P6F_PM, 0x02),
248 P6_EVDESCR(EMON_THERMAL_TRIP, 0x59, P6F_ALL_CTRS | P6F_PM, 0x00),
249 P6_EVDESCR(BR_INST_EXEC, 0x88, P6F_ALL_CTRS | P6F_PM, 0x00),
250 P6_EVDESCR(BR_MISSP_EXEC, 0x89, P6F_ALL_CTRS | P6F_PM, 0x00),
251 P6_EVDESCR(BR_BAC_MISSP_EXEC, 0x8A, P6F_ALL_CTRS | P6F_PM, 0x00),
252 P6_EVDESCR(BR_CND_EXEC, 0x8B, P6F_ALL_CTRS | P6F_PM, 0x00),
253 P6_EVDESCR(BR_CND_MISSP_EXEC, 0x8C, P6F_ALL_CTRS | P6F_PM, 0x00),
254 P6_EVDESCR(BR_IND_EXEC, 0x8D, P6F_ALL_CTRS | P6F_PM, 0x00),
255 P6_EVDESCR(BR_IND_MISSP_EXEC, 0x8E, P6F_ALL_CTRS | P6F_PM, 0x00),
256 P6_EVDESCR(BR_RET_EXEC, 0x8F, P6F_ALL_CTRS | P6F_PM, 0x00),
257 P6_EVDESCR(BR_RET_MISSP_EXEC, 0x90, P6F_ALL_CTRS | P6F_PM, 0x00),
258 P6_EVDESCR(BR_RET_BAC_MISSP_EXEC, 0x91, P6F_ALL_CTRS | P6F_PM, 0x00),
259 P6_EVDESCR(BR_CALL_EXEC, 0x92, P6F_ALL_CTRS | P6F_PM, 0x00),
260 P6_EVDESCR(BR_CALL_MISSP_EXEC, 0x93, P6F_ALL_CTRS | P6F_PM, 0x00),
261 P6_EVDESCR(BR_IND_CALL_EXEC, 0x94, P6F_ALL_CTRS | P6F_PM, 0x00),
262 P6_EVDESCR(EMON_SIMD_INSTR_RETIRED, 0xCE, P6F_ALL_CTRS | P6F_PM, 0x00),
263 P6_EVDESCR(EMON_SYNCH_UOPS, 0xD3, P6F_ALL_CTRS | P6F_PM, 0x00),
264 P6_EVDESCR(EMON_ESP_UOPS, 0xD7, P6F_ALL_CTRS | P6F_PM, 0x00),
265 P6_EVDESCR(EMON_FUSED_UOPS_RET, 0xDA, P6F_ALL_CTRS | P6F_PM, 0x03),
266 P6_EVDESCR(EMON_UNFUSION, 0xDB, P6F_ALL_CTRS | P6F_PM, 0x00),
267 P6_EVDESCR(EMON_PREF_RQSTS_UP, 0xF0, P6F_ALL_CTRS | P6F_PM, 0x00),
268 P6_EVDESCR(EMON_PREF_RQSTS_DN, 0xD8, P6F_ALL_CTRS | P6F_PM, 0x00),
269 P6_EVDESCR(EMON_SSE_SSE2_INST_RETIRED, 0xD8, P6F_ALL_CTRS | P6F_PM, 0x03),
270 P6_EVDESCR(EMON_SSE_SSE2_COMP_INST_RETIRED, 0xD9, P6F_ALL_CTRS | P6F_PM, 0x03)
271
272 #undef P6_EVDESCR
273 };
274
275 #define P6_NEVENTS (PMC_EV_P6_LAST - PMC_EV_P6_FIRST + 1)
276
277 static const struct p6_event_descr *
278 p6_find_event(enum pmc_event ev)
279 {
280 int n;
281
282 for (n = 0; n < P6_NEVENTS; n++)
283 if (p6_events[n].pm_event == ev)
284 break;
285 if (n == P6_NEVENTS)
286 return NULL;
287 return &p6_events[n];
288 }
289
290 /*
291 * Per-CPU data structure for P6 class CPUs
292 *
293 * [common stuff]
294 * [flags for maintaining PMC start/stop state]
295 * [3 struct pmc_hw pointers]
296 * [3 struct pmc_hw structures]
297 */
298
299 struct p6_cpu {
300 struct pmc_hw pc_p6pmcs[P6_NPMCS];
301 uint32_t pc_state;
302 };
303
304 static struct p6_cpu **p6_pcpu;
305
306 /*
307 * If CTR1 is active, we need to keep the 'EN' bit if CTR0 set,
308 * with the rest of CTR0 being zero'ed out.
309 */
310 #define P6_SYNC_CTR_STATE(PC) do { \
311 uint32_t _config, _enable; \
312 _enable = 0; \
313 if ((PC)->pc_state & 0x02) \
314 _enable |= P6_EVSEL_EN; \
315 if ((PC)->pc_state & 0x01) \
316 _config = rdmsr(P6_MSR_EVSEL0) | \
317 P6_EVSEL_EN; \
318 else \
319 _config = 0; \
320 wrmsr(P6_MSR_EVSEL0, _config | _enable); \
321 } while (0)
322
323 #define P6_MARK_STARTED(PC,RI) do { \
324 (PC)->pc_state |= (1 << ((RI)-1)); \
325 } while (0)
326
327 #define P6_MARK_STOPPED(PC,RI) do { \
328 (PC)->pc_state &= ~(1<< ((RI)-1)); \
329 } while (0)
330
331 static int
332 p6_pcpu_init(struct pmc_mdep *md, int cpu)
333 {
334 int first_ri, n;
335 struct p6_cpu *p6c;
336 struct pmc_cpu *pc;
337 struct pmc_hw *phw;
338
339 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
340 ("[p6,%d] bad cpu %d", __LINE__, cpu));
341
342 PMCDBG(MDP,INI,0,"p6-init cpu=%d", cpu);
343
344 p6c = malloc(sizeof (struct p6_cpu), M_PMC, M_WAITOK|M_ZERO);
345 pc = pmc_pcpu[cpu];
346
347 KASSERT(pc != NULL, ("[p6,%d] cpu %d null per-cpu", __LINE__, cpu));
348
349 phw = p6c->pc_p6pmcs;
350 p6_pcpu[cpu] = p6c;
351
352 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
353
354 for (n = 0; n < P6_NPMCS; n++, phw++) {
355 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
356 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
357 phw->phw_pmc = NULL;
358 pc->pc_hwpmcs[n + first_ri] = phw;
359 }
360
361 return (0);
362 }
363
364 static int
365 p6_pcpu_fini(struct pmc_mdep *md, int cpu)
366 {
367 int first_ri, n;
368 struct p6_cpu *p6c;
369 struct pmc_cpu *pc;
370
371 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
372 ("[p6,%d] bad cpu %d", __LINE__, cpu));
373
374 PMCDBG(MDP,INI,0,"p6-cleanup cpu=%d", cpu);
375
376 p6c = p6_pcpu[cpu];
377 p6_pcpu[cpu] = NULL;
378
379 KASSERT(p6c != NULL, ("[p6,%d] null pcpu", __LINE__));
380
381 free(p6c, M_PMC);
382
383 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
384 pc = pmc_pcpu[cpu];
385 for (n = 0; n < P6_NPMCS; n++)
386 pc->pc_hwpmcs[n + first_ri] = NULL;
387
388 return (0);
389 }
390
391 static int
392 p6_read_pmc(int cpu, int ri, pmc_value_t *v)
393 {
394 struct pmc *pm;
395 struct p6pmc_descr *pd;
396 pmc_value_t tmp;
397
398 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
399 ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
400 KASSERT(ri >= 0 && ri < P6_NPMCS,
401 ("[p6,%d] illegal row-index %d", __LINE__, ri));
402
403 pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
404 pd = &p6_pmcdesc[ri];
405
406 KASSERT(pm,
407 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
408
409 tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK;
410 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
411 *v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
412 else
413 *v = tmp;
414
415 PMCDBG(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
416 pd->pm_pmc_msr, *v);
417
418 return (0);
419 }
420
421 static int
422 p6_write_pmc(int cpu, int ri, pmc_value_t v)
423 {
424 struct pmc *pm;
425 struct p6pmc_descr *pd;
426
427 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
428 ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
429 KASSERT(ri >= 0 && ri < P6_NPMCS,
430 ("[p6,%d] illegal row-index %d", __LINE__, ri));
431
432 pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
433 pd = &p6_pmcdesc[ri];
434
435 KASSERT(pm,
436 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
437
438 PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
439 pd->pm_pmc_msr, v);
440
441 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
442 v = P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
443
444 wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK);
445
446 return (0);
447 }
448
449 static int
450 p6_config_pmc(int cpu, int ri, struct pmc *pm)
451 {
452 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
453 ("[p6,%d] illegal CPU %d", __LINE__, cpu));
454
455 KASSERT(ri >= 0 && ri < P6_NPMCS,
456 ("[p6,%d] illegal row-index %d", __LINE__, ri));
457
458 PMCDBG(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
459
460 KASSERT(p6_pcpu[cpu] != NULL, ("[p6,%d] null per-cpu %d", __LINE__,
461 cpu));
462
463 p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc = pm;
464
465 return (0);
466 }
467
468 /*
469 * Retrieve a configured PMC pointer from hardware state.
470 */
471
472 static int
473 p6_get_config(int cpu, int ri, struct pmc **ppm)
474 {
475
476 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
477 ("[p6,%d] illegal CPU %d", __LINE__, cpu));
478 KASSERT(ri >= 0 && ri < P6_NPMCS,
479 ("[p6,%d] illegal row-index %d", __LINE__, ri));
480
481 *ppm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
482
483 return (0);
484 }
485
486
487 /*
488 * A pmc may be allocated to a given row index if:
489 * - the event is valid for this CPU
490 * - the event is valid for this counter index
491 */
492
493 static int
494 p6_allocate_pmc(int cpu, int ri, struct pmc *pm,
495 const struct pmc_op_pmcallocate *a)
496 {
497 uint32_t allowed_unitmask, caps, config, unitmask;
498 const struct p6pmc_descr *pd;
499 const struct p6_event_descr *pevent;
500 enum pmc_event ev;
501
502 (void) cpu;
503
504 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
505 ("[p6,%d] illegal CPU %d", __LINE__, cpu));
506 KASSERT(ri >= 0 && ri < P6_NPMCS,
507 ("[p6,%d] illegal row-index value %d", __LINE__, ri));
508
509 pd = &p6_pmcdesc[ri];
510
511 PMCDBG(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x "
512 "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
513 pm->pm_caps);
514
515 /* check class */
516 if (pd->pm_descr.pd_class != a->pm_class)
517 return (EINVAL);
518
519 /* check requested capabilities */
520 caps = a->pm_caps;
521 if ((pd->pm_descr.pd_caps & caps) != caps)
522 return (EPERM);
523
524 ev = pm->pm_event;
525
526 if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST)
527 return (EINVAL);
528
529 if ((pevent = p6_find_event(ev)) == NULL)
530 return (ESRCH);
531
532 if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) ||
533 !P6_EVENT_VALID_FOR_CTR(pevent, (ri-1)))
534 return (EINVAL);
535
536 /* For certain events, Pentium M differs from the stock P6 */
537 allowed_unitmask = 0;
538 if (p6_cputype == PMC_CPU_INTEL_PM) {
539 if (ev == PMC_EV_P6_L2_LD || ev == PMC_EV_P6_L2_LINES_IN ||
540 ev == PMC_EV_P6_L2_LINES_OUT)
541 allowed_unitmask = P6_EVSEL_TO_UMASK(0x3F);
542 else if (ev == PMC_EV_P6_L2_M_LINES_OUTM)
543 allowed_unitmask = P6_EVSEL_TO_UMASK(0x30);
544 } else
545 allowed_unitmask = P6_EVSEL_TO_UMASK(pevent->pm_unitmask);
546
547 unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK;
548 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
549 return (EINVAL);
550
551 if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */
552 unitmask = P6_EVSEL_TO_UMASK(0x0F);
553
554 config = 0;
555
556 config |= P6_EVSEL_EVENT_SELECT(pevent->pm_evsel);
557
558 if (unitmask & (caps & PMC_CAP_QUALIFIER))
559 config |= unitmask;
560
561 if (caps & PMC_CAP_THRESHOLD)
562 config |= a->pm_md.pm_ppro.pm_ppro_config &
563 P6_EVSEL_CMASK_MASK;
564
565 /* set at least one of the 'usr' or 'os' caps */
566 if (caps & PMC_CAP_USER)
567 config |= P6_EVSEL_USR;
568 if (caps & PMC_CAP_SYSTEM)
569 config |= P6_EVSEL_OS;
570 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0)
571 config |= (P6_EVSEL_USR|P6_EVSEL_OS);
572
573 if (caps & PMC_CAP_EDGE)
574 config |= P6_EVSEL_E;
575 if (caps & PMC_CAP_INVERT)
576 config |= P6_EVSEL_INV;
577 if (caps & PMC_CAP_INTERRUPT)
578 config |= P6_EVSEL_INT;
579
580 pm->pm_md.pm_ppro.pm_ppro_evsel = config;
581
582 PMCDBG(MDP,ALL,2, "p6-allocate config=0x%x", config);
583
584 return (0);
585 }
586
587 static int
588 p6_release_pmc(int cpu, int ri, struct pmc *pm)
589 {
590 (void) pm;
591
592 PMCDBG(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm);
593
594 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
595 ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
596 KASSERT(ri >= 0 && ri < P6_NPMCS,
597 ("[p6,%d] illegal row-index %d", __LINE__, ri));
598
599 KASSERT(p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc == NULL,
600 ("[p6,%d] PHW pmc non-NULL", __LINE__));
601
602 return (0);
603 }
604
605 static int
606 p6_start_pmc(int cpu, int ri)
607 {
608 uint32_t config;
609 struct pmc *pm;
610 struct p6_cpu *pc;
611 const struct p6pmc_descr *pd;
612
613 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
614 ("[p6,%d] illegal CPU value %d", __LINE__, cpu));
615 KASSERT(ri >= 0 && ri < P6_NPMCS,
616 ("[p6,%d] illegal row-index %d", __LINE__, ri));
617
618 pc = p6_pcpu[cpu];
619 pm = pc->pc_p6pmcs[ri].phw_pmc;
620 pd = &p6_pmcdesc[ri];
621
622 KASSERT(pm,
623 ("[p6,%d] starting cpu%d,ri%d with no pmc configured",
624 __LINE__, cpu, ri));
625
626 PMCDBG(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri);
627
628 config = pm->pm_md.pm_ppro.pm_ppro_evsel;
629
630 PMCDBG(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x",
631 cpu, ri, pd->pm_evsel_msr, config);
632
633 P6_MARK_STARTED(pc, ri);
634 wrmsr(pd->pm_evsel_msr, config);
635
636 P6_SYNC_CTR_STATE(pc);
637
638 return (0);
639 }
640
641 static int
642 p6_stop_pmc(int cpu, int ri)
643 {
644 struct pmc *pm;
645 struct p6_cpu *pc;
646 struct p6pmc_descr *pd;
647
648 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
649 ("[p6,%d] illegal cpu value %d", __LINE__, cpu));
650 KASSERT(ri >= 0 && ri < P6_NPMCS,
651 ("[p6,%d] illegal row index %d", __LINE__, ri));
652
653 pc = p6_pcpu[cpu];
654 pm = pc->pc_p6pmcs[ri].phw_pmc;
655 pd = &p6_pmcdesc[ri];
656
657 KASSERT(pm,
658 ("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
659 cpu, ri));
660
661 PMCDBG(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri);
662
663 wrmsr(pd->pm_evsel_msr, 0); /* stop hw */
664 P6_MARK_STOPPED(pc, ri); /* update software state */
665
666 P6_SYNC_CTR_STATE(pc); /* restart CTR1 if need be */
667
668 PMCDBG(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri);
669
670 return (0);
671 }
672
673 static int
674 p6_intr(int cpu, struct trapframe *tf)
675 {
676 int error, retval, ri;
677 uint32_t perf0cfg;
678 struct pmc *pm;
679 struct p6_cpu *pc;
680 pmc_value_t v;
681
682 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
683 ("[p6,%d] CPU %d out of range", __LINE__, cpu));
684
685 retval = 0;
686 pc = p6_pcpu[cpu];
687
688 /* stop both PMCs */
689 perf0cfg = rdmsr(P6_MSR_EVSEL0);
690 wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN);
691
692 for (ri = 0; ri < P6_NPMCS; ri++) {
693
694 if ((pm = pc->pc_p6pmcs[ri].phw_pmc) == NULL ||
695 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
696 continue;
697 }
698
699 if (!P6_PMC_HAS_OVERFLOWED(ri))
700 continue;
701
702 retval = 1;
703
704 if (pm->pm_state != PMC_STATE_RUNNING)
705 continue;
706
707 error = pmc_process_interrupt(cpu, pm, tf,
708 TRAPF_USERMODE(tf));
709 if (error)
710 P6_MARK_STOPPED(pc,ri);
711
712 /* reload sampling count */
713 v = pm->pm_sc.pm_reloadcount;
714 wrmsr(P6_MSR_PERFCTR0 + ri,
715 P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
716
717 }
718
719 /*
720 * On P6 processors, the LAPIC needs to have its PMC interrupt
721 * unmasked after a PMC interrupt.
722 */
723 if (retval)
724 lapic_reenable_pmc();
725
726 atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
727 &pmc_stats.pm_intr_ignored, 1);
728
729 /* restart counters that can be restarted */
730 P6_SYNC_CTR_STATE(pc);
731
732 return (retval);
733 }
734
735 static int
736 p6_describe(int cpu, int ri, struct pmc_info *pi,
737 struct pmc **ppmc)
738 {
739 int error;
740 size_t copied;
741 struct pmc_hw *phw;
742 struct p6pmc_descr *pd;
743
744 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
745 ("[p6,%d] illegal CPU %d", __LINE__, cpu));
746 KASSERT(ri >= 0 && ri < P6_NPMCS,
747 ("[p6,%d] row-index %d out of range", __LINE__, ri));
748
749 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
750 pd = &p6_pmcdesc[ri];
751
752 KASSERT(phw == &p6_pcpu[cpu]->pc_p6pmcs[ri],
753 ("[p6,%d] phw mismatch", __LINE__));
754
755 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
756 PMC_NAME_MAX, &copied)) != 0)
757 return (error);
758
759 pi->pm_class = pd->pm_descr.pd_class;
760
761 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
762 pi->pm_enabled = TRUE;
763 *ppmc = phw->phw_pmc;
764 } else {
765 pi->pm_enabled = FALSE;
766 *ppmc = NULL;
767 }
768
769 return (0);
770 }
771
772 static int
773 p6_get_msr(int ri, uint32_t *msr)
774 {
775 KASSERT(ri >= 0 && ri < P6_NPMCS,
776 ("[p6,%d ri %d out of range", __LINE__, ri));
777
778 *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0;
779
780 return (0);
781 }
782
783 int
784 pmc_p6_initialize(struct pmc_mdep *md, int ncpus)
785 {
786 struct pmc_classdep *pcd;
787
788 KASSERT(cpu_vendor_id == CPU_VENDOR_INTEL,
789 ("[p6,%d] Initializing non-intel processor", __LINE__));
790
791 PMCDBG(MDP,INI,1, "%s", "p6-initialize");
792
793 /* Allocate space for pointers to per-cpu descriptors. */
794 p6_pcpu = malloc(sizeof(struct p6_cpu **) * ncpus, M_PMC,
795 M_ZERO|M_WAITOK);
796
797 /* Fill in the class dependent descriptor. */
798 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6];
799
800 switch (md->pmd_cputype) {
801
802 /*
803 * P6 Family Processors
804 */
805 case PMC_CPU_INTEL_P6:
806 case PMC_CPU_INTEL_CL:
807 case PMC_CPU_INTEL_PII:
808 case PMC_CPU_INTEL_PIII:
809 case PMC_CPU_INTEL_PM:
810
811 p6_cputype = md->pmd_cputype;
812
813 pcd->pcd_caps = P6_PMC_CAPS;
814 pcd->pcd_class = PMC_CLASS_P6;
815 pcd->pcd_num = P6_NPMCS;
816 pcd->pcd_ri = md->pmd_npmc;
817 pcd->pcd_width = 40;
818
819 pcd->pcd_allocate_pmc = p6_allocate_pmc;
820 pcd->pcd_config_pmc = p6_config_pmc;
821 pcd->pcd_describe = p6_describe;
822 pcd->pcd_get_config = p6_get_config;
823 pcd->pcd_get_msr = p6_get_msr;
824 pcd->pcd_pcpu_fini = p6_pcpu_fini;
825 pcd->pcd_pcpu_init = p6_pcpu_init;
826 pcd->pcd_read_pmc = p6_read_pmc;
827 pcd->pcd_release_pmc = p6_release_pmc;
828 pcd->pcd_start_pmc = p6_start_pmc;
829 pcd->pcd_stop_pmc = p6_stop_pmc;
830 pcd->pcd_write_pmc = p6_write_pmc;
831
832 md->pmd_pcpu_fini = NULL;
833 md->pmd_pcpu_init = NULL;
834 md->pmd_intr = p6_intr;
835
836 md->pmd_npmc += P6_NPMCS;
837
838 break;
839
840 default:
841 KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__));
842 return ENOSYS;
843 }
844
845 return (0);
846 }
847
848 void
849 pmc_p6_finalize(struct pmc_mdep *md)
850 {
851 #if defined(INVARIANTS)
852 int i, ncpus;
853 #endif
854
855 KASSERT(p6_pcpu != NULL, ("[p6,%d] NULL p6_pcpu", __LINE__));
856
857 #if defined(INVARIANTS)
858 ncpus = pmc_cpu_max();
859 for (i = 0; i < ncpus; i++)
860 KASSERT(p6_pcpu[i] == NULL, ("[p6,%d] non-null pcpu %d",
861 __LINE__, i));
862 #endif
863
864 free(p6_pcpu, M_PMC);
865 p6_pcpu = NULL;
866 }
Cache object: d11c7ae087dae4692fcf91790952b2a8
|