FreeBSD/Linux Kernel Cross Reference
sys/arm/arm/pl310.c
1 /*-
2 * Copyright (c) 2012 Olivier Houchard <cognet@FreeBSD.org>
3 * Copyright (c) 2011
4 * Ben Gray <ben.r.gray@gmail.com>.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the company nor the name of the author may be used to
16 * endorse or promote products derived from this software without specific
17 * prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
25 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
27 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
28 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD: releng/10.2/sys/arm/arm/pl310.c 273695 2014-10-26 16:09:59Z ian $");
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/kernel.h>
37 #include <sys/rman.h>
38 #include <sys/module.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <machine/intr.h>
42
43 #include <machine/bus.h>
44 #include <machine/pl310.h>
45
46 #include <dev/fdt/fdt_common.h>
47 #include <dev/ofw/openfirm.h>
48 #include <dev/ofw/ofw_bus.h>
49 #include <dev/ofw/ofw_bus_subr.h>
50
51 /*
52 * Define this if you need to disable PL310 for debugging purpose
53 * Spec:
54 * http://infocenter.arm.com/help/topic/com.arm.doc.ddi0246e/DDI0246E_l2c310_r3p1_trm.pdf
55 */
56
57 /*
58 * Hardcode errata for now
59 * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0246b/pr01s02s02.html
60 */
61 #define PL310_ERRATA_588369
62 #define PL310_ERRATA_753970
63 #define PL310_ERRATA_727915
64
65 #define PL310_LOCK(sc) do { \
66 mtx_lock_spin(&(sc)->sc_mtx); \
67 } while(0);
68
69 #define PL310_UNLOCK(sc) do { \
70 mtx_unlock_spin(&(sc)->sc_mtx); \
71 } while(0);
72
73 static int pl310_enabled = 1;
74 TUNABLE_INT("hw.pl310.enabled", &pl310_enabled);
75
76 static uint32_t g_l2cache_way_mask;
77
78 static const uint32_t g_l2cache_line_size = 32;
79 static const uint32_t g_l2cache_align_mask = (32 - 1);
80
81 static uint32_t g_l2cache_size;
82 static uint32_t g_way_size;
83 static uint32_t g_ways_assoc;
84
85 static struct pl310_softc *pl310_softc;
86
87 static struct ofw_compat_data compat_data[] = {
88 {"arm,pl310", true}, /* Non-standard, FreeBSD. */
89 {"arm,pl310-cache", true},
90 {NULL, false}
91 };
92
93 void
94 pl310_print_config(struct pl310_softc *sc)
95 {
96 uint32_t aux, prefetch;
97 const char *dis = "disabled";
98 const char *ena = "enabled";
99
100 aux = pl310_read4(sc, PL310_AUX_CTRL);
101 prefetch = pl310_read4(sc, PL310_PREFETCH_CTRL);
102
103 device_printf(sc->sc_dev, "Early BRESP response: %s\n",
104 (aux & AUX_CTRL_EARLY_BRESP) ? ena : dis);
105 device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
106 (aux & AUX_CTRL_INSTR_PREFETCH) ? ena : dis);
107 device_printf(sc->sc_dev, "Data prefetch: %s\n",
108 (aux & AUX_CTRL_DATA_PREFETCH) ? ena : dis);
109 device_printf(sc->sc_dev, "Non-secure interrupt control: %s\n",
110 (aux & AUX_CTRL_NS_INT_CTRL) ? ena : dis);
111 device_printf(sc->sc_dev, "Non-secure lockdown: %s\n",
112 (aux & AUX_CTRL_NS_LOCKDOWN) ? ena : dis);
113 device_printf(sc->sc_dev, "Share override: %s\n",
114 (aux & AUX_CTRL_SHARE_OVERRIDE) ? ena : dis);
115
116 device_printf(sc->sc_dev, "Double linefill: %s\n",
117 (prefetch & PREFETCH_CTRL_DL) ? ena : dis);
118 device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
119 (prefetch & PREFETCH_CTRL_INSTR_PREFETCH) ? ena : dis);
120 device_printf(sc->sc_dev, "Data prefetch: %s\n",
121 (prefetch & PREFETCH_CTRL_DATA_PREFETCH) ? ena : dis);
122 device_printf(sc->sc_dev, "Double linefill on WRAP request: %s\n",
123 (prefetch & PREFETCH_CTRL_DL_ON_WRAP) ? ena : dis);
124 device_printf(sc->sc_dev, "Prefetch drop: %s\n",
125 (prefetch & PREFETCH_CTRL_PREFETCH_DROP) ? ena : dis);
126 device_printf(sc->sc_dev, "Incr double Linefill: %s\n",
127 (prefetch & PREFETCH_CTRL_INCR_DL) ? ena : dis);
128 device_printf(sc->sc_dev, "Not same ID on exclusive sequence: %s\n",
129 (prefetch & PREFETCH_CTRL_NOTSAMEID) ? ena : dis);
130 device_printf(sc->sc_dev, "Prefetch offset: %d\n",
131 (prefetch & PREFETCH_CTRL_OFFSET_MASK));
132 }
133
134 void
135 pl310_set_ram_latency(struct pl310_softc *sc, uint32_t which_reg,
136 uint32_t read, uint32_t write, uint32_t setup)
137 {
138 uint32_t v;
139
140 KASSERT(which_reg == PL310_TAG_RAM_CTRL ||
141 which_reg == PL310_DATA_RAM_CTRL,
142 ("bad pl310 ram latency register address"));
143
144 v = pl310_read4(sc, which_reg);
145 if (setup != 0) {
146 KASSERT(setup <= 8, ("bad pl310 setup latency: %d", setup));
147 v &= ~RAM_CTRL_SETUP_MASK;
148 v |= (setup - 1) << RAM_CTRL_SETUP_SHIFT;
149 }
150 if (read != 0) {
151 KASSERT(read <= 8, ("bad pl310 read latency: %d", read));
152 v &= ~RAM_CTRL_READ_MASK;
153 v |= (read - 1) << RAM_CTRL_READ_SHIFT;
154 }
155 if (write != 0) {
156 KASSERT(write <= 8, ("bad pl310 write latency: %d", write));
157 v &= ~RAM_CTRL_WRITE_MASK;
158 v |= (write - 1) << RAM_CTRL_WRITE_SHIFT;
159 }
160 pl310_write4(sc, which_reg, v);
161 }
162
163 static int
164 pl310_filter(void *arg)
165 {
166 struct pl310_softc *sc = arg;
167 uint32_t intr;
168
169 intr = pl310_read4(sc, PL310_INTR_MASK);
170
171 if (!sc->sc_enabled && (intr & INTR_MASK_ECNTR)) {
172 /*
173 * This is for debug purpose, so be blunt about it
174 * We disable PL310 only when something fishy is going
175 * on and we need to make sure L2 cache is 100% disabled
176 */
177 panic("pl310: caches disabled but cache event detected\n");
178 }
179
180 return (FILTER_HANDLED);
181 }
182
183 static __inline void
184 pl310_wait_background_op(uint32_t off, uint32_t mask)
185 {
186
187 while (pl310_read4(pl310_softc, off) & mask)
188 continue;
189 }
190
191
192 /**
193 * pl310_cache_sync - performs a cache sync operation
194 *
195 * According to the TRM:
196 *
197 * "Before writing to any other register you must perform an explicit
198 * Cache Sync operation. This is particularly important when the cache is
199 * enabled and changes to how the cache allocates new lines are to be made."
200 *
201 *
202 */
203 static __inline void
204 pl310_cache_sync(void)
205 {
206
207 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
208 return;
209
210 #ifdef PL310_ERRATA_753970
211 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
212 /* Write uncached PL310 register */
213 pl310_write4(pl310_softc, 0x740, 0xffffffff);
214 else
215 #endif
216 pl310_write4(pl310_softc, PL310_CACHE_SYNC, 0xffffffff);
217 }
218
219
220 static void
221 pl310_wbinv_all(void)
222 {
223
224 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
225 return;
226
227 PL310_LOCK(pl310_softc);
228 #ifdef PL310_ERRATA_727915
229 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r2p0) {
230 int i, j;
231
232 for (i = 0; i < g_ways_assoc; i++) {
233 for (j = 0; j < g_way_size / g_l2cache_line_size; j++) {
234 pl310_write4(pl310_softc,
235 PL310_CLEAN_INV_LINE_IDX,
236 (i << 28 | j << 5));
237 }
238 }
239 pl310_cache_sync();
240 PL310_UNLOCK(pl310_softc);
241 return;
242
243 }
244 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
245 platform_pl310_write_debug(pl310_softc, 3);
246 #endif
247 pl310_write4(pl310_softc, PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
248 pl310_wait_background_op(PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
249 pl310_cache_sync();
250 #ifdef PL310_ERRATA_727915
251 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
252 platform_pl310_write_debug(pl310_softc, 0);
253 #endif
254 PL310_UNLOCK(pl310_softc);
255 }
256
257 static void
258 pl310_wbinv_range(vm_paddr_t start, vm_size_t size)
259 {
260
261 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
262 return;
263
264 PL310_LOCK(pl310_softc);
265 if (start & g_l2cache_align_mask) {
266 size += start & g_l2cache_align_mask;
267 start &= ~g_l2cache_align_mask;
268 }
269 if (size & g_l2cache_align_mask) {
270 size &= ~g_l2cache_align_mask;
271 size += g_l2cache_line_size;
272 }
273
274
275 #ifdef PL310_ERRATA_727915
276 platform_pl310_write_debug(pl310_softc, 3);
277 #endif
278 while (size > 0) {
279 #ifdef PL310_ERRATA_588369
280 if (pl310_softc->sc_rtl_revision <= CACHE_ID_RELEASE_r1p0) {
281 /*
282 * Errata 588369 says that clean + inv may keep the
283 * cache line if it was clean, the recommanded
284 * workaround is to clean then invalidate the cache
285 * line, with write-back and cache linefill disabled.
286 */
287 pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
288 pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
289 } else
290 #endif
291 pl310_write4(pl310_softc, PL310_CLEAN_INV_LINE_PA,
292 start);
293 start += g_l2cache_line_size;
294 size -= g_l2cache_line_size;
295 }
296 #ifdef PL310_ERRATA_727915
297 platform_pl310_write_debug(pl310_softc, 0);
298 #endif
299
300 pl310_cache_sync();
301 PL310_UNLOCK(pl310_softc);
302 }
303
304 static void
305 pl310_wb_range(vm_paddr_t start, vm_size_t size)
306 {
307
308 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
309 return;
310
311 PL310_LOCK(pl310_softc);
312 if (start & g_l2cache_align_mask) {
313 size += start & g_l2cache_align_mask;
314 start &= ~g_l2cache_align_mask;
315 }
316
317 if (size & g_l2cache_align_mask) {
318 size &= ~g_l2cache_align_mask;
319 size += g_l2cache_line_size;
320 }
321
322 while (size > 0) {
323 pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
324 start += g_l2cache_line_size;
325 size -= g_l2cache_line_size;
326 }
327
328 pl310_cache_sync();
329 PL310_UNLOCK(pl310_softc);
330 }
331
332 static void
333 pl310_inv_range(vm_paddr_t start, vm_size_t size)
334 {
335
336 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
337 return;
338
339 PL310_LOCK(pl310_softc);
340 if (start & g_l2cache_align_mask) {
341 size += start & g_l2cache_align_mask;
342 start &= ~g_l2cache_align_mask;
343 }
344 if (size & g_l2cache_align_mask) {
345 size &= ~g_l2cache_align_mask;
346 size += g_l2cache_line_size;
347 }
348 while (size > 0) {
349 pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
350 start += g_l2cache_line_size;
351 size -= g_l2cache_line_size;
352 }
353
354 pl310_cache_sync();
355 PL310_UNLOCK(pl310_softc);
356 }
357
358 static void
359 pl310_drain_writebuf(void)
360 {
361
362 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
363 return;
364
365 PL310_LOCK(pl310_softc);
366 pl310_cache_sync();
367 PL310_UNLOCK(pl310_softc);
368 }
369
370 static void
371 pl310_set_way_sizes(struct pl310_softc *sc)
372 {
373 uint32_t aux_value;
374
375 aux_value = pl310_read4(sc, PL310_AUX_CTRL);
376 g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >>
377 AUX_CTRL_WAY_SIZE_SHIFT;
378 g_way_size = 1 << (g_way_size + 13);
379 if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT))
380 g_ways_assoc = 16;
381 else
382 g_ways_assoc = 8;
383 g_l2cache_way_mask = (1 << g_ways_assoc) - 1;
384 g_l2cache_size = g_way_size * g_ways_assoc;
385 }
386
387 /*
388 * Setup interrupt handling. This is done only if the cache controller is
389 * disabled, for debugging. We set counters so when a cache event happens we'll
390 * get interrupted and be warned that something is wrong, because no cache
391 * events should happen if we're disabled.
392 */
393 static void
394 pl310_config_intr(void *arg)
395 {
396 struct pl310_softc * sc;
397
398 sc = arg;
399
400 /* activate the interrupt */
401 bus_setup_intr(sc->sc_dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
402 pl310_filter, NULL, sc, &sc->sc_irq_h);
403
404 /* Cache Line Eviction for Counter 0 */
405 pl310_write4(sc, PL310_EVENT_COUNTER0_CONF,
406 EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_CO);
407 /* Data Read Request for Counter 1 */
408 pl310_write4(sc, PL310_EVENT_COUNTER1_CONF,
409 EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_DRREQ);
410
411 /* Enable and clear pending interrupts */
412 pl310_write4(sc, PL310_INTR_CLEAR, INTR_MASK_ECNTR);
413 pl310_write4(sc, PL310_INTR_MASK, INTR_MASK_ALL);
414
415 /* Enable counters and reset C0 and C1 */
416 pl310_write4(sc, PL310_EVENT_COUNTER_CTRL,
417 EVENT_COUNTER_CTRL_ENABLED |
418 EVENT_COUNTER_CTRL_C0_RESET |
419 EVENT_COUNTER_CTRL_C1_RESET);
420
421 config_intrhook_disestablish(sc->sc_ich);
422 free(sc->sc_ich, M_DEVBUF);
423 }
424
425 static int
426 pl310_probe(device_t dev)
427 {
428
429 if (!ofw_bus_status_okay(dev))
430 return (ENXIO);
431 if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data)
432 return (ENXIO);
433 device_set_desc(dev, "PL310 L2 cache controller");
434 return (0);
435 }
436
437 static int
438 pl310_attach(device_t dev)
439 {
440 struct pl310_softc *sc = device_get_softc(dev);
441 int rid;
442 uint32_t cache_id, debug_ctrl;
443
444 sc->sc_dev = dev;
445 rid = 0;
446 sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
447 RF_ACTIVE);
448 if (sc->sc_mem_res == NULL)
449 panic("%s: Cannot map registers", device_get_name(dev));
450
451 /* Allocate an IRQ resource */
452 rid = 0;
453 sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
454 RF_ACTIVE | RF_SHAREABLE);
455 if (sc->sc_irq_res == NULL) {
456 panic("Cannot allocate IRQ\n");
457 }
458
459 pl310_softc = sc;
460 mtx_init(&sc->sc_mtx, "pl310lock", NULL, MTX_SPIN);
461
462 cache_id = pl310_read4(sc, PL310_CACHE_ID);
463 sc->sc_rtl_revision = (cache_id >> CACHE_ID_RELEASE_SHIFT) &
464 CACHE_ID_RELEASE_MASK;
465 device_printf(dev, "Part number: 0x%x, release: 0x%x\n",
466 (cache_id >> CACHE_ID_PARTNUM_SHIFT) & CACHE_ID_PARTNUM_MASK,
467 (cache_id >> CACHE_ID_RELEASE_SHIFT) & CACHE_ID_RELEASE_MASK);
468
469 /*
470 * If L2 cache is already enabled then something has violated the rules,
471 * because caches are supposed to be off at kernel entry. The cache
472 * must be disabled to write the configuration registers without
473 * triggering an access error (SLVERR), but there's no documented safe
474 * procedure for disabling the L2 cache in the manual. So we'll try to
475 * invent one:
476 * - Use the debug register to force write-through mode and prevent
477 * linefills (allocation of new lines on read); now anything we do
478 * will not cause new data to come into the L2 cache.
479 * - Writeback and invalidate the current contents.
480 * - Disable the controller.
481 * - Restore the original debug settings.
482 */
483 if (pl310_read4(sc, PL310_CTRL) & CTRL_ENABLED) {
484 device_printf(dev, "Warning: L2 Cache should not already be "
485 "active; trying to de-activate and re-initialize...\n");
486 sc->sc_enabled = 1;
487 debug_ctrl = pl310_read4(sc, PL310_DEBUG_CTRL);
488 platform_pl310_write_debug(sc, debug_ctrl |
489 DEBUG_CTRL_DISABLE_WRITEBACK | DEBUG_CTRL_DISABLE_LINEFILL);
490 pl310_set_way_sizes(sc);
491 pl310_wbinv_all();
492 platform_pl310_write_ctrl(sc, CTRL_DISABLED);
493 platform_pl310_write_debug(sc, debug_ctrl);
494 }
495 sc->sc_enabled = pl310_enabled;
496
497 if (sc->sc_enabled) {
498 platform_pl310_init(sc);
499 pl310_set_way_sizes(sc); /* platform init might change these */
500 pl310_write4(pl310_softc, PL310_INV_WAY, 0xffff);
501 pl310_wait_background_op(PL310_INV_WAY, 0xffff);
502 platform_pl310_write_ctrl(sc, CTRL_ENABLED);
503 device_printf(dev, "L2 Cache enabled: %uKB/%dB %d ways\n",
504 (g_l2cache_size / 1024), g_l2cache_line_size, g_ways_assoc);
505 if (bootverbose)
506 pl310_print_config(sc);
507 } else {
508 malloc(sizeof(*sc->sc_ich), M_DEVBUF, M_WAITOK);
509 sc->sc_ich->ich_func = pl310_config_intr;
510 sc->sc_ich->ich_arg = sc;
511 if (config_intrhook_establish(sc->sc_ich) != 0) {
512 device_printf(dev,
513 "config_intrhook_establish failed\n");
514 return(ENXIO);
515 }
516 device_printf(dev, "L2 Cache disabled\n");
517 }
518
519 /* Set the l2 functions in the set of cpufuncs */
520 cpufuncs.cf_l2cache_wbinv_all = pl310_wbinv_all;
521 cpufuncs.cf_l2cache_wbinv_range = pl310_wbinv_range;
522 cpufuncs.cf_l2cache_inv_range = pl310_inv_range;
523 cpufuncs.cf_l2cache_wb_range = pl310_wb_range;
524 cpufuncs.cf_l2cache_drain_writebuf = pl310_drain_writebuf;
525
526 return (0);
527 }
528
529 static device_method_t pl310_methods[] = {
530 DEVMETHOD(device_probe, pl310_probe),
531 DEVMETHOD(device_attach, pl310_attach),
532 DEVMETHOD_END
533 };
534
535 static driver_t pl310_driver = {
536 "l2cache",
537 pl310_methods,
538 sizeof(struct pl310_softc),
539 };
540 static devclass_t pl310_devclass;
541
542 EARLY_DRIVER_MODULE(pl310, simplebus, pl310_driver, pl310_devclass, 0, 0,
543 BUS_PASS_CPU + BUS_PASS_ORDER_MIDDLE);
544
Cache object: 6f1b21bdac8e4f2bcf584151c2bcc74c
|