1 /*-
2 * Copyright (c) 1982, 1986 The Regents of the University of California.
3 * Copyright (c) 1989, 1990 William Jolitz
4 * Copyright (c) 1994 John Dyson
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to Berkeley by
8 * the Systems Programming Group of the University of Utah Computer
9 * Science Department, and William Jolitz.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by the University of
22 * California, Berkeley and its contributors.
23 * 4. Neither the name of the University nor the names of its contributors
24 * may be used to endorse or promote products derived from this software
25 * without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE.
38 *
39 * from: @(#)vm_machdep.c 7.3 (Berkeley) 5/13/91
40 * Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$
41 */
42
43 #include <sys/cdefs.h>
44 __FBSDID("$FreeBSD: releng/6.1/sys/arm/arm/vm_machdep.c 147889 2005-07-10 23:31:11Z davidxu $");
45
46 #include <sys/param.h>
47 #include <sys/systm.h>
48 #include <sys/kernel.h>
49 #include <sys/malloc.h>
50 #include <sys/mbuf.h>
51 #include <sys/proc.h>
52 #include <sys/socketvar.h>
53 #include <sys/sf_buf.h>
54 #include <sys/unistd.h>
55 #include <machine/cpu.h>
56 #include <machine/pcb.h>
57 #include <machine/sysarch.h>
58 #include <vm/vm.h>
59 #include <vm/pmap.h>
60 #include <sys/lock.h>
61 #include <sys/mutex.h>
62
63 #include <vm/vm.h>
64 #include <vm/vm_extern.h>
65 #include <vm/vm_kern.h>
66 #include <vm/vm_page.h>
67 #include <vm/vm_map.h>
68 #include <vm/vm_param.h>
69 #include <vm/uma.h>
70 #include <vm/uma_int.h>
71
72 #ifndef NSFBUFS
73 #define NSFBUFS (512 + maxusers * 16)
74 #endif
75
76 static void sf_buf_init(void *arg);
77 SYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL)
78
79 LIST_HEAD(sf_head, sf_buf);
80
81
82 /*
83 * A hash table of active sendfile(2) buffers
84 */
85 static struct sf_head *sf_buf_active;
86 static u_long sf_buf_hashmask;
87
88 #define SF_BUF_HASH(m) (((m) - vm_page_array) & sf_buf_hashmask)
89
90 static TAILQ_HEAD(, sf_buf) sf_buf_freelist;
91 static u_int sf_buf_alloc_want;
92
93 /*
94 * A lock used to synchronize access to the hash table and free list
95 */
96 static struct mtx sf_buf_lock;
97
98 /*
99 * Finish a fork operation, with process p2 nearly set up.
100 * Copy and update the pcb, set up the stack so that the child
101 * ready to run and return to user mode.
102 */
103 void
104 cpu_fork(register struct thread *td1, register struct proc *p2,
105 struct thread *td2, int flags)
106 {
107 struct pcb *pcb1, *pcb2;
108 struct trapframe *tf;
109 struct switchframe *sf;
110 struct mdproc *mdp2;
111
112 if ((flags & RFPROC) == 0)
113 return;
114 pcb1 = td1->td_pcb;
115 pcb2 = (struct pcb *)(td2->td_kstack + td2->td_kstack_pages * PAGE_SIZE) - 1;
116 #ifdef __XSCALE__
117 pmap_use_minicache(td2->td_kstack, td2->td_kstack_pages * PAGE_SIZE);
118 #endif
119 td2->td_pcb = pcb2;
120 bcopy(td1->td_pcb, pcb2, sizeof(*pcb2));
121 mdp2 = &p2->p_md;
122 bcopy(&td1->td_proc->p_md, mdp2, sizeof(*mdp2));
123 pcb2->un_32.pcb32_und_sp = td2->td_kstack + USPACE_UNDEF_STACK_TOP;
124 pcb2->un_32.pcb32_sp = td2->td_kstack +
125 USPACE_SVC_STACK_TOP - sizeof(*pcb2);
126 pmap_activate(td2);
127 td2->td_frame = tf =
128 (struct trapframe *)pcb2->un_32.pcb32_sp - 1;
129 *tf = *td1->td_frame;
130 sf = (struct switchframe *)tf - 1;
131 sf->sf_r4 = (u_int)fork_return;
132 sf->sf_r5 = (u_int)td2;
133 sf->sf_pc = (u_int)fork_trampoline;
134 tf->tf_spsr &= ~PSR_C_bit;
135 tf->tf_r0 = 0;
136 tf->tf_r1 = 0;
137 pcb2->un_32.pcb32_sp = (u_int)sf;
138
139 /* Setup to release sched_lock in fork_exit(). */
140 td2->td_md.md_spinlock_count = 1;
141 td2->td_md.md_saved_cspr = 0;
142 td2->td_md.md_tp = *(uint32_t **)ARM_TP_ADDRESS;
143 }
144
145 void
146 cpu_thread_swapin(struct thread *td)
147 {
148 }
149
150 void
151 cpu_thread_swapout(struct thread *td)
152 {
153 }
154
155 /*
156 * Detatch mapped page and release resources back to the system.
157 */
158 void
159 sf_buf_free(struct sf_buf *sf)
160 {
161 mtx_lock(&sf_buf_lock);
162 sf->ref_count--;
163 if (sf->ref_count == 0) {
164 TAILQ_INSERT_TAIL(&sf_buf_freelist, sf, free_entry);
165 nsfbufsused--;
166 if (sf_buf_alloc_want > 0)
167 wakeup_one(&sf_buf_freelist);
168 }
169 mtx_unlock(&sf_buf_lock);
170 }
171
172 /*
173 * * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
174 * */
175 static void
176 sf_buf_init(void *arg)
177 {
178 struct sf_buf *sf_bufs;
179 vm_offset_t sf_base;
180 int i;
181
182 nsfbufs = NSFBUFS;
183 TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs);
184
185 sf_buf_active = hashinit(nsfbufs, M_TEMP, &sf_buf_hashmask);
186 TAILQ_INIT(&sf_buf_freelist);
187 sf_base = kmem_alloc_nofault(kernel_map, nsfbufs * PAGE_SIZE);
188 sf_bufs = malloc(nsfbufs * sizeof(struct sf_buf), M_TEMP,
189 M_NOWAIT | M_ZERO);
190 for (i = 0; i < nsfbufs; i++) {
191 sf_bufs[i].kva = sf_base + i * PAGE_SIZE;
192 TAILQ_INSERT_TAIL(&sf_buf_freelist, &sf_bufs[i], free_entry);
193 }
194 sf_buf_alloc_want = 0;
195 mtx_init(&sf_buf_lock, "sf_buf", NULL, MTX_DEF);
196 }
197
198 /*
199 * Get an sf_buf from the freelist. Will block if none are available.
200 */
201 struct sf_buf *
202 sf_buf_alloc(struct vm_page *m, int flags)
203 {
204 struct sf_head *hash_list;
205 struct sf_buf *sf;
206 int error;
207
208 hash_list = &sf_buf_active[SF_BUF_HASH(m)];
209 mtx_lock(&sf_buf_lock);
210 LIST_FOREACH(sf, hash_list, list_entry) {
211 if (sf->m == m) {
212 sf->ref_count++;
213 if (sf->ref_count == 1) {
214 TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
215 nsfbufsused++;
216 nsfbufspeak = imax(nsfbufspeak, nsfbufsused);
217 }
218 goto done;
219 }
220 }
221 while ((sf = TAILQ_FIRST(&sf_buf_freelist)) == NULL) {
222 if (flags & SFB_NOWAIT)
223 goto done;
224 sf_buf_alloc_want++;
225 mbstat.sf_allocwait++;
226 error = msleep(&sf_buf_freelist, &sf_buf_lock,
227 (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
228 sf_buf_alloc_want--;
229
230
231 /*
232 * If we got a signal, don't risk going back to sleep.
233 */
234 if (error)
235 goto done;
236 }
237 TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
238 if (sf->m != NULL)
239 LIST_REMOVE(sf, list_entry);
240 LIST_INSERT_HEAD(hash_list, sf, list_entry);
241 sf->ref_count = 1;
242 sf->m = m;
243 nsfbufsused++;
244 nsfbufspeak = imax(nsfbufspeak, nsfbufsused);
245 pmap_qenter(sf->kva, &sf->m, 1);
246 done:
247 mtx_unlock(&sf_buf_lock);
248 return (sf);
249
250 }
251
252 /*
253 * Initialize machine state (pcb and trap frame) for a new thread about to
254 * upcall. Put enough state in the new thread's PCB to get it to go back
255 * userret(), where we can intercept it again to set the return (upcall)
256 * Address and stack, along with those from upcals that are from other sources
257 * such as those generated in thread_userret() itself.
258 */
259 void
260 cpu_set_upcall(struct thread *td, struct thread *td0)
261 {
262 struct trapframe *tf;
263 struct switchframe *sf;
264
265 bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe));
266 bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb));
267 tf = td->td_frame;
268 sf = (struct switchframe *)tf - 1;
269 sf->sf_r4 = (u_int)fork_return;
270 sf->sf_r5 = (u_int)td;
271 sf->sf_pc = (u_int)fork_trampoline;
272 tf->tf_spsr &= ~PSR_C_bit;
273 tf->tf_r0 = 0;
274 td->td_pcb->un_32.pcb32_sp = (u_int)sf;
275 td->td_pcb->un_32.pcb32_und_sp = td->td_kstack + USPACE_UNDEF_STACK_TOP;
276
277 /* Setup to release sched_lock in fork_exit(). */
278 td->td_md.md_spinlock_count = 1;
279 td->td_md.md_saved_cspr = 0;
280 }
281
282 /*
283 * Set that machine state for performing an upcall that has to
284 * be done in thread_userret() so that those upcalls generated
285 * in thread_userret() itself can be done as well.
286 */
287 void
288 cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
289 stack_t *stack)
290 {
291 struct trapframe *tf = td->td_frame;
292
293 tf->tf_usr_sp = ((int)stack->ss_sp + stack->ss_size
294 - sizeof(struct trapframe)) & ~7;
295 tf->tf_pc = (int)entry;
296 tf->tf_r0 = (int)arg;
297 tf->tf_spsr = PSR_USR32_MODE;
298 }
299
300 int
301 cpu_set_user_tls(struct thread *td, void *tls_base)
302 {
303
304 if (td != curthread)
305 td->td_md.md_tp = tls_base;
306 else {
307 critical_enter();
308 *(void **)ARM_TP_ADDRESS = tls_base;
309 critical_exit();
310 }
311 return (0);
312 }
313
314 void
315 cpu_thread_exit(struct thread *td)
316 {
317 }
318
319 void
320 cpu_thread_setup(struct thread *td)
321 {
322 td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_pages *
323 PAGE_SIZE) - 1;
324 td->td_frame = (struct trapframe *)
325 ((u_int)td->td_kstack + USPACE_SVC_STACK_TOP - sizeof(struct pcb)) - 1;
326 #ifdef __XSCALE__
327 pmap_use_minicache(td->td_kstack, td->td_kstack_pages * PAGE_SIZE);
328 #endif
329
330 }
331 void
332 cpu_thread_clean(struct thread *td)
333 {
334 }
335
336 /*
337 * Intercept the return address from a freshly forked process that has NOT
338 * been scheduled yet.
339 *
340 * This is needed to make kernel threads stay in kernel mode.
341 */
342 void
343 cpu_set_fork_handler(struct thread *td, void (*func)(void *), void *arg)
344 {
345 struct switchframe *sf;
346 struct trapframe *tf;
347
348 tf = td->td_frame;
349 sf = (struct switchframe *)tf - 1;
350 sf->sf_r4 = (u_int)func;
351 sf->sf_r5 = (u_int)arg;
352 td->td_pcb->un_32.pcb32_sp = (u_int)sf;
353 }
354
355 /*
356 * Software interrupt handler for queued VM system processing.
357 */
358 void
359 swi_vm(void *dummy)
360 {
361 }
362
363 void
364 cpu_exit(struct thread *td)
365 {
366 }
367
368 #ifdef ARM_USE_SMALL_ALLOC
369
370 static TAILQ_HEAD(,arm_small_page) pages_normal =
371 TAILQ_HEAD_INITIALIZER(pages_normal);
372 static TAILQ_HEAD(,arm_small_page) pages_wt =
373 TAILQ_HEAD_INITIALIZER(pages_wt);
374 static TAILQ_HEAD(,arm_small_page) free_pgdesc =
375 TAILQ_HEAD_INITIALIZER(free_pgdesc);
376
377 extern uma_zone_t l2zone;
378
379 struct mtx smallalloc_mtx;
380
381 MALLOC_DEFINE(M_VMSMALLALLOC, "VM Small alloc", "VM Small alloc data");
382
383 vm_offset_t alloc_curaddr;
384 vm_offset_t alloc_firstaddr;
385
386 extern int doverbose;
387
388 void
389 arm_add_smallalloc_pages(void *list, void *mem, int bytes, int pagetable)
390 {
391 struct arm_small_page *pg;
392
393 bytes &= ~PAGE_SIZE;
394 while (bytes > 0) {
395 pg = (struct arm_small_page *)list;
396 pg->addr = mem;
397 if (pagetable)
398 TAILQ_INSERT_HEAD(&pages_wt, pg, pg_list);
399 else
400 TAILQ_INSERT_HEAD(&pages_normal, pg, pg_list);
401 list = (char *)list + sizeof(*pg);
402 mem = (char *)mem + PAGE_SIZE;
403 bytes -= PAGE_SIZE;
404 }
405 }
406
407 static void *
408 arm_uma_do_alloc(struct arm_small_page **pglist, int bytes, int pagetable)
409 {
410 void *ret;
411 vm_page_t page_array = NULL;
412
413
414 *pglist = (void *)kmem_malloc(kmem_map, (0x100000 / PAGE_SIZE) *
415 sizeof(struct arm_small_page), M_WAITOK);
416 if (alloc_curaddr < 0xf0000000) {/* XXX */
417 mtx_lock(&Giant);
418 page_array = vm_page_alloc_contig(0x100000 / PAGE_SIZE,
419 0, 0xffffffff, 0x100000, 0);
420 mtx_unlock(&Giant);
421 }
422 if (page_array) {
423 vm_paddr_t pa = VM_PAGE_TO_PHYS(page_array);
424 mtx_lock(&smallalloc_mtx);
425 ret = (void*)alloc_curaddr;
426 alloc_curaddr += 0x100000;
427 /* XXX: ARM_TP_ADDRESS should probably be move elsewhere. */
428 if (alloc_curaddr == ARM_TP_ADDRESS)
429 alloc_curaddr += 0x100000;
430 mtx_unlock(&smallalloc_mtx);
431 pmap_kenter_section((vm_offset_t)ret, pa
432 , pagetable);
433
434
435 } else {
436 kmem_free(kmem_map, (vm_offset_t)*pglist,
437 (0x100000 / PAGE_SIZE) * sizeof(struct arm_small_page));
438 *pglist = NULL;
439 ret = (void *)kmem_malloc(kmem_map, bytes, M_WAITOK);
440 }
441 return (ret);
442 }
443
444 void *
445 uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
446 {
447 void *ret;
448 struct arm_small_page *sp, *tmp;
449 TAILQ_HEAD(,arm_small_page) *head;
450
451 *flags = UMA_SLAB_PRIV;
452 /*
453 * For CPUs where we setup page tables as write back, there's no
454 * need to maintain two separate pools.
455 */
456 if (zone == l2zone && pte_l1_s_cache_mode != pte_l1_s_cache_mode_pt)
457 head = (void *)&pages_wt;
458 else
459 head = (void *)&pages_normal;
460
461 mtx_lock(&smallalloc_mtx);
462 sp = TAILQ_FIRST(head);
463
464 if (!sp) {
465 /* No more free pages, need to alloc more. */
466 mtx_unlock(&smallalloc_mtx);
467 if (!(wait & M_WAITOK)) {
468 ret = (void *)kmem_malloc(kmem_map, bytes, wait);
469 return (ret);
470 }
471 /* Try to alloc 1MB of contiguous memory. */
472 ret = arm_uma_do_alloc(&sp, bytes, zone == l2zone ?
473 SECTION_PT : SECTION_CACHE);
474 mtx_lock(&smallalloc_mtx);
475 if (sp) {
476 for (int i = 0; i < (0x100000 / PAGE_SIZE) - 1;
477 i++) {
478 tmp = &sp[i];
479 tmp->addr = (char *)ret + i * PAGE_SIZE;
480 TAILQ_INSERT_HEAD(head, tmp, pg_list);
481 }
482 ret = (char *)ret + 0x100000 - PAGE_SIZE;
483 TAILQ_INSERT_HEAD(&free_pgdesc, &sp[(0x100000 / (
484 PAGE_SIZE)) - 1], pg_list);
485 }
486
487 } else {
488 sp = TAILQ_FIRST(head);
489 TAILQ_REMOVE(head, sp, pg_list);
490 TAILQ_INSERT_HEAD(&free_pgdesc, sp, pg_list);
491 ret = sp->addr;
492 }
493 mtx_unlock(&smallalloc_mtx);
494 if ((wait & M_ZERO))
495 bzero(ret, bytes);
496 return (ret);
497 }
498
499 void
500 uma_small_free(void *mem, int size, u_int8_t flags)
501 {
502 pd_entry_t *pd;
503 pt_entry_t *pt;
504
505 if (mem < (void *)alloc_firstaddr)
506 kmem_free(kmem_map, (vm_offset_t)mem, size);
507 else {
508 struct arm_small_page *sp;
509
510 mtx_lock(&smallalloc_mtx);
511 sp = TAILQ_FIRST(&free_pgdesc);
512 KASSERT(sp != NULL, ("No more free page descriptor ?"));
513 TAILQ_REMOVE(&free_pgdesc, sp, pg_list);
514 sp->addr = mem;
515 pmap_get_pde_pte(kernel_pmap, (vm_offset_t)mem, &pd, &pt);
516 if ((*pd & pte_l1_s_cache_mask) == pte_l1_s_cache_mode_pt &&
517 pte_l1_s_cache_mode_pt != pte_l1_s_cache_mode)
518 TAILQ_INSERT_HEAD(&pages_wt, sp, pg_list);
519 else
520 TAILQ_INSERT_HEAD(&pages_normal, sp, pg_list);
521 mtx_unlock(&smallalloc_mtx);
522 }
523 }
524
525 #endif
Cache object: 68a783e956237bfb57846202cfc5fef8
|