FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_kmem.c
1 /* $NetBSD: subr_kmem.c,v 1.11.2.2 2007/03/04 14:47:55 bouyer Exp $ */
2
3 /*-
4 * Copyright (c)2006 YAMAMOTO Takashi,
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 /*
30 * allocator of kernel wired memory.
31 *
32 * TODO:
33 * - worth to have "intrsafe" version? maybe..
34 */
35
36 #include <sys/cdefs.h>
37 __KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.11.2.2 2007/03/04 14:47:55 bouyer Exp $");
38
39 #include <sys/param.h>
40 #include <sys/callback.h>
41 #include <sys/kmem.h>
42 #include <sys/vmem.h>
43
44 #include <uvm/uvm_extern.h>
45 #include <uvm/uvm_map.h>
46
47 #include <lib/libkern/libkern.h>
48
49 #define KMEM_QUANTUM_SIZE (ALIGNBYTES + 1)
50
51 static vmem_t *kmem_arena;
52 static struct callback_entry kmem_kva_reclaim_entry;
53
54 #if defined(DEBUG)
55 static void kmem_poison_fill(void *, size_t);
56 static void kmem_poison_check(void *, size_t);
57 #else /* defined(DEBUG) */
58 #define kmem_poison_fill(p, sz) /* nothing */
59 #define kmem_poison_check(p, sz) /* nothing */
60 #endif /* defined(DEBUG) */
61
62 static vmem_addr_t kmem_backend_alloc(vmem_t *, vmem_size_t, vmem_size_t *,
63 vm_flag_t);
64 static void kmem_backend_free(vmem_t *, vmem_addr_t, vmem_size_t);
65 static int kmem_kva_reclaim_callback(struct callback_entry *, void *, void *);
66
67 static inline vm_flag_t
68 kmf_to_vmf(km_flag_t kmflags)
69 {
70 vm_flag_t vmflags;
71
72 KASSERT((kmflags & (KM_SLEEP|KM_NOSLEEP)) != 0);
73 KASSERT((~kmflags & (KM_SLEEP|KM_NOSLEEP)) != 0);
74
75 vmflags = 0;
76 if ((kmflags & KM_SLEEP) != 0) {
77 vmflags |= VM_SLEEP;
78 }
79 if ((kmflags & KM_NOSLEEP) != 0) {
80 vmflags |= VM_NOSLEEP;
81 }
82
83 return vmflags;
84 }
85
86 /* ---- kmem API */
87
88 /*
89 * kmem_alloc: allocate wired memory.
90 *
91 * => must not be called from interrupt context.
92 */
93
94 void *
95 kmem_alloc(size_t size, km_flag_t kmflags)
96 {
97 void *p;
98
99 p = (void *)vmem_alloc(kmem_arena, size,
100 kmf_to_vmf(kmflags) | VM_INSTANTFIT);
101 if (p != NULL) {
102 kmem_poison_check(p, size);
103 }
104 return p;
105 }
106
107 /*
108 * kmem_zalloc: allocate wired memory.
109 *
110 * => must not be called from interrupt context.
111 */
112
113 void *
114 kmem_zalloc(size_t size, km_flag_t kmflags)
115 {
116 void *p;
117
118 p = kmem_alloc(size, kmflags);
119 if (p != NULL) {
120 memset(p, 0, size);
121 }
122 return p;
123 }
124
125 /*
126 * kmem_free: free wired memory allocated by kmem_alloc.
127 *
128 * => must not be called from interrupt context.
129 */
130
131 void
132 kmem_free(void *p, size_t size)
133 {
134
135 kmem_poison_fill(p, size);
136 vmem_free(kmem_arena, (vmem_addr_t)p, size);
137 }
138
139 void
140 kmem_init(void)
141 {
142
143 kmem_arena = vmem_create("kmem", 0, 0, KMEM_QUANTUM_SIZE,
144 kmem_backend_alloc, kmem_backend_free, NULL,
145 KMEM_QUANTUM_SIZE * 32, VM_SLEEP);
146 callback_register(&vm_map_to_kernel(kernel_map)->vmk_reclaim_callback,
147 &kmem_kva_reclaim_entry, kmem_arena, kmem_kva_reclaim_callback);
148 }
149
150 size_t
151 kmem_roundup_size(size_t size)
152 {
153
154 return vmem_roundup_size(kmem_arena, size);
155 }
156
157 /* ---- uvm glue */
158
159 #include <uvm/uvm_extern.h>
160
161 static vmem_addr_t
162 kmem_backend_alloc(vmem_t *dummy, vmem_size_t size, vmem_size_t *resultsize,
163 vm_flag_t vmflags)
164 {
165 uvm_flag_t uflags;
166 vaddr_t va;
167
168 KASSERT(dummy == NULL);
169 KASSERT(size != 0);
170 KASSERT((vmflags & (VM_SLEEP|VM_NOSLEEP)) != 0);
171 KASSERT((~vmflags & (VM_SLEEP|VM_NOSLEEP)) != 0);
172
173 if ((vmflags & VM_NOSLEEP) != 0) {
174 uflags = UVM_KMF_TRYLOCK | UVM_KMF_NOWAIT;
175 } else {
176 uflags = UVM_KMF_WAITVA;
177 }
178 *resultsize = size = round_page(size);
179 va = uvm_km_alloc(kernel_map, size, 0,
180 uflags | UVM_KMF_WIRED | UVM_KMF_CANFAIL);
181 if (va != 0) {
182 kmem_poison_fill((void *)va, size);
183 }
184 return (vmem_addr_t)va;
185 }
186
187 static void
188 kmem_backend_free(vmem_t *dummy, vmem_addr_t addr, vmem_size_t size)
189 {
190
191 KASSERT(dummy == NULL);
192 KASSERT(addr != 0);
193 KASSERT(size != 0);
194 KASSERT(size == round_page(size));
195
196 kmem_poison_check((void *)addr, size);
197 uvm_km_free(kernel_map, (vaddr_t)addr, size, UVM_KMF_WIRED);
198 }
199
200 static int
201 kmem_kva_reclaim_callback(struct callback_entry *ce, void *obj, void *arg)
202 {
203 vmem_t *vm = obj;
204
205 vmem_reap(vm);
206 return CALLBACK_CHAIN_CONTINUE;
207 }
208
209 /* ---- debug */
210
211 #if defined(DEBUG)
212
213 #if defined(_LP64)
214 #define PRIME 0x9e37fffffffc0001UL
215 #else /* defined(_LP64) */
216 #define PRIME 0x9e3779b1
217 #endif /* defined(_LP64) */
218
219 static inline uint8_t
220 kmem_poison_pattern(const void *p)
221 {
222
223 return (uint8_t)((((uintptr_t)p) * PRIME)
224 >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT);
225 }
226
227 static void
228 kmem_poison_fill(void *p, size_t sz)
229 {
230 uint8_t *cp;
231 const uint8_t *ep;
232
233 cp = p;
234 ep = cp + sz;
235 while (cp < ep) {
236 *cp = kmem_poison_pattern(cp);
237 cp++;
238 }
239 }
240
241 static void
242 kmem_poison_check(void *p, size_t sz)
243 {
244 uint8_t *cp;
245 const uint8_t *ep;
246
247 cp = p;
248 ep = cp + sz;
249 while (cp < ep) {
250 const uint8_t expected = kmem_poison_pattern(cp);
251
252 if (*cp != expected) {
253 panic("%s: %p: 0x%02x != 0x%02x\n",
254 __func__, cp, *cp, expected);
255 }
256 cp++;
257 }
258 }
259
260 #endif /* defined(DEBUG) */
Cache object: 62f0f2239ce16b271b9f6557318456c2
|