1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2018 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include <sys/param.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/pcpu.h>
38 #include <sys/proc.h>
39 #include <sys/sched.h>
40 #include <sys/sysctl.h>
41 #include <sys/systm.h>
42 #include <vm/vm.h>
43 #include <vm/vm_param.h>
44 #include <vm/vm_extern.h>
45 #include <vm/pmap.h>
46 #include <vm/vm_map.h>
47 #include <vm/vm_page.h>
48
49 int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int);
50 static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int);
51 int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int);
52 static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int);
53 int fubyte_fast(volatile const void *base, u_int kcr3);
54 static int (*fubyte_fast_tramp)(volatile const void *, u_int);
55 int fuword16_fast(volatile const void *base, u_int kcr3);
56 static int (*fuword16_fast_tramp)(volatile const void *, u_int);
57 int fueword_fast(volatile const void *base, long *val, u_int kcr3);
58 static int (*fueword_fast_tramp)(volatile const void *, long *, u_int);
59 int subyte_fast(volatile void *base, int val, u_int kcr3);
60 static int (*subyte_fast_tramp)(volatile void *, int, u_int);
61 int suword16_fast(volatile void *base, int val, u_int kcr3);
62 static int (*suword16_fast_tramp)(volatile void *, int, u_int);
63 int suword_fast(volatile void *base, long val, u_int kcr3);
64 static int (*suword_fast_tramp)(volatile void *, long, u_int);
65
66 static int fast_copyout = 1;
67 SYSCTL_INT(_machdep, OID_AUTO, fast_copyout, CTLFLAG_RWTUN,
68 &fast_copyout, 0,
69 "");
70
71 void
72 copyout_init_tramp(void)
73 {
74
75 copyin_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
76 (uintptr_t)copyin_fast + setidt_disp);
77 copyout_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
78 (uintptr_t)copyout_fast + setidt_disp);
79 fubyte_fast_tramp = (int (*)(volatile const void *, u_int))(
80 (uintptr_t)fubyte_fast + setidt_disp);
81 fuword16_fast_tramp = (int (*)(volatile const void *, u_int))(
82 (uintptr_t)fuword16_fast + setidt_disp);
83 fueword_fast_tramp = (int (*)(volatile const void *, long *, u_int))(
84 (uintptr_t)fueword_fast + setidt_disp);
85 subyte_fast_tramp = (int (*)(volatile void *, int, u_int))(
86 (uintptr_t)subyte_fast + setidt_disp);
87 suword16_fast_tramp = (int (*)(volatile void *, int, u_int))(
88 (uintptr_t)suword16_fast + setidt_disp);
89 suword_fast_tramp = (int (*)(volatile void *, long, u_int))(
90 (uintptr_t)suword_fast + setidt_disp);
91 }
92
93 int
94 cp_slow0(vm_offset_t uva, size_t len, bool write,
95 void (*f)(vm_offset_t, void *), void *arg)
96 {
97 struct pcpu *pc;
98 vm_page_t m[2];
99 vm_offset_t kaddr;
100 int error, i, plen;
101 bool sleepable;
102
103 plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE);
104 MPASS(plen <= nitems(m));
105 error = 0;
106 i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len,
107 (write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT,
108 m, nitems(m));
109 if (i != plen)
110 return (EFAULT);
111 sched_pin();
112 pc = get_pcpu();
113 if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 ||
114 (curthread->td_pflags & TDP_NOFAULTING) != 0) {
115 sleepable = false;
116 mtx_lock(&pc->pc_copyout_mlock);
117 kaddr = pc->pc_copyout_maddr;
118 } else {
119 sleepable = true;
120 sx_xlock(&pc->pc_copyout_slock);
121 kaddr = pc->pc_copyout_saddr;
122 }
123 pmap_cp_slow0_map(kaddr, plen, m);
124 kaddr += uva - trunc_page(uva);
125 f(kaddr, arg);
126 sched_unpin();
127 if (sleepable)
128 sx_xunlock(&pc->pc_copyout_slock);
129 else
130 mtx_unlock(&pc->pc_copyout_mlock);
131 vm_page_unhold_pages(m, plen);
132 return (error);
133 }
134
135 struct copyinstr_arg0 {
136 vm_offset_t kc;
137 size_t len;
138 size_t alen;
139 bool end;
140 };
141
142 static void
143 copyinstr_slow0(vm_offset_t kva, void *arg)
144 {
145 struct copyinstr_arg0 *ca;
146 char c;
147
148 ca = arg;
149 MPASS(ca->alen == 0 && ca->len > 0 && !ca->end);
150 while (ca->alen < ca->len && !ca->end) {
151 c = *(char *)(kva + ca->alen);
152 *(char *)ca->kc = c;
153 ca->alen++;
154 ca->kc++;
155 if (c == '\0')
156 ca->end = true;
157 }
158 }
159
160 int
161 copyinstr(const void *udaddr, void *kaddr, size_t maxlen, size_t *lencopied)
162 {
163 struct copyinstr_arg0 ca;
164 vm_offset_t uc;
165 size_t plen;
166 int error;
167
168 error = 0;
169 ca.end = false;
170 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
171 plen < maxlen && !ca.end; uc += ca.alen, plen += ca.alen) {
172 ca.len = round_page(uc) - uc;
173 if (ca.len == 0)
174 ca.len = PAGE_SIZE;
175 if (plen + ca.len > maxlen)
176 ca.len = maxlen - plen;
177 ca.alen = 0;
178 if (cp_slow0(uc, ca.len, false, copyinstr_slow0, &ca) != 0) {
179 error = EFAULT;
180 break;
181 }
182 }
183 if (!ca.end && plen == maxlen && error == 0)
184 error = ENAMETOOLONG;
185 if (lencopied != NULL)
186 *lencopied = plen;
187 return (error);
188 }
189
190 struct copyin_arg0 {
191 vm_offset_t kc;
192 size_t len;
193 };
194
195 static void
196 copyin_slow0(vm_offset_t kva, void *arg)
197 {
198 struct copyin_arg0 *ca;
199
200 ca = arg;
201 bcopy((void *)kva, (void *)ca->kc, ca->len);
202 }
203
204 int
205 copyin(const void *udaddr, void *kaddr, size_t len)
206 {
207 struct copyin_arg0 ca;
208 vm_offset_t uc;
209 size_t plen;
210
211 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
212 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
213 return (EFAULT);
214 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
215 copyin_fast_tramp(udaddr, kaddr, len, pmap_get_kcr3()) == 0))
216 return (0);
217 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
218 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
219 ca.len = round_page(uc) - uc;
220 if (ca.len == 0)
221 ca.len = PAGE_SIZE;
222 if (plen + ca.len > len)
223 ca.len = len - plen;
224 if (cp_slow0(uc, ca.len, false, copyin_slow0, &ca) != 0)
225 return (EFAULT);
226 }
227 return (0);
228 }
229
230 static void
231 copyout_slow0(vm_offset_t kva, void *arg)
232 {
233 struct copyin_arg0 *ca;
234
235 ca = arg;
236 bcopy((void *)ca->kc, (void *)kva, ca->len);
237 }
238
239 int
240 copyout(const void *kaddr, void *udaddr, size_t len)
241 {
242 struct copyin_arg0 ca;
243 vm_offset_t uc;
244 size_t plen;
245
246 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
247 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
248 return (EFAULT);
249 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
250 copyout_fast_tramp(kaddr, udaddr, len, pmap_get_kcr3()) == 0))
251 return (0);
252 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
253 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
254 ca.len = round_page(uc) - uc;
255 if (ca.len == 0)
256 ca.len = PAGE_SIZE;
257 if (plen + ca.len > len)
258 ca.len = len - plen;
259 if (cp_slow0(uc, ca.len, true, copyout_slow0, &ca) != 0)
260 return (EFAULT);
261 }
262 return (0);
263 }
264
265 /*
266 * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user
267 * memory.
268 */
269
270 static void
271 fubyte_slow0(vm_offset_t kva, void *arg)
272 {
273
274 *(int *)arg = *(u_char *)kva;
275 }
276
277 int
278 fubyte(volatile const void *base)
279 {
280 int res;
281
282 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
283 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
284 return (-1);
285 if (fast_copyout) {
286 res = fubyte_fast_tramp(base, pmap_get_kcr3());
287 if (res != -1)
288 return (res);
289 }
290 if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0,
291 &res) != 0)
292 return (-1);
293 return (res);
294 }
295
296 static void
297 fuword16_slow0(vm_offset_t kva, void *arg)
298 {
299
300 *(int *)arg = *(uint16_t *)kva;
301 }
302
303 int
304 fuword16(volatile const void *base)
305 {
306 int res;
307
308 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
309 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
310 return (-1);
311 if (fast_copyout) {
312 res = fuword16_fast_tramp(base, pmap_get_kcr3());
313 if (res != -1)
314 return (res);
315 }
316 if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false,
317 fuword16_slow0, &res) != 0)
318 return (-1);
319 return (res);
320 }
321
322 static void
323 fueword_slow0(vm_offset_t kva, void *arg)
324 {
325
326 *(uint32_t *)arg = *(uint32_t *)kva;
327 }
328
329 int
330 fueword(volatile const void *base, long *val)
331 {
332 uint32_t res;
333
334 if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base ||
335 (uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS)
336 return (-1);
337 if (fast_copyout) {
338 if (fueword_fast_tramp(base, val, pmap_get_kcr3()) == 0)
339 return (0);
340 }
341 if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0,
342 &res) != 0)
343 return (-1);
344 *val = res;
345 return (0);
346 }
347
348 int
349 fueword32(volatile const void *base, int32_t *val)
350 {
351
352 return (fueword(base, (long *)val));
353 }
354
355 /*
356 * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory.
357 */
358
359 static void
360 subyte_slow0(vm_offset_t kva, void *arg)
361 {
362
363 *(u_char *)kva = *(int *)arg;
364 }
365
366 int
367 subyte(volatile void *base, int byte)
368 {
369
370 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
371 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
372 return (-1);
373 if (fast_copyout && subyte_fast_tramp(base, byte, pmap_get_kcr3()) == 0)
374 return (0);
375 return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0,
376 &byte) != 0 ? -1 : 0);
377 }
378
379 static void
380 suword16_slow0(vm_offset_t kva, void *arg)
381 {
382
383 *(int *)kva = *(uint16_t *)arg;
384 }
385
386 int
387 suword16(volatile void *base, int word)
388 {
389
390 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
391 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
392 return (-1);
393 if (fast_copyout && suword16_fast_tramp(base, word, pmap_get_kcr3())
394 == 0)
395 return (0);
396 return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true,
397 suword16_slow0, &word) != 0 ? -1 : 0);
398 }
399
400 static void
401 suword_slow0(vm_offset_t kva, void *arg)
402 {
403
404 *(int *)kva = *(uint32_t *)arg;
405 }
406
407 int
408 suword(volatile void *base, long word)
409 {
410
411 if ((uintptr_t)base + sizeof(word) < (uintptr_t)base ||
412 (uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS)
413 return (-1);
414 if (fast_copyout && suword_fast_tramp(base, word, pmap_get_kcr3()) == 0)
415 return (0);
416 return (cp_slow0((vm_offset_t)base, sizeof(long), true,
417 suword_slow0, &word) != 0 ? -1 : 0);
418 }
419
420 int
421 suword32(volatile void *base, int32_t word)
422 {
423
424 return (suword(base, word));
425 }
426
427 struct casueword_arg0 {
428 uint32_t oldval;
429 uint32_t newval;
430 int res;
431 };
432
433 static void
434 casueword_slow0(vm_offset_t kva, void *arg)
435 {
436 struct casueword_arg0 *ca;
437
438 ca = arg;
439 ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval,
440 ca->newval);
441 }
442
443 int
444 casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
445 uint32_t newval)
446 {
447 struct casueword_arg0 ca;
448 int res;
449
450 ca.oldval = oldval;
451 ca.newval = newval;
452 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
453 casueword_slow0, &ca);
454 if (res == 0) {
455 *oldvalp = ca.oldval;
456 return (ca.res);
457 }
458 return (-1);
459 }
460
461 int
462 casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval)
463 {
464 struct casueword_arg0 ca;
465 int res;
466
467 ca.oldval = oldval;
468 ca.newval = newval;
469 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
470 casueword_slow0, &ca);
471 if (res == 0) {
472 *oldvalp = ca.oldval;
473 return (ca.res);
474 }
475 return (-1);
476 }
Cache object: 00129a7da831160d7630b2e7650de943
|