FreeBSD/Linux Kernel Cross Reference
sys/kern/exception.c
1 /*
2 * Mach Operating System
3 * Copyright (c) 1993,1992,1991,1990,1989,1988,1987 Carnegie Mellon University
4 * All Rights Reserved.
5 *
6 * Permission to use, copy, modify and distribute this software and its
7 * documentation is hereby granted, provided that both the copyright
8 * notice and this permission notice appear in all copies of the
9 * software, derivative works or modified versions, and any portions
10 * thereof, and that both notices appear in supporting documentation.
11 *
12 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
15 *
16 * Carnegie Mellon requests users of this software to return to
17 *
18 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
19 * School of Computer Science
20 * Carnegie Mellon University
21 * Pittsburgh PA 15213-3890
22 *
23 * any improvements or extensions that they make and grant Carnegie Mellon
24 * the rights to redistribute these changes.
25 */
26 /*
27 * HISTORY
28 * $Log: exception.c,v $
29 * Revision 2.18 93/12/07 13:54:02 dbg
30 * Use 'MACH_MSG_TYPE_INTEGER_T' for type of integer_t.
31 * [93/11/19 dbg]
32 *
33 * Revision 2.17 93/11/17 17:09:45 dbg
34 * 64-bit cleanup. Subcode must be integer_t, since it holds an
35 * address for EXC_BAD_ADDRESS. I made exception and code be
36 * natural_t for consistency.
37 * [93/11/09 dbg]
38 *
39 * Use thread_lock instead of ith_lock, task_lock instead of
40 * itk_lock.
41 * [93/07/12 dbg]
42 *
43 * Declared continuation functions as not returning.
44 * [93/05/04 dbg]
45 *
46 * Added ANSI function prototypes. Removed include of
47 * kern/sched.h. Removed KEEP_STACKS code.
48 * [93/03/25 dbg]
49 *
50 * Revision 2.16 93/03/09 10:55:01 danner
51 * Removed duplicated decl for thread_syscall_return().
52 * Added a tiny GNUism to shutup GCC.
53 * [93/03/06 af]
54 *
55 * Revision 2.15 93/01/14 17:34:09 danner
56 * 64bit cleanup.
57 * [92/12/01 af]
58 *
59 * Revision 2.14 92/08/03 17:37:00 jfriedl
60 * removed silly prototypes
61 * [92/08/02 jfriedl]
62 *
63 * Revision 2.13 92/05/21 17:13:19 jfriedl
64 * tried prototypes.
65 * [92/05/20 jfriedl]
66 *
67 * 16-May-92 Jeffrey Friedl (jfriedl) at Carnegie-Mellon University
68 * Cleanup to quiet gcc warnings. Renamed arguments 'exception'
69 * to '_exception' to avoid name conflict.
70 * Revision 2.12 92/03/10 16:26:24 jsb
71 * Don't convert exception kmsg to network format.
72 * [92/02/21 09:02:59 jsb]
73 *
74 * Revision 2.11 92/03/03 00:44:45 rpd
75 * Changed debug_user_with_kdb to FALSE.
76 * [92/03/02 rpd]
77 *
78 * Revision 2.10 91/12/14 14:18:23 jsb
79 * Removed ipc_fields.h hack.
80 *
81 * Revision 2.9 91/12/13 13:41:38 jsb
82 * Added NORMA_IPC support.
83 *
84 * Revision 2.8 91/08/28 11:14:26 jsb
85 * Added seqno arguments to ipc_mqueue_receive
86 * and exception_raise_continue_slow.
87 * [91/08/10 rpd]
88 *
89 * Changed msgh_kind to msgh_seqno.
90 * [91/08/10 rpd]
91 *
92 * Changed exception_no_server to print an informative message
93 * before invoking kdb, so this doesn't look like a kernel bug.
94 * [91/08/09 rpd]
95 *
96 * Revision 2.7 91/06/25 10:28:06 rpd
97 * Fixed ikm_cache critical sections to avoid blocking operations.
98 * [91/05/23 rpd]
99 *
100 * Revision 2.6 91/05/14 16:41:02 mrt
101 * Correcting copyright
102 *
103 * Revision 2.5 91/03/16 14:49:47 rpd
104 * Fixed assertion typo.
105 * [91/03/08 rpd]
106 * Replaced ipc_thread_switch with thread_handoff.
107 * Replaced ith_saved with ikm_cache.
108 * [91/02/16 rpd]
109 *
110 * Revision 2.4 91/02/05 17:26:05 mrt
111 * Changed to new Mach copyright
112 * [91/02/01 16:11:53 mrt]
113 *
114 * Revision 2.3 91/01/08 15:15:36 rpd
115 * Added KEEP_STACKS support.
116 * [91/01/08 14:11:40 rpd]
117 *
118 * Replaced thread_doexception with new, optimized exception path.
119 * [90/12/22 rpd]
120 *
121 * Revision 2.2 90/06/02 14:53:44 rpd
122 * Converted to new IPC.
123 * [90/03/26 22:04:29 rpd]
124 *
125 *
126 * Condensed history:
127 * Changed thread_doexception to return boolean (dbg).
128 * Removed non-MACH code (dbg).
129 * Added thread_exception_abort (dlb).
130 * Use port_alloc, object_copyout (rpd).
131 * Removed exception-port routines (dbg).
132 * Check for thread halt condition if exception rpc fails (dlb).
133 * Switch to master before calling uprintf and exit (dbg).
134 * If rpc fails in thread_doexception, leave ports alone (dlb).
135 * Translate global port names to local port names (dlb).
136 * Rewrote for exc interface (dlb).
137 * Created (dlb).
138 */
139
140 #include <norma_ipc.h>
141 #include <mach_kdb.h>
142
143 #include <mach/boolean.h>
144 #include <mach/kern_return.h>
145 #include <mach/message.h>
146 #include <mach/port.h>
147 #include <mach/mig_errors.h>
148 #include <ipc/port.h>
149 #include <ipc/ipc_entry.h>
150 #include <ipc/ipc_notify.h>
151 #include <ipc/ipc_object.h>
152 #include <ipc/ipc_space.h>
153 #include <ipc/ipc_port.h>
154 #include <ipc/ipc_pset.h>
155 #include <ipc/mach_msg.h>
156 #include <ipc/ipc_machdep.h>
157 #include <kern/counters.h>
158 #include <kern/ipc_tt.h>
159 #include <kern/memory.h>
160 #include <kern/task.h>
161 #include <kern/thread.h>
162 #include <kern/processor.h>
163 #include <kern/sched_prim.h>
164
165 #include <mach/machine/vm_types.h>
166
167
168 /*
169 * Forward declarations
170 */
171
172 no_return exception(
173 integer_t _exception,
174 integer_t code,
175 integer_t subcode);
176 no_return exception_try_task(
177 integer_t _exception,
178 integer_t code,
179 integer_t subcode);
180 no_return exception_no_server(void);
181
182 no_return
183 exception_raise(
184 ipc_port_t dest_port,
185 ipc_port_t thread_port,
186 ipc_port_t task_port,
187 integer_t _exception,
188 integer_t code,
189 integer_t subcode);
190
191 kern_return_t exception_parse_reply(
192 ipc_kmsg_t kmsg);
193
194 no_return exception_raise_continue(void);
195 no_return exception_raise_continue_slow(
196 mach_msg_return_t mr,
197 ipc_kmsg_t kmsg,
198 mach_port_seqno_t seqno);
199 no_return exception_raise_continue_fast(
200 ipc_port_t reply_port,
201 ipc_kmsg_t kmsg);
202
203 #if MACH_KDB
204 #include <ddb/db_output.h>
205
206 extern void thread_kdb_return(void);
207 boolean_t debug_user_with_kdb = FALSE;
208 #endif /* MACH_KDB */
209
210 /*
211 * Routine: exception
212 * Purpose:
213 * The current thread caught an exception.
214 * We make an up-call to the thread's exception server.
215 * Conditions:
216 * Nothing locked and no resources held.
217 * Called from an exception context, so
218 * thread_exception_return and thread_kdb_return
219 * are possible.
220 * Returns:
221 * Doesn't return.
222 */
223
224 no_return
225 exception(
226 integer_t _exception,
227 integer_t code,
228 integer_t subcode)
229 {
230 register ipc_thread_t self = current_thread();
231 register ipc_port_t exc_port;
232
233 if (_exception == KERN_SUCCESS)
234 panic("exception");
235
236 /*
237 * Optimized version of retrieve_thread_exception.
238 */
239
240 thread_lock(self);
241 assert(self->active);
242 exc_port = self->ith_exception;
243 if (!IP_VALID(exc_port)) {
244 thread_unlock(self);
245 exception_try_task(_exception, code, subcode);
246 /*NOTREACHED*/
247 }
248
249 ip_lock(exc_port);
250 thread_unlock(self);
251 if (!ip_active(exc_port)) {
252 ip_unlock(exc_port);
253 exception_try_task(_exception, code, subcode);
254 /*NOTREACHED*/
255 }
256
257 /*
258 * Make a naked send right for the exception port.
259 */
260
261 ip_reference(exc_port);
262 exc_port->ip_srights++;
263 ip_unlock(exc_port);
264
265 /*
266 * If this exception port doesn't work,
267 * we will want to try the task's exception port.
268 * Indicate this by saving the exception state.
269 */
270
271 self->ith_exc = _exception;
272 self->ith_exc_code = code;
273 self->ith_exc_subcode = subcode;
274
275 exception_raise(exc_port,
276 retrieve_thread_self_fast(self),
277 retrieve_task_self_fast(self->task),
278 _exception, code, subcode);
279 /*NOTREACHED*/
280 }
281
282 /*
283 * Routine: exception_try_task
284 * Purpose:
285 * The current thread caught an exception.
286 * We make an up-call to the task's exception server.
287 * Conditions:
288 * Nothing locked and no resources held.
289 * Called from an exception context, so
290 * thread_exception_return and thread_kdb_return
291 * are possible.
292 * Returns:
293 * Doesn't return.
294 */
295
296 no_return
297 exception_try_task(
298 integer_t _exception,
299 integer_t code,
300 integer_t subcode)
301 {
302 ipc_thread_t self = current_thread();
303 register task_t task = self->task;
304 register ipc_port_t exc_port;
305
306 /*
307 * Optimized version of retrieve_task_exception.
308 */
309
310 task_lock(task);
311 assert(task->active);
312 exc_port = task->itk_exception;
313 if (!IP_VALID(exc_port)) {
314 task_unlock(task);
315 exception_no_server();
316 /*NOTREACHED*/
317 }
318
319 ip_lock(exc_port);
320 task_unlock(task);
321 if (!ip_active(exc_port)) {
322 ip_unlock(exc_port);
323 exception_no_server();
324 /*NOTREACHED*/
325 }
326
327 /*
328 * Make a naked send right for the exception port.
329 */
330
331 ip_reference(exc_port);
332 exc_port->ip_srights++;
333 ip_unlock(exc_port);
334
335 /*
336 * This is the thread's last chance.
337 * Clear the saved exception state.
338 */
339
340 self->ith_exc = KERN_SUCCESS;
341
342 exception_raise(exc_port,
343 retrieve_thread_self_fast(self),
344 retrieve_task_self_fast(task),
345 _exception, code, subcode);
346 /*NOTREACHED*/
347 }
348
349 /*
350 * Routine: exception_no_server
351 * Purpose:
352 * The current thread took an exception,
353 * and no exception server took responsibility
354 * for the exception. So good bye, charlie.
355 * Conditions:
356 * Nothing locked and no resources held.
357 * Called from an exception context, so
358 * thread_kdb_return is possible.
359 * Returns:
360 * Doesn't return.
361 */
362
363 no_return
364 exception_no_server(void)
365 {
366 register ipc_thread_t self = current_thread();
367
368 /*
369 * If this thread is being terminated, cooperate.
370 */
371
372 if (self->ast & AST_HALT) {
373 thread_halt_self(thread_exception_return);
374 /*NOTREACHED*/
375 }
376
377 if (self->ast & AST_TERMINATE) {
378 thread_terminate_self();
379 /*NOTREACHED*/
380 }
381
382 #if MACH_KDB
383 if (debug_user_with_kdb) {
384 /*
385 * Debug the exception with kdb.
386 * If kdb handles the exception,
387 * then thread_kdb_return won't return.
388 */
389
390 db_printf("No exception server, calling kdb...\n");
391 thread_kdb_return();
392 }
393 #endif /* MACH_KDB */
394
395 /*
396 * All else failed; terminate task.
397 */
398
399 (void) task_terminate(self->task);
400 thread_terminate_self();
401 /*NOTREACHED*/
402 }
403
404 #define MACH_EXCEPTION_ID 2400 /* from mach/exc.defs */
405 #define MACH_EXCEPTION_REPLY_ID (MACH_EXCEPTION_ID + 100)
406
407 struct mach_exception {
408 mach_msg_header_t Head;
409 mach_msg_type_t threadType;
410 mach_port_t thread;
411 mach_msg_type_t taskType;
412 mach_port_t task;
413 mach_msg_type_t exceptionType;
414 integer_t exception;
415 mach_msg_type_t codeType;
416 integer_t code;
417 mach_msg_type_t subcodeType;
418 integer_t subcode;
419 };
420
421 #define INTEGER_T_SIZE_IN_BITS (8 * sizeof(integer_t))
422 #define INTEGER_T_TYPE MACH_MSG_TYPE_INTEGER_T
423 /* in mach/machine/vm_types.h */
424
425 mach_msg_type_t exc_port_proto = {
426 /* msgt_name = */ MACH_MSG_TYPE_PORT_SEND,
427 /* msgt_size = */ PORT_T_SIZE_IN_BITS,
428 /* msgt_number = */ 1,
429 /* msgt_inline = */ TRUE,
430 /* msgt_longform = */ FALSE,
431 /* msgt_deallocate = */ FALSE,
432 /* msgt_unused = */ 0
433 };
434
435 mach_msg_type_t exc_code_proto = {
436 /* msgt_name = */ INTEGER_T_TYPE,
437 /* msgt_size = */ INTEGER_T_SIZE_IN_BITS,
438 /* msgt_number = */ 1,
439 /* msgt_inline = */ TRUE,
440 /* msgt_longform = */ FALSE,
441 /* msgt_deallocate = */ FALSE,
442 /* msgt_unused = */ 0
443 };
444
445 /*
446 * Routine: exception_raise
447 * Purpose:
448 * Make an exception_raise up-call to an exception server.
449 *
450 * dest_port must be a valid naked send right.
451 * thread_port and task_port are naked send rights.
452 * All three are always consumed.
453 *
454 * self->ith_exc, self->ith_exc_code, self->ith_exc_subcode
455 * must be appropriately initialized.
456 * Conditions:
457 * Nothing locked. We are being called in an exception context,
458 * so thread_exception_return may be called.
459 * Returns:
460 * Doesn't return.
461 */
462
463 int exception_raise_misses = 0;
464
465 no_return
466 exception_raise(
467 ipc_port_t dest_port,
468 ipc_port_t thread_port,
469 ipc_port_t task_port,
470 integer_t _exception,
471 integer_t code,
472 integer_t subcode)
473 {
474 ipc_thread_t self = current_thread();
475 ipc_thread_t receiver;
476 ipc_port_t reply_port;
477 ipc_mqueue_t dest_mqueue;
478 ipc_mqueue_t reply_mqueue;
479 ipc_kmsg_t kmsg;
480 mach_msg_return_t mr;
481
482 assert(IP_VALID(dest_port));
483
484 /*
485 * We will eventually need a message buffer.
486 * Grab the buffer now, while nothing is locked.
487 * This buffer will get handed to the exception server,
488 * and it will give the buffer back with its reply.
489 */
490
491 kmsg = ikm_cache();
492 if (kmsg != IKM_NULL) {
493 ikm_cache() = IKM_NULL;
494 ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
495 } else {
496 kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
497 if (kmsg == IKM_NULL)
498 panic("exception_raise");
499 ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
500 }
501
502 /*
503 * We need a reply port for the RPC.
504 * Check first for a cached port.
505 */
506
507 thread_lock(self);
508 assert(self->active);
509
510 reply_port = self->ith_rpc_reply;
511 if (reply_port == IP_NULL) {
512 thread_unlock(self);
513 reply_port = ipc_port_alloc_reply();
514 thread_lock(self);
515 if ((reply_port == IP_NULL) ||
516 (self->ith_rpc_reply != IP_NULL))
517 panic("exception_raise");
518 self->ith_rpc_reply = reply_port;
519 }
520
521 ip_lock(reply_port);
522 assert(ip_active(reply_port));
523 thread_unlock(self);
524
525 /*
526 * Make a naked send-once right for the reply port,
527 * to hand to the exception server.
528 * Make an extra reference for the reply port,
529 * to receive on. This protects us against
530 * mach_msg_abort_rpc.
531 */
532
533 reply_port->ip_sorights++;
534 ip_reference(reply_port);
535
536 ip_reference(reply_port);
537 self->ith_port = reply_port;
538
539 reply_mqueue = &reply_port->ip_messages;
540 imq_lock(reply_mqueue);
541 assert(ipc_kmsg_queue_empty(&reply_mqueue->imq_messages));
542 ip_unlock(reply_port);
543
544 /*
545 * Make sure we can queue to the destination port.
546 */
547
548 if (!ip_lock_try(dest_port)) {
549 imq_unlock(reply_mqueue);
550 goto slow_exception_raise;
551 }
552
553 if (!ip_active(dest_port) ||
554 #if NORMA_IPC
555 IP_NORMA_IS_PROXY(dest_port) ||
556 #endif /* NORMA_IPC */
557 (dest_port->ip_receiver == ipc_space_kernel)) {
558 imq_unlock(reply_mqueue);
559 ip_unlock(dest_port);
560 goto slow_exception_raise;
561 }
562
563 /*
564 * Find the destination message queue.
565 */
566
567 {
568 register ipc_pset_t dest_pset;
569
570 dest_pset = dest_port->ip_pset;
571 if (dest_pset == IPS_NULL)
572 dest_mqueue = &dest_port->ip_messages;
573 else
574 dest_mqueue = &dest_pset->ips_messages;
575 }
576
577 if (!imq_lock_try(dest_mqueue)) {
578 imq_unlock(reply_mqueue);
579 ip_unlock(dest_port);
580 goto slow_exception_raise;
581 }
582
583 /*
584 * Safe to unlock dest_port, because we hold
585 * dest_mqueue locked. We never bother changing
586 * dest_port->ip_msgcount.
587 */
588
589 ip_unlock(dest_port);
590
591 receiver = ipc_thread_queue_first(&dest_mqueue->imq_threads);
592 if ((receiver == ITH_NULL) ||
593 !((receiver->swap_func == mach_msg_continue) ||
594 ((receiver->swap_func == mach_msg_receive_continue) &&
595 (sizeof(struct mach_exception) <= receiver->ith_msize) &&
596 ((receiver->ith_option & MACH_RCV_NOTIFY) == 0))) ||
597 !thread_handoff(self, exception_raise_continue, receiver)) {
598 imq_unlock(reply_mqueue);
599 imq_unlock(dest_mqueue);
600 goto slow_exception_raise;
601 }
602 counter(c_exception_raise_block++);
603
604 assert(current_thread() == receiver);
605
606 /*
607 * We need to finish preparing self for its
608 * time asleep in reply_mqueue. self is left
609 * holding the extra ref for reply_port.
610 */
611
612 ipc_thread_enqueue_macro(&reply_mqueue->imq_threads, self);
613 self->ith_state = MACH_RCV_IN_PROGRESS;
614 self->ith_msize = MACH_MSG_SIZE_MAX;
615 imq_unlock(reply_mqueue);
616
617 /*
618 * Finish extracting receiver from dest_mqueue.
619 */
620
621 ipc_thread_rmqueue_first_macro(
622 &dest_mqueue->imq_threads, receiver);
623 imq_unlock(dest_mqueue);
624
625 /*
626 * Release the receiver's reference for his object.
627 */
628 {
629 register ipc_object_t object = receiver->ith_object;
630
631 io_lock(object);
632 io_release(object);
633 io_check_unlock(object);
634 }
635
636 {
637 register struct mach_exception *exc =
638 (struct mach_exception *) &kmsg->ikm_header;
639 ipc_space_t space = receiver->task->itk_space;
640
641 /*
642 * We are running as the receiver now. We hold
643 * the following resources, which must be consumed:
644 * kmsg, send-once right for reply_port
645 * send rights for dest_port, thread_port, task_port
646 * Synthesize a kmsg for copyout to the receiver.
647 */
648
649 exc->Head.msgh_bits = (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
650 MACH_MSG_TYPE_PORT_SEND) |
651 MACH_MSGH_BITS_COMPLEX);
652 exc->Head.msgh_size = sizeof *exc;
653 /* exc->Head.msgh_remote_port later */
654 /* exc->Head.msgh_local_port later */
655 exc->Head.msgh_seqno = 0;
656 exc->Head.msgh_id = MACH_EXCEPTION_ID;
657 exc->threadType = exc_port_proto;
658 /* exc->thread later */
659 exc->taskType = exc_port_proto;
660 /* exc->task later */
661 exc->exceptionType = exc_code_proto;
662 exc->exception = _exception;
663 exc->codeType = exc_code_proto;
664 exc->code = code;
665 exc->subcodeType = exc_code_proto;
666 exc->subcode = subcode;
667
668 /*
669 * Check that the receiver can handle the message.
670 */
671
672 if (receiver->ith_rcv_size < sizeof(struct mach_exception)) {
673 /*
674 * ipc_kmsg_destroy is a handy way to consume
675 * the resources we hold, but it requires setup.
676 */
677
678 exc->Head.msgh_bits =
679 (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
680 MACH_MSG_TYPE_PORT_SEND_ONCE) |
681 MACH_MSGH_BITS_COMPLEX);
682 exc->Head.msgh_remote_port = (mach_port_t) dest_port;
683 exc->Head.msgh_local_port = (mach_port_t) reply_port;
684 exc->thread = (mach_port_t) thread_port;
685 exc->task = (mach_port_t) task_port;
686
687 ipc_kmsg_destroy(kmsg);
688 thread_syscall_return(MACH_RCV_TOO_LARGE);
689 /*NOTREACHED*/
690 }
691
692 is_write_lock(space);
693 assert(space->is_active);
694
695 /*
696 * To do an atomic copyout, need simultaneous
697 * locks on both ports and the space.
698 */
699
700 ip_lock(dest_port);
701 if (!ip_active(dest_port) ||
702 !ip_lock_try(reply_port)) {
703 abort_copyout:
704 ip_unlock(dest_port);
705 is_write_unlock(space);
706
707 /*
708 * Oh well, we have to do the header the slow way.
709 * First make it look like it's in-transit.
710 */
711
712 exc->Head.msgh_bits =
713 (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
714 MACH_MSG_TYPE_PORT_SEND_ONCE) |
715 MACH_MSGH_BITS_COMPLEX);
716 exc->Head.msgh_remote_port = (mach_port_t) dest_port;
717 exc->Head.msgh_local_port = (mach_port_t) reply_port;
718
719 mr = ipc_kmsg_copyout_header(&exc->Head, space,
720 MACH_PORT_NULL);
721 if (mr == MACH_MSG_SUCCESS)
722 goto copyout_body;
723
724 /*
725 * Ack! Prepare for ipc_kmsg_copyout_dest.
726 * It will consume thread_port and task_port.
727 */
728
729 exc->thread = (mach_port_t) thread_port;
730 exc->task = (mach_port_t) task_port;
731
732 ipc_kmsg_copyout_dest(kmsg, space);
733 (void) ipc_kmsg_put(receiver->ith_msg, kmsg,
734 sizeof(mach_msg_header_t));
735 thread_syscall_return(mr);
736 /*NOTREACHED*/
737 }
738
739 if (!ip_active(reply_port)) {
740 ip_unlock(reply_port);
741 goto abort_copyout;
742 }
743
744 assert(reply_port->ip_sorights > 0);
745 ip_unlock(reply_port);
746
747 {
748 register ipc_entry_t table;
749 register ipc_entry_t entry;
750 register mach_port_index_t index;
751
752 /* optimized ipc_entry_get */
753
754 table = space->is_table;
755 index = table->ie_next;
756
757 if (index == 0)
758 goto abort_copyout;
759
760 entry = &table[index];
761 table->ie_next = entry->ie_next;
762 entry->ie_request = 0;
763
764 {
765 register mach_port_gen_t gen;
766
767 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
768 gen = entry->ie_bits + IE_BITS_GEN_ONE;
769
770 exc->Head.msgh_remote_port = MACH_PORT_MAKE(index, gen);
771
772 /* optimized ipc_right_copyout */
773
774 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
775 }
776
777 entry->ie_object = (ipc_object_t) reply_port;
778 is_write_unlock(space);
779 }
780
781 /* optimized ipc_object_copyout_dest */
782
783 assert(dest_port->ip_srights > 0);
784 ip_release(dest_port);
785
786 exc->Head.msgh_local_port =
787 ((dest_port->ip_receiver == space) ?
788 dest_port->ip_receiver_name : MACH_PORT_NULL);
789
790 if ((--dest_port->ip_srights == 0) &&
791 (dest_port->ip_nsrequest != IP_NULL)) {
792 ipc_port_t nsrequest;
793 mach_port_mscount_t mscount;
794
795 /* a rather rare case */
796
797 nsrequest = dest_port->ip_nsrequest;
798 mscount = dest_port->ip_mscount;
799 dest_port->ip_nsrequest = IP_NULL;
800 ip_unlock(dest_port);
801
802 ipc_notify_no_senders(nsrequest, mscount);
803 } else
804 ip_unlock(dest_port);
805
806 copyout_body:
807 /*
808 * Optimized version of ipc_kmsg_copyout_body,
809 * to handle the two ports in the body.
810 */
811
812 mr = (ipc_kmsg_copyout_object(space, (ipc_object_t) thread_port,
813 MACH_MSG_TYPE_PORT_SEND, &exc->thread) |
814 ipc_kmsg_copyout_object(space, (ipc_object_t) task_port,
815 MACH_MSG_TYPE_PORT_SEND, &exc->task));
816 if (mr != MACH_MSG_SUCCESS) {
817 (void) ipc_kmsg_put(receiver->ith_msg, kmsg,
818 kmsg->ikm_header.msgh_size);
819 thread_syscall_return(mr | MACH_RCV_BODY_ERROR);
820 /*NOTREACHED*/
821 }
822 }
823
824 /*
825 * Optimized version of ipc_kmsg_put.
826 * We must check ikm_cache after copyoutmsg.
827 */
828
829 ikm_check_initialized(kmsg, kmsg->ikm_size);
830 assert(kmsg->ikm_size == IKM_SAVED_KMSG_SIZE);
831
832 if (copyoutmsg(&kmsg->ikm_header,
833 receiver->ith_msg,
834 sizeof(struct mach_exception)) ||
835 (ikm_cache() != IKM_NULL)) {
836 mr = ipc_kmsg_put(receiver->ith_msg, kmsg,
837 kmsg->ikm_header.msgh_size);
838 thread_syscall_return(mr);
839 /*NOTREACHED*/
840 }
841
842 ikm_cache() = kmsg;
843 thread_syscall_return(MACH_MSG_SUCCESS);
844 /*NOTREACHED*/
845 #ifndef __GNUC__
846 return; /* help for the compiler */
847 #endif
848
849 slow_exception_raise: {
850 register struct mach_exception *exc =
851 (struct mach_exception *) &kmsg->ikm_header;
852 union ipc_kmsg_return reply_kmsg_ret;
853 mach_port_seqno_t reply_seqno;
854
855 exception_raise_misses++;
856
857 /*
858 * We hold the following resources, which must be consumed:
859 * kmsg, send-once right and ref for reply_port
860 * send rights for dest_port, thread_port, task_port
861 * Synthesize a kmsg to send.
862 */
863
864 exc->Head.msgh_bits = (MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
865 MACH_MSG_TYPE_PORT_SEND_ONCE) |
866 MACH_MSGH_BITS_COMPLEX);
867 exc->Head.msgh_size = sizeof *exc;
868 exc->Head.msgh_remote_port = (mach_port_t) dest_port;
869 exc->Head.msgh_local_port = (mach_port_t) reply_port;
870 exc->Head.msgh_seqno = 0;
871 exc->Head.msgh_id = MACH_EXCEPTION_ID;
872 exc->threadType = exc_port_proto;
873 exc->thread = (mach_port_t) thread_port;
874 exc->taskType = exc_port_proto;
875 exc->task = (mach_port_t) task_port;
876 exc->exceptionType = exc_code_proto;
877 exc->exception = _exception;
878 exc->codeType = exc_code_proto;
879 exc->code = code;
880 exc->subcodeType = exc_code_proto;
881 exc->subcode = subcode;
882
883 ipc_mqueue_send_always(kmsg);
884
885 /*
886 * We are left with a ref for reply_port,
887 * which we use to receive the reply message.
888 */
889
890 ip_lock(reply_port);
891 if (!ip_active(reply_port)) {
892 ip_unlock(reply_port);
893 exception_raise_continue_slow(MACH_RCV_PORT_DIED,
894 IKM_NULL,
895 /*dummy*/0);
896 /*NOTREACHED*/
897 }
898
899 imq_lock(reply_mqueue);
900 ip_unlock(reply_port);
901
902 mr = ipc_mqueue_receive(reply_mqueue, MACH_MSG_OPTION_NONE,
903 MACH_MSG_SIZE_MAX,
904 MACH_MSG_TIMEOUT_NONE,
905 FALSE, exception_raise_continue,
906 &reply_kmsg_ret, &reply_seqno);
907 /* reply_mqueue is unlocked */
908
909 exception_raise_continue_slow(mr, reply_kmsg_ret.kmsg, reply_seqno);
910 /*NOTREACHED*/
911 }
912 }
913
914 mach_msg_type_t exc_RetCode_proto = {
915 /* msgt_name = */ MACH_MSG_TYPE_INTEGER_32,
916 /* msgt_size = */ 32,
917 /* msgt_number = */ 1,
918 /* msgt_inline = */ TRUE,
919 /* msgt_longform = */ FALSE,
920 /* msgt_deallocate = */ FALSE,
921 /* msgt_unused = */ 0
922 };
923
924 /*
925 * Routine: exception_parse_reply
926 * Purpose:
927 * Parse and consume an exception reply message.
928 * Conditions:
929 * The destination port right has already been consumed.
930 * The message buffer and anything else in it is consumed.
931 * Returns:
932 * The reply return code.
933 */
934
935 kern_return_t
936 exception_parse_reply(
937 ipc_kmsg_t kmsg)
938 {
939 register mig_reply_header_t *msg =
940 (mig_reply_header_t *) &kmsg->ikm_header;
941 kern_return_t kr;
942
943 if ((msg->Head.msgh_bits !=
944 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0)) ||
945 (msg->Head.msgh_size != sizeof *msg) ||
946 (msg->Head.msgh_id != MACH_EXCEPTION_REPLY_ID) ||
947 (* (int *) &msg->RetCodeType != * (int *) &exc_RetCode_proto)) {
948 /*
949 * Bozo user sent us a misformatted reply.
950 */
951
952 kmsg->ikm_header.msgh_remote_port = MACH_PORT_NULL;
953 ipc_kmsg_destroy(kmsg);
954 return MIG_REPLY_MISMATCH;
955 }
956
957 kr = msg->RetCode;
958
959 if ((kmsg->ikm_size == IKM_SAVED_KMSG_SIZE) &&
960 (ikm_cache() == IKM_NULL))
961 ikm_cache() = kmsg;
962 else
963 ikm_free(kmsg);
964
965 return kr;
966 }
967
968 /*
969 * Routine: exception_raise_continue
970 * Purpose:
971 * Continue after blocking for an exception.
972 * Conditions:
973 * Nothing locked. We are running on a new kernel stack,
974 * with the exception state saved in the thread. From here
975 * control goes back to user space.
976 * Returns:
977 * Doesn't return.
978 */
979
980 no_return
981 exception_raise_continue(void)
982 {
983 ipc_thread_t self = current_thread();
984 ipc_port_t reply_port = self->ith_port;
985 ipc_mqueue_t reply_mqueue = &reply_port->ip_messages;
986 union ipc_kmsg_return kmsg_ret;
987 mach_port_seqno_t seqno;
988 mach_msg_return_t mr;
989
990 mr = ipc_mqueue_receive(reply_mqueue, MACH_MSG_OPTION_NONE,
991 MACH_MSG_SIZE_MAX,
992 MACH_MSG_TIMEOUT_NONE,
993 TRUE, exception_raise_continue,
994 &kmsg_ret, &seqno);
995 /* reply_mqueue is unlocked */
996
997 exception_raise_continue_slow(mr, kmsg_ret.kmsg, seqno);
998 /*NOTREACHED*/
999 }
1000
1001 /*
1002 * Routine: exception_raise_continue_slow
1003 * Purpose:
1004 * Continue after finishing an ipc_mqueue_receive
1005 * for an exception reply message.
1006 * Conditions:
1007 * Nothing locked. We hold a ref for reply_port.
1008 * Returns:
1009 * Doesn't return.
1010 */
1011
1012 no_return
1013 exception_raise_continue_slow(
1014 mach_msg_return_t mr,
1015 ipc_kmsg_t kmsg,
1016 mach_port_seqno_t seqno)
1017 {
1018 ipc_thread_t self = current_thread();
1019 ipc_port_t reply_port = self->ith_port;
1020 ipc_mqueue_t reply_mqueue = &reply_port->ip_messages;
1021
1022 while (mr == MACH_RCV_INTERRUPTED) {
1023 /*
1024 * Somebody is trying to force this thread
1025 * to a clean point. We must cooperate
1026 * and then resume the receive.
1027 */
1028 union ipc_kmsg_return kmsg_ret;
1029
1030 if (self->ast & AST_HALT) {
1031 thread_halt_self(CONTINUE_NULL);
1032 }
1033
1034 if (self->ast & AST_TERMINATE) {
1035 /* don't terminate while holding a reference */
1036 ipc_port_release(reply_port);
1037 thread_terminate_self();
1038 /*NOTREACHED*/
1039 }
1040
1041 ip_lock(reply_port);
1042 if (!ip_active(reply_port)) {
1043 ip_unlock(reply_port);
1044 mr = MACH_RCV_PORT_DIED;
1045 break;
1046 }
1047
1048 imq_lock(reply_mqueue);
1049 ip_unlock(reply_port);
1050
1051 mr = ipc_mqueue_receive(reply_mqueue, MACH_MSG_OPTION_NONE,
1052 MACH_MSG_SIZE_MAX,
1053 MACH_MSG_TIMEOUT_NONE,
1054 FALSE, exception_raise_continue,
1055 &kmsg_ret, &seqno);
1056 /* reply_mqueue is unlocked */
1057 kmsg = kmsg_ret.kmsg;
1058 }
1059 ipc_port_release(reply_port);
1060
1061 assert((mr == MACH_MSG_SUCCESS) ||
1062 (mr == MACH_RCV_PORT_DIED));
1063
1064 if (mr == MACH_MSG_SUCCESS) {
1065 /*
1066 * Consume the reply message.
1067 */
1068
1069 ipc_port_release_sonce(reply_port);
1070 mr = exception_parse_reply(kmsg);
1071 }
1072
1073 if ((mr == KERN_SUCCESS) ||
1074 (mr == MACH_RCV_PORT_DIED)) {
1075 thread_exception_return();
1076 /*NOTREACHED*/
1077 }
1078
1079 if (self->ith_exc != KERN_SUCCESS) {
1080 exception_try_task(self->ith_exc,
1081 self->ith_exc_code,
1082 self->ith_exc_subcode);
1083 /*NOTREACHED*/
1084 }
1085
1086 exception_no_server();
1087 /*NOTREACHED*/
1088 }
1089
1090 /*
1091 * Routine: exception_raise_continue_fast
1092 * Purpose:
1093 * Special-purpose fast continuation for exceptions.
1094 * Conditions:
1095 * reply_port is locked and alive.
1096 * kmsg is our reply message.
1097 * Returns:
1098 * Doesn't return.
1099 */
1100
1101 no_return
1102 exception_raise_continue_fast(
1103 ipc_port_t reply_port,
1104 ipc_kmsg_t kmsg)
1105 {
1106 ipc_thread_t self = current_thread();
1107 kern_return_t kr;
1108
1109 assert(ip_active(reply_port));
1110 assert(reply_port == self->ith_port);
1111 assert(reply_port == (ipc_port_t) kmsg->ikm_header.msgh_remote_port);
1112 assert(MACH_MSGH_BITS_REMOTE(kmsg->ikm_header.msgh_bits) ==
1113 MACH_MSG_TYPE_PORT_SEND_ONCE);
1114
1115 /*
1116 * Release the send-once right (from the message header)
1117 * and the saved reference (from self->ith_port).
1118 */
1119
1120 reply_port->ip_sorights--;
1121 ip_release(reply_port);
1122 ip_release(reply_port);
1123 ip_unlock(reply_port);
1124
1125 /*
1126 * Consume the reply message.
1127 */
1128
1129 kr = exception_parse_reply(kmsg);
1130 if (kr == KERN_SUCCESS) {
1131 thread_exception_return();
1132 /*NOTREACHED*/
1133 }
1134
1135 if (self->ith_exc != KERN_SUCCESS) {
1136 exception_try_task(self->ith_exc,
1137 self->ith_exc_code,
1138 self->ith_exc_subcode);
1139 /*NOTREACHED*/
1140 }
1141
1142 exception_no_server();
1143 /*NOTREACHED*/
1144 }
Cache object: d53cb522533ea155195a89e319cd7b07
|