FreeBSD/Linux Kernel Cross Reference
sys/ipc/ipc_kmsg.c

     /* 
      * Mach Operating System
      * Copyright (c) 1993-1989 Carnegie Mellon University
      * All Rights Reserved.
      * 
      * Permission to use, copy, modify and distribute this software and its
      * documentation is hereby granted, provided that both the copyright
      * notice and this permission notice appear in all copies of the
      * software, derivative works or modified versions, and any portions
     * thereof, and that both notices appear in supporting documentation.
     * 
     * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
     * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
     * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
     * 
     * Carnegie Mellon requests users of this software to return to
     * 
     *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
     *  School of Computer Science
     *  Carnegie Mellon University
     *  Pittsburgh PA 15213-3890
     * 
     * any improvements or extensions that they make and grant Carnegie Mellon
     * the rights to redistribute these changes.
     */
    /*
     * HISTORY
     * $Log:        ipc_kmsg.c,v $
     * Revision 2.33  93/11/17  16:57:14  dbg
     *      Added kmsg type for kernel reply messages.  Added ANSI
     *      function prototypes.
     *      [93/03/29            dbg]
     * 
     * Revision 2.32  93/08/31  15:16:33  mrt
     *      Defend against the case of a long type that is mis-aligned.
     *      This can be used (horrible fix, sorry) to solve the
     *      issue of what to put in the filler spot of a troublesome
     *      message.  This happens, for instance, with varsize inline
     *      arrays followed by OOL data described by a long type.
     *      One can use a bogus short-type that has the longform bit on...
     *      [93/08/16            af]
     * 
     * Revision 2.31  93/08/10  15:15:52  mrt
     *      Religious-fix for GCC on alpha.  This compiler insists that
     *      if a structure contains a long then it must be aligned and
     *      also all fields in it must be.  So our trick with short and
     *      long type descriptors would not play quite right, some 64bit
     *      loads were generated for 32bit quantities.
     *      Put casts all over the place for this.
     *      [93/08/04            af]
     * 
     * Revision 2.30  93/01/14  17:32:42  danner
     *      64bit cleanup
     *      [92/11/30            af]
     * 
     * Revision 2.29  92/08/03  17:34:23  jfriedl
     *      removed silly prototypes
     *      [92/08/02            jfriedl]
     * 
     * Revision 2.28  92/05/21  17:10:21  jfriedl
     *      Added void to fcns that needed it.  Renamed one of the two local
     *      variables 'object' to 'obj' in ipc_kmsg_clean_partial to keep clean.
     *      Added return after error in ipc_msg_print().
     *      [92/05/16            jfriedl]
     * 
     * Revision 2.27  92/03/10  16:25:43  jsb
     *      Merged in norma branch changes as of NORMA_MK7:
     *      Moved ipc_kmsg_copyout_to_network to norma/ipc_output.c.
     *      Moved ipc_kmsg_uncopyout_to_network to norma/ipc_clean.c.
     *      [92/02/21  10:34:46  jsb]
     *      We no longer convert to network format directly from user format;
     *      this greatly reduces the number of NORMA_IPC conditionals here.
     *      Added code to detect and recover from vm_map_convert_to_page_list
     *      failure. Streamlined and fixed ipc_kmsg_copyout_to_network.
     *      [92/02/21  09:01:52  jsb]
     * 
     * Revision 2.26  92/03/05  18:53:54  rpd
     *      Undid norma changes that were inadvertently picked up in 2.24.
     *      [92/03/05            rpd]
     * 
     * Revision 2.25  92/03/01  15:14:54  rpd
     *      Picked up page-list/norma code for ipc_kmsg_copyin_compat, from jsb.
     *      [92/03/01            rpd]
     * 
     * Revision 2.24  92/02/23  19:52:31  elf
     *      Eliminate keep_wired argument from vm_map_copyin().
     * 
     * Revision 2.23  92/01/14  16:44:23  rpd
     *      Fixed ipc_kmsg_copyin, ipc_kmsg_copyout, etc
     *      to use copyinmap and copyoutmap for out-of-line ports.
     *      [91/12/16            rpd]
     * 
     * Revision 2.22  91/12/15  10:37:53  jsb
     *      Improved ddb 'show kmsg' support.
     * 
     * Revision 2.21  91/12/14  14:26:03  jsb
     *      Removed ipc_fields.h hack.
     *      Made ipc_kmsg_clean_{body,partial} aware of remote ports.
     *      They don't yet clean up remote ports, but at least they
    *      no longer pass port uids to ipc_object_destroy.
    * 
    * Revision 2.20  91/12/13  13:51:58  jsb
    *      Use norma_ipc_copyin_page_list when sending to remote port.
    * 
    * Revision 2.19  91/12/10  13:25:46  jsb
    *      Added ipc_kmsg_copyout_to_network, as required by ipc_kserver.c.
    *      Picked up vm_map_convert_to_page_list call changes from dlb.
    *      Changed NORMA_VM conditional for ipc_kmsg_copyout_to_kernel
    *      to NORMA_IPC.
    *      [91/12/10  11:20:36  jsb]
    * 
    * Revision 2.18  91/11/14  16:55:57  rpd
    *      Picked up mysterious norma changes.
    *      [91/11/14            rpd]
    * 
    * Revision 2.17  91/10/09  16:09:08  af
    *      Changed msgh_kind to msgh_seqno in ipc_msg_print.
    *      [91/10/05            rpd]
    * 
    *      Removed unused variables.  Added <ipc/ipc_notify.h>.
    *      [91/09/02            rpd]
    * 
    * Revision 2.16  91/08/28  11:13:20  jsb
    *      Changed msgh_kind to msgh_seqno.
    *      [91/08/09            rpd]
    *      Changed for new vm_map_copyout failure behavior.
    *      [91/08/03            rpd]
    *      Update page list discriminant logic to allow use of page list for
    *      kernel objects that do not require page stealing (devices).
    *      [91/07/31  15:00:55  dlb]
    * 
    *      Add arg to vm_map_copyin_page_list.
    *      [91/07/30  14:10:38  dlb]
    * 
    *      Turn page lists on by default.
    *      [91/07/03  14:01:00  dlb]
    *      Renamed clport fields in struct ipc_port to ip_norma fields.
    *      Added checks for sending receive rights remotely.
    *      [91/08/15  08:22:20  jsb]
    * 
    * Revision 2.15  91/08/03  18:18:16  jsb
    *      Added support for ddb commands ``show msg'' and ``show kmsg''.
    *      Made changes for elimination of intermediate clport structure.
    *      [91/07/27  22:25:06  jsb]
    * 
    *      Moved MACH_MSGH_BITS_COMPLEX_{PORTS,DATA} to mach/message.h.
    *      Removed complex_data_hint_xxx[] garbage.
    *      Adopted new vm_map_copy_t page_list technology.
    *      [91/07/04  13:09:45  jsb]
    * 
    * Revision 2.14  91/07/01  08:24:34  jsb
    *      From David Black at OSF: generalized page list support.
    *      [91/06/29  16:29:29  jsb]
    * 
    * Revision 2.13  91/06/17  15:46:04  jsb
    *      Renamed NORMA conditionals.
    *      [91/06/17  10:45:05  jsb]
    * 
    * Revision 2.12  91/06/06  17:05:52  jsb
    *      More NORMA_IPC stuff. Cleanup will follow.
    *      [91/06/06  16:00:08  jsb]
    * 
    * Revision 2.11  91/05/14  16:33:01  mrt
    *      Correcting copyright
    * 
    * Revision 2.10  91/03/16  14:47:57  rpd
    *      Replaced ith_saved with ipc_kmsg_cache.
    *      [91/02/16            rpd]
    * 
    * Revision 2.9  91/02/05  17:21:52  mrt
    *      Changed to new Mach copyright
    *      [91/02/01  15:45:30  mrt]
    * 
    * Revision 2.8  91/01/08  15:13:49  rpd
    *      Added ipc_kmsg_free.
    *      [91/01/05            rpd]
    *      Optimized ipc_kmsg_copyout_object for send rights.
    *      [90/12/21            rpd]
    *      Changed to use new copyinmsg/copyoutmsg operations.
    *      Changed ipc_kmsg_get to check that the size is multiple of four.
    *      [90/12/05            rpd]
    *      Removed MACH_IPC_GENNOS.
    *      [90/11/08            rpd]
    * 
    * Revision 2.7  90/11/05  14:28:36  rpd
    *      Changed ip_reference to ipc_port_reference.
    *      Changed ip_release to ipc_port_release.
    *      Use new io_reference and io_release.
    *      Use new ip_reference and ip_release.
    *      [90/10/29            rpd]
    * 
    * Revision 2.6  90/09/09  14:31:50  rpd
    *      Fixed ipc_kmsg_copyin_compat to clear unused bits instead
    *      of returning an error when they are non-zero.
    *      [90/09/08            rpd]
    * 
    * Revision 2.5  90/08/06  17:05:53  rpd
    *      Fixed ipc_kmsg_copyout_body to turn off msgt_deallocate
    *      for in-line data.  It might be on if the compatibility mode
    *      generated the message.
    * 
    *      Fixed ipc_kmsg_copyin, ipc_kmsg_copyin_compat to check
    *      that msgt_name, msgt_size, msgt_number are zero
    *      in long-form type descriptors.
    *      [90/08/04            rpd]
    * 
    *      Fixed atomicity bug in ipc_kmsg_copyout_header,
    *      when the destination and reply ports are the same.
    *      [90/08/02            rpd]
    * 
    * Revision 2.4  90/08/06  15:07:31  rwd
    *      Fixed ipc_kmsg_clean_partial to deallocate correctly
    *      the OOL memory in the last type spec.
    *      Removed debugging panic in ipc_kmsg_put.
    *      [90/06/21            rpd]
    * 
    * Revision 2.3  90/06/19  22:58:03  rpd
    *      For debugging: added panic to ipc_kmsg_put.
    *      [90/06/04            rpd]
    * 
    * Revision 2.2  90/06/02  14:50:05  rpd
    *      Changed ocurrences of inline; it is a gcc keyword.
    *      [90/06/02            rpd]
    * 
    *      For out-of-line memory, if length is zero allow any address.
    *      This is more compatible with old IPC.
    *      [90/04/23            rpd]
    *      Created for new IPC.
    *      [90/03/26  20:55:45  rpd]
    * 
    */
   /*
    *      File:   ipc/ipc_kmsg.c
    *      Author: Rich Draves
    *      Date:   1989
    *
    *      Operations on kernel messages.
    */
   
   #include <cpus.h>
   #include <mach_ipc_compat.h>
   #include <norma_ipc.h>
   #include <norma_vm.h>
   
   #include <mach/boolean.h>
   #include <mach/kern_return.h>
   #include <mach/message.h>
   #include <mach/port.h>
   #include <kern/assert.h>
   #include <kern/kalloc.h>
   #include <kern/kern_kmsg.h>
   #include <kern/memory.h>
   #include <vm/vm_map.h>
   #include <vm/vm_object.h>
   #include <vm/vm_kern.h>
   #include <vm/vm_user.h>
   #include <ipc/port.h>
   #include <ipc/ipc_entry.h>
   #include <ipc/ipc_kmsg.h>
   #include <ipc/ipc_hash.h>
   #include <ipc/ipc_thread.h>
   #include <ipc/ipc_marequest.h>
   #include <ipc/ipc_notify.h>
   #include <ipc/ipc_object.h>
   #include <ipc/ipc_space.h>
   #include <ipc/ipc_port.h>
   #include <ipc/ipc_right.h>
   
   #include <device/net_io.h>      /* net_kmsg_put */
   
   #include <ipc/ipc_machdep.h>
   
   extern int copyinmap();
   extern int copyoutmap();
   void ipc_msg_print(); /* forward */
   
   #define is_misaligned(x)        ( ((vm_offset_t)(x)) & (sizeof(vm_offset_t)-1) )
   #define ptr_align(x)    \
           ( ( ((vm_offset_t)(x)) + (sizeof(vm_offset_t)-1) ) & ~(sizeof(vm_offset_t)-1) )
   
   ipc_kmsg_t ipc_kmsg_cache[NCPUS];
   
   /*
    *      Routine:        ipc_kmsg_enqueue
    *      Purpose:
    *              Enqueue a kmsg.
    */
   
   void
   ipc_kmsg_enqueue(
           ipc_kmsg_queue_t queue,
           ipc_kmsg_t kmsg)
   {
           ipc_kmsg_enqueue_macro(queue, kmsg);
   }
   
   /*
    *      Routine:        ipc_kmsg_dequeue
    *      Purpose:
    *              Dequeue and return a kmsg.
    */
   
   ipc_kmsg_t
   ipc_kmsg_dequeue(
           ipc_kmsg_queue_t queue)
   {
           ipc_kmsg_t first;
   
           first = ipc_kmsg_queue_first(queue);
   
           if (first != IKM_NULL)
                   ipc_kmsg_rmqueue_first_macro(queue, first);
   
           return first;
   }
   
   /*
    *      Routine:        ipc_kmsg_rmqueue
    *      Purpose:
    *              Pull a kmsg out of a queue.
    */
   
   void
   ipc_kmsg_rmqueue(
           ipc_kmsg_queue_t queue,
           ipc_kmsg_t kmsg)
   {
           ipc_kmsg_t next, prev;
   
           assert(queue->ikmq_base != IKM_NULL);
   
           next = kmsg->ikm_next;
           prev = kmsg->ikm_prev;
   
           if (next == kmsg) {
                   assert(prev == kmsg);
                   assert(queue->ikmq_base == kmsg);
   
                   queue->ikmq_base = IKM_NULL;
           } else {
                   if (queue->ikmq_base == kmsg)
                           queue->ikmq_base = next;
   
                   next->ikm_prev = prev;
                   prev->ikm_next = next;
           }
   }
   
   /*
    *      Routine:        ipc_kmsg_queue_next
    *      Purpose:
    *              Return the kmsg following the given kmsg.
    *              (Or IKM_NULL if it is the last one in the queue.)
    */
   
   ipc_kmsg_t
   ipc_kmsg_queue_next(
           ipc_kmsg_queue_t queue,
           ipc_kmsg_t kmsg)
   {
           ipc_kmsg_t next;
   
           assert(queue->ikmq_base != IKM_NULL);
   
           next = kmsg->ikm_next;
           if (queue->ikmq_base == next)
                   next = IKM_NULL;
   
           return next;
   }
   
   /*
    *      Routine:        ipc_kmsg_destroy
    *      Purpose:
    *              Destroys a kernel message.  Releases all rights,
    *              references, and memory held by the message.
    *              Frees the message.
    *      Conditions:
    *              No locks held.
    */
   
   void
   ipc_kmsg_destroy(
           ipc_kmsg_t kmsg)
   {
           ipc_kmsg_queue_t queue;
           boolean_t empty;
   
           /*
            *      ipc_kmsg_clean can cause more messages to be destroyed.
            *      Curtail recursion by queueing messages.  If a message
            *      is already queued, then this is a recursive call.
            */
   
           queue = &current_thread()->ith_messages;
           empty = ipc_kmsg_queue_empty(queue);
           ipc_kmsg_enqueue(queue, kmsg);
   
           if (empty) {
                   /* must leave kmsg in queue while cleaning it */
   
                   while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) {
                           ipc_kmsg_clean(kmsg);
                           ipc_kmsg_rmqueue(queue, kmsg);
                           ikm_free(kmsg);
                   }
           }
   }
   
   /*
    *      Routine:        ipc_kmsg_clean_body
    *      Purpose:
    *              Cleans the body of a kernel message.
    *              Releases all rights, references, and memory.
    *
    *              The last type/data pair might stretch past eaddr.
    *              (See the usage in ipc_kmsg_copyout.)
    *      Conditions:
    *              No locks held.
    */
   
   void
   ipc_kmsg_clean_body(
           vm_offset_t saddr,
           vm_offset_t eaddr)
   {
           while (saddr < eaddr) {
                   mach_msg_type_long_t *type;
                   mach_msg_type_name_t name;
                   mach_msg_type_size_t size;
                   mach_msg_type_number_t number;
                   boolean_t is_inline, is_port;
                   vm_size_t length;
   
                   type = (mach_msg_type_long_t *) saddr;
                   is_inline = ((mach_msg_type_t*)type)->msgt_inline;
                   if (((mach_msg_type_t*)type)->msgt_longform) {
                           /* This must be aligned */
                           if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                               (is_misaligned(type))) {
                                   saddr = ptr_align(saddr);
                                   continue;
                           }
                           name = type->msgtl_name;
                           size = type->msgtl_size;
                           number = type->msgtl_number;
                           saddr += sizeof(mach_msg_type_long_t);
                   } else {
                           name = ((mach_msg_type_t*)type)->msgt_name;
                           size = ((mach_msg_type_t*)type)->msgt_size;
                           number = ((mach_msg_type_t*)type)->msgt_number;
                           saddr += sizeof(mach_msg_type_t);
                   }
   
                   /* padding (ptrs and ports) ? */
                   if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                       ((size >> 3) == sizeof(natural_t)))
                           saddr = ptr_align(saddr);
   
                   /* calculate length of data in bytes, rounding up */
   
                   length = ((number * size) + 7) >> 3;
   
                   is_port = MACH_MSG_TYPE_PORT_ANY(name);
   
                   if (is_port) {
                           ipc_object_t *objects;
                           mach_msg_type_number_t i;
   
                           if (is_inline) {
                                   objects = (ipc_object_t *) saddr;
                                   /* sanity check */
                                   while (eaddr < (vm_offset_t)&objects[number])
                                           number--;
                           } else {
                                   objects = (ipc_object_t *)
                                                   * (vm_offset_t *) saddr;
                           }
   
                           /* destroy port rights carried in the message */
   
                           for (i = 0; i < number; i++) {
                                   ipc_object_t object = objects[i];
   
                                   if (!IO_VALID(object))
                                           continue;
   
                                   ipc_object_destroy(object, name);
                           }
                   }
   
                   if (is_inline) {
                           /* inline data sizes round up to int boundaries */
   
                           saddr += (length + 3) &~ 3;
                   } else {
                           vm_offset_t data = * (vm_offset_t *) saddr;
   
                           /* destroy memory carried in the message */
   
                           if (length == 0)
                                   assert(data == 0);
                           else if (is_port)
                                   kfree(data, length);
                           else
                                   vm_map_copy_discard((vm_map_copy_t) data);
   
                           saddr += sizeof(vm_offset_t);
                   }
           }
   }
   
   /*
    *      Routine:        ipc_kmsg_clean
    *      Purpose:
    *              Cleans a kernel message.  Releases all rights,
    *              references, and memory held by the message.
    *      Conditions:
    *              No locks held.
    */
   
   void
   ipc_kmsg_clean(
           ipc_kmsg_t kmsg)
   {
           ipc_marequest_t marequest;
           ipc_object_t object;
           mach_msg_bits_t mbits = kmsg->ikm_header.msgh_bits;
   
           marequest = kmsg->ikm_marequest;
           if (marequest != IMAR_NULL)
                   ipc_marequest_destroy(marequest);
   
           object = (ipc_object_t) kmsg->ikm_header.msgh_remote_port;
           if (IO_VALID(object))
                   ipc_object_destroy(object, MACH_MSGH_BITS_REMOTE(mbits));
   
           object = (ipc_object_t) kmsg->ikm_header.msgh_local_port;
           if (IO_VALID(object))
                   ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
   
           if (mbits & MACH_MSGH_BITS_COMPLEX) {
                   vm_offset_t saddr, eaddr;
   
                   saddr = (vm_offset_t) (&kmsg->ikm_header + 1);
                   eaddr = (vm_offset_t) &kmsg->ikm_header +
                                   kmsg->ikm_header.msgh_size;
   
                   ipc_kmsg_clean_body(saddr, eaddr);
           }
   }
   
   /*
    *      Routine:        ipc_kmsg_clean_partial
    *      Purpose:
    *              Cleans a partially-acquired kernel message.
    *              eaddr is the address of the type specification
    *              in the body of the message that contained the error.
    *              If dolast, the memory and port rights in this last
    *              type spec are also cleaned.  In that case, number
    *              specifies the number of port rights to clean.
    *      Conditions:
    *              Nothing locked.
    */
   
   void
   ipc_kmsg_clean_partial(
           ipc_kmsg_t      kmsg,
           vm_offset_t     eaddr,
           boolean_t       dolast,
           mach_msg_type_number_t number)
   {
           ipc_object_t object;
           mach_msg_bits_t mbits = kmsg->ikm_header.msgh_bits;
           vm_offset_t saddr;
   
           assert(kmsg->ikm_marequest == IMAR_NULL);
   
           object = (ipc_object_t) kmsg->ikm_header.msgh_remote_port;
           assert(IO_VALID(object));
           ipc_object_destroy(object, MACH_MSGH_BITS_REMOTE(mbits));
   
           object = (ipc_object_t) kmsg->ikm_header.msgh_local_port;
           if (IO_VALID(object))
                   ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits));
   
           saddr = (vm_offset_t) (&kmsg->ikm_header + 1);
           ipc_kmsg_clean_body(saddr, eaddr);
   
           if (dolast) {
                   mach_msg_type_long_t *type;
                   mach_msg_type_name_t name;
                   mach_msg_type_size_t size;
                   mach_msg_type_number_t rnumber;
                   boolean_t is_inline, is_port;
                   vm_size_t length;
   
   xxx:            type = (mach_msg_type_long_t *) eaddr;
                   is_inline = ((mach_msg_type_t*)type)->msgt_inline;
                   if (((mach_msg_type_t*)type)->msgt_longform) {
                           /* This must be aligned */
                           if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                               (is_misaligned(type))) {
                                   eaddr = ptr_align(eaddr);
                                   goto xxx;
                           }
                           name = type->msgtl_name;
                           size = type->msgtl_size;
                           rnumber = type->msgtl_number;
                           eaddr += sizeof(mach_msg_type_long_t);
                   } else {
                           name = ((mach_msg_type_t*)type)->msgt_name;
                           size = ((mach_msg_type_t*)type)->msgt_size;
                           rnumber = ((mach_msg_type_t*)type)->msgt_number;
                           eaddr += sizeof(mach_msg_type_t);
                   }
   
                   /* padding (ptrs and ports) ? */
                   if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                       ((size >> 3) == sizeof(natural_t)))
                           eaddr = ptr_align(eaddr);
   
                   /* calculate length of data in bytes, rounding up */
   
                   length = ((rnumber * size) + 7) >> 3;
   
                   is_port = MACH_MSG_TYPE_PORT_ANY(name);
   
                   if (is_port) {
                           ipc_object_t *objects;
                           mach_msg_type_number_t i;
   
                           objects = (ipc_object_t *)
                                   (is_inline ? eaddr : * (vm_offset_t *) eaddr);
   
                           /* destroy port rights carried in the message */
   
                           for (i = 0; i < number; i++) {
                                   ipc_object_t obj = objects[i];
   
                                   if (!IO_VALID(obj))
                                           continue;
   
                                   ipc_object_destroy(obj, name);
                           }
                   }
   
                   if (!is_inline) {
                           vm_offset_t data = * (vm_offset_t *) eaddr;
   
                           /* destroy memory carried in the message */
   
                           if (length == 0)
                                   assert(data == 0);
                           else if (is_port)
                                   kfree(data, length);
                           else
                                   vm_map_copy_discard((vm_map_copy_t) data);
                   }
           }
   }
   
   /*
    *      Routine:        ipc_kmsg_free
    *      Purpose:
    *              Free a kernel message buffer.
    *      Conditions:
    *              Nothing locked.
    */
   
   void
   ipc_kmsg_free(
           ipc_kmsg_t kmsg)
   {
           vm_size_t size = kmsg->ikm_size;
   
           switch (size) {
   #if     NORMA_IPC
               case IKM_SIZE_NORMA:
                   /* return it to the norma ipc code */
                   norma_kmsg_put(kmsg);
                   break;
   #endif  /* NORMA_IPC */
   
               case IKM_SIZE_NETWORK:
                   /* return it to the network code */
                   net_kmsg_put(kmsg);
                   break;
   
               case IKM_SIZE_KERN_REPLY:
                   /* return it to the kernel object system */
                   kernel_reply_kmsg_put(kmsg);
                   break;
   
               default:
                   kfree((vm_offset_t) kmsg, size);
                   break;
           }
   }
   
   /*
    *      Routine:        ipc_kmsg_get
    *      Purpose:
    *              Allocates a kernel message buffer.
    *              Copies a user message to the message buffer.
    *      Conditions:
    *              Nothing locked.
    *      Returns:
    *              MACH_MSG_SUCCESS        Acquired a message buffer.
    *              MACH_SEND_MSG_TOO_SMALL Message smaller than a header.
    *              MACH_SEND_MSG_TOO_SMALL Message size not long-word multiple.
    *              MACH_SEND_NO_BUFFER     Couldn't allocate a message buffer.
    *              MACH_SEND_INVALID_DATA  Couldn't copy message data.
    */
   
   mach_msg_return_t
   ipc_kmsg_get(
           mach_msg_header_t *msg,
           mach_msg_size_t size,
           ipc_kmsg_t      *kmsgp)
   {
           ipc_kmsg_t kmsg;
   
           if ((size < sizeof(mach_msg_header_t)) || (size & 3))
                   return MACH_SEND_MSG_TOO_SMALL;
   
           if (size <= IKM_SAVED_MSG_SIZE) {
                   kmsg = ikm_cache();
                   if (kmsg != IKM_NULL) {
                           ikm_cache() = IKM_NULL;
                           ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
                   } else {
                           kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
                           if (kmsg == IKM_NULL)
                                   return MACH_SEND_NO_BUFFER;
                           ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
                   }
           } else {
                   kmsg = ikm_alloc(size);
                   if (kmsg == IKM_NULL)
                           return MACH_SEND_NO_BUFFER;
                   ikm_init(kmsg, size);
           }
   
           if (copyinmsg((char *) msg, (char *) &kmsg->ikm_header, size)) {
                   ikm_free(kmsg);
                   return MACH_SEND_INVALID_DATA;
           }
   
           kmsg->ikm_header.msgh_size = size;
           *kmsgp = kmsg;
           return MACH_MSG_SUCCESS;
   }
   
   /*
    *      Routine:        ipc_kmsg_get_from_kernel
    *      Purpose:
    *              Allocates a kernel message buffer.
    *              Copies a kernel message to the message buffer.
    *              Only resource errors are allowed.
    *      Conditions:
    *              Nothing locked.
    *      Returns:
    *              MACH_MSG_SUCCESS        Acquired a message buffer.
    *              MACH_SEND_NO_BUFFER     Couldn't allocate a message buffer.
    */
   
   extern mach_msg_return_t
   ipc_kmsg_get_from_kernel(
           mach_msg_header_t *msg,
           mach_msg_size_t size,
           ipc_kmsg_t      *kmsgp)
   {
           ipc_kmsg_t kmsg;
   
           assert(size >= sizeof(mach_msg_header_t));
           assert((size & 3) == 0);
   
           kmsg = ikm_alloc(size);
           if (kmsg == IKM_NULL)
                   return MACH_SEND_NO_BUFFER;
           ikm_init(kmsg, size);
   
           bcopy((char *) msg, (char *) &kmsg->ikm_header, size);
   
           kmsg->ikm_header.msgh_size = size;
           *kmsgp = kmsg;
           return MACH_MSG_SUCCESS;
   }
   
   /*
    *      Routine:        ipc_kmsg_put
    *      Purpose:
    *              Copies a message buffer to a user message.
    *              Copies only the specified number of bytes.
    *              Frees the message buffer.
    *      Conditions:
    *              Nothing locked.  The message buffer must have clean
    *              header (ikm_marequest) fields.
    *      Returns:
    *              MACH_MSG_SUCCESS        Copied data out of message buffer.
    *              MACH_RCV_INVALID_DATA   Couldn't copy to user message.
    */
   
   mach_msg_return_t
   ipc_kmsg_put(
           mach_msg_header_t *msg,
           ipc_kmsg_t      kmsg,
           mach_msg_size_t size)
   {
           mach_msg_return_t mr;
   
           ikm_check_initialized(kmsg, kmsg->ikm_size);
   
           if (copyoutmsg((char *) &kmsg->ikm_header, (char *) msg, size))
                   mr = MACH_RCV_INVALID_DATA;
           else
                   mr = MACH_MSG_SUCCESS;
   
           if ((kmsg->ikm_size == IKM_SAVED_KMSG_SIZE) &&
               (ikm_cache() == IKM_NULL))
                   ikm_cache() = kmsg;
           else
                   ikm_free(kmsg);
   
           return mr;
   }
   
   /*
    *      Routine:        ipc_kmsg_put_to_kernel
    *      Purpose:
    *              Copies a message buffer to a kernel message.
    *              Frees the message buffer.
    *              No errors allowed.
    *      Conditions:
    *              Nothing locked.
    */
   
   void
   ipc_kmsg_put_to_kernel(
           mach_msg_header_t *msg,
           ipc_kmsg_t      kmsg,
           mach_msg_size_t size)
   {
           bcopy((char *) &kmsg->ikm_header, (char *) msg, size);
   
           ikm_free(kmsg);
   }
   
   /*
    *      Routine:        ipc_kmsg_copyin_header
    *      Purpose:
    *              "Copy-in" port rights in the header of a message.
    *              Operates atomically; if it doesn't succeed the
    *              message header and the space are left untouched.
    *              If it does succeed the remote/local port fields
    *              contain object pointers instead of port names,
    *              and the bits field is updated.  The destination port
    *              will be a valid port pointer.
    *
    *              The notify argument implements the MACH_SEND_CANCEL option.
    *              If it is not MACH_PORT_NULL, it should name a receive right.
    *              If the processing of the destination port would generate
    *              a port-deleted notification (because the right for the
    *              destination port is destroyed and it had a request for
    *              a dead-name notification registered), and the port-deleted
    *              notification would be sent to the named receive right,
    *              then it isn't sent and the send-once right for the notify
    *              port is quietly destroyed.
    *
    *              [MACH_IPC_COMPAT] There is an atomicity problem if the
    *              reply port is a compat entry and dies at an inopportune
    *              time.  This doesn't have any serious consequences
    *              (an observant user task might conceivably notice that
    *              the destination and reply ports were handled inconsistently),
    *              only happens in compat mode, and is extremely unlikely.
    *      Conditions:
    *              Nothing locked.
    *      Returns:
    *              MACH_MSG_SUCCESS        Successful copyin.
    *              MACH_SEND_INVALID_HEADER
    *                      Illegal value in the message header bits.
    *              MACH_SEND_INVALID_DEST  The space is dead.
    *              MACH_SEND_INVALID_NOTIFY
    *                      Notify is non-null and doesn't name a receive right.
    *                      (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
    *              MACH_SEND_INVALID_DEST  Can't copyin destination port.
    *                      (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
    *              MACH_SEND_INVALID_REPLY Can't copyin reply port.
    *                      (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
    */
   
   mach_msg_return_t
   ipc_kmsg_copyin_header(
           mach_msg_header_t *msg,
           ipc_space_t     space,
           mach_port_t     notify)
   {
           mach_msg_bits_t mbits = msg->msgh_bits &~ MACH_MSGH_BITS_CIRCULAR;
           mach_port_t dest_name = msg->msgh_remote_port;
           mach_port_t reply_name = msg->msgh_local_port;
           kern_return_t kr;
   
           /* first check for common cases */
   
           if (notify == MACH_PORT_NULL) switch (MACH_MSGH_BITS_PORTS(mbits)) {
               case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0): {
                   ipc_entry_t entry;
                   ipc_entry_bits_t bits;
                   ipc_port_t dest_port;
   
                   /* sending an asynchronous message */
   
                   if (reply_name != MACH_PORT_NULL)
                           break;
   
                   is_read_lock(space);
                   if (!space->is_active)
                           goto abort_async;
   
                   /* optimized ipc_entry_lookup */
   
               {
                   mach_port_index_t index = MACH_PORT_INDEX(dest_name);
                   mach_port_gen_t gen = MACH_PORT_GEN(dest_name);
   
                   if (index >= space->is_table_size)
                           goto abort_async;
   
                   entry = &space->is_table[index];
                   bits = entry->ie_bits;
   
                   /* check generation number and type bit */
   
                   if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) !=
                       (gen | MACH_PORT_TYPE_SEND))
                           goto abort_async;
               }
   
                   /* optimized ipc_right_copyin */
   
                   assert(IE_BITS_UREFS(bits) > 0);
   
                   dest_port = (ipc_port_t) entry->ie_object;
                   assert(dest_port != IP_NULL);
   
                   ip_lock(dest_port);
                   /* can unlock space now without compromising atomicity */
                   is_read_unlock(space);
   
                   if (!ip_active(dest_port)) {
                           ip_unlock(dest_port);
                           break;
                   }
   
                   assert(dest_port->ip_srights > 0);
                   dest_port->ip_srights++;
                   ip_reference(dest_port);
                   ip_unlock(dest_port);
   
                   msg->msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                                     MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
                   msg->msgh_remote_port = (mach_port_t) dest_port;
                   return MACH_MSG_SUCCESS;
   
               abort_async:
                   is_read_unlock(space);
                   break;
               }
   
               case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
                                   MACH_MSG_TYPE_MAKE_SEND_ONCE): {
                   ipc_entry_num_t size;
                   ipc_entry_t table;
                   ipc_entry_t entry;
                   ipc_entry_bits_t bits;
                   ipc_port_t dest_port, reply_port;
   
                   /* sending a request message */
   
                   is_read_lock(space);
                   if (!space->is_active)
                           goto abort_request;
   
                   size = space->is_table_size;
                   table = space->is_table;
   
                   /* optimized ipc_entry_lookup of dest_name */
   
               {
                   mach_port_index_t index = MACH_PORT_INDEX(dest_name);
                   mach_port_gen_t gen = MACH_PORT_GEN(dest_name);
   
                   if (index >= size)
                           goto abort_request;
   
                   entry = &table[index];
                   bits = entry->ie_bits;
   
                  /* check generation number and type bit */
  
                  if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) !=
                      (gen | MACH_PORT_TYPE_SEND))
                          goto abort_request;
              }
  
                  assert(IE_BITS_UREFS(bits) > 0);
  
                  dest_port = (ipc_port_t) entry->ie_object;
                  assert(dest_port != IP_NULL);
  
                  /* optimized ipc_entry_lookup of reply_name */
  
              {
                  mach_port_index_t index = MACH_PORT_INDEX(reply_name);
                  mach_port_gen_t gen = MACH_PORT_GEN(reply_name);
  
                  if (index >= size)
                          goto abort_request;
  
                  entry = &table[index];
                  bits = entry->ie_bits;
  
                  /* check generation number and type bit */
  
                  if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_RECEIVE)) !=
                      (gen | MACH_PORT_TYPE_RECEIVE))
                          goto abort_request;
              }
  
                  reply_port = (ipc_port_t) entry->ie_object;
                  assert(reply_port != IP_NULL);
  
                  /*
                   *      To do an atomic copyin, need simultaneous
                   *      locks on both ports and the space.  If
                   *      dest_port == reply_port, and simple locking is
                   *      enabled, then we will abort.  Otherwise it's
                   *      OK to unlock twice.
                   */
  
                  ip_lock(dest_port);
                  if (!ip_active(dest_port) || !ip_lock_try(reply_port)) {
                          ip_unlock(dest_port);
                          goto abort_request;
                  }
                  /* can unlock space now without compromising atomicity */
                  is_read_unlock(space);
  
                  assert(dest_port->ip_srights > 0);
                  dest_port->ip_srights++;
                  ip_reference(dest_port);
                  ip_unlock(dest_port);
  
                  assert(ip_active(reply_port));
                  assert(reply_port->ip_receiver_name == reply_name);
                  assert(reply_port->ip_receiver == space);
  
                  reply_port->ip_sorights++;
                  ip_reference(reply_port);
                  ip_unlock(reply_port);
  
                  msg->msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                          MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
                                         MACH_MSG_TYPE_PORT_SEND_ONCE));
                  msg->msgh_remote_port = (mach_port_t) dest_port;
                  msg->msgh_local_port = (mach_port_t) reply_port;
                  return MACH_MSG_SUCCESS;
  
              abort_request:
                  is_read_unlock(space);
                  break;
              }
  
              case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
                  mach_port_index_t index;
                  mach_port_gen_t gen;
                  ipc_entry_t table;
                  ipc_entry_t entry;
                  ipc_entry_bits_t bits;
                  ipc_port_t dest_port;
  
                  /* sending a reply message */
  
                  if (reply_name != MACH_PORT_NULL)
                          break;
  
                  is_write_lock(space);
                  if (!space->is_active)
                          goto abort_reply;
  
                  /* optimized ipc_entry_lookup */
  
                  table = space->is_table;
  
                  index = MACH_PORT_INDEX(dest_name);
                  gen = MACH_PORT_GEN(dest_name);
  
                  if (index >= space->is_table_size)
                          goto abort_reply;
  
                  entry = &table[index];
                  bits = entry->ie_bits;
  
                  /* check generation number, collision bit, and type bit */
  
                  if ((bits & (IE_BITS_GEN_MASK|IE_BITS_COLLISION|
                               MACH_PORT_TYPE_SEND_ONCE)) !=
                      (gen | MACH_PORT_TYPE_SEND_ONCE))
                          goto abort_reply;
  
                  /* optimized ipc_right_copyin */
  
                  assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE);
                  assert(IE_BITS_UREFS(bits) == 1);
                  assert((bits & IE_BITS_MAREQUEST) == 0);
  
                  if (entry->ie_request != 0)
                          goto abort_reply;
  
                  dest_port = (ipc_port_t) entry->ie_object;
                  assert(dest_port != IP_NULL);
  
                  ip_lock(dest_port);
                  if (!ip_active(dest_port)) {
                          ip_unlock(dest_port);
                          goto abort_reply;
                  }
  
                  assert(dest_port->ip_sorights > 0);
                  ip_unlock(dest_port);
  
                  /* optimized ipc_entry_dealloc */
  
                  entry->ie_next = table->ie_next;
                  table->ie_next = index;
                  entry->ie_bits = gen;
                  entry->ie_object = IO_NULL;
                  is_write_unlock(space);
  
                  msg->msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                                    MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
                                                   0));
                  msg->msgh_remote_port = (mach_port_t) dest_port;
                  return MACH_MSG_SUCCESS;
  
              abort_reply:
                  is_write_unlock(space);
                  break;
              }
  
              default:
                  /* don't bother optimizing */
                  break;
          }
  
      {
          mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
          mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
          ipc_object_t dest_port, reply_port;
          ipc_port_t dest_soright, reply_soright;
          ipc_port_t notify_port = 0; /* '=0' to quiet gcc warnings */
  
          if (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type))
                  return MACH_SEND_INVALID_HEADER;
  
          if ((reply_type == 0) ?
              (reply_name != MACH_PORT_NULL) :
              !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type))
                  return MACH_SEND_INVALID_HEADER;
  
          is_write_lock(space);
          if (!space->is_active)
                  goto invalid_dest;
  
          if (notify != MACH_PORT_NULL) {
                  ipc_entry_t entry;
  
                  if (((entry = ipc_entry_lookup(space, notify)) == IE_NULL) ||
                      ((entry->ie_bits & MACH_PORT_TYPE_RECEIVE) == 0)) {
                          is_write_unlock(space);
                          return MACH_SEND_INVALID_NOTIFY;
                  }
  
                  notify_port = (ipc_port_t) entry->ie_object;
          }
  
          if (dest_name == reply_name) {
                  ipc_entry_t entry;
                  mach_port_t name = dest_name;
  
                  /*
                   *      Destination and reply ports are the same!
                   *      This is a little tedious to make atomic, because
                   *      there are 25 combinations of dest_type/reply_type.
                   *      However, most are easy.  If either is move-sonce,
                   *      then there must be an error.  If either are
                   *      make-send or make-sonce, then we must be looking
                   *      at a receive right so the port can't die.
                   *      The hard cases are the combinations of
                   *      copy-send and make-send.
                   */
  
                  entry = ipc_entry_lookup(space, name);
                  if (entry == IE_NULL)
                          goto invalid_dest;
  
                  assert(reply_type != 0); /* because name not null */
  
                  if (!ipc_right_copyin_check(space, name, entry, reply_type))
                          goto invalid_reply;
  
                  if ((dest_type == MACH_MSG_TYPE_MOVE_SEND_ONCE) ||
                      (reply_type == MACH_MSG_TYPE_MOVE_SEND_ONCE)) {
                          /*
                           *      Why must there be an error?  To get a valid
                           *      destination, this entry must name a live
                           *      port (not a dead name or dead port).  However
                           *      a successful move-sonce will destroy a
                           *      live entry.  Therefore the other copyin,
                           *      whatever it is, would fail.  We've already
                           *      checked for reply port errors above,
                           *      so report a destination error.
                           */
  
                          goto invalid_dest;
                  } else if ((dest_type == MACH_MSG_TYPE_MAKE_SEND) ||
                             (dest_type == MACH_MSG_TYPE_MAKE_SEND_ONCE) ||
                             (reply_type == MACH_MSG_TYPE_MAKE_SEND) ||
                             (reply_type == MACH_MSG_TYPE_MAKE_SEND_ONCE)) {
                          kr = ipc_right_copyin(space, name, entry,
                                                dest_type, FALSE,
                                                &dest_port, &dest_soright);
                          if (kr != KERN_SUCCESS)
                                  goto invalid_dest;
  
                          /*
                           *      Either dest or reply needs a receive right.
                           *      We know the receive right is there, because
                           *      of the copyin_check and copyin calls.  Hence
                           *      the port is not in danger of dying.  If dest
                           *      used the receive right, then the right needed
                           *      by reply (and verified by copyin_check) will
                           *      still be there.
                           */
  
                          assert(IO_VALID(dest_port));
                          assert(entry->ie_bits & MACH_PORT_TYPE_RECEIVE);
                          assert(dest_soright == IP_NULL);
  
                          kr = ipc_right_copyin(space, name, entry,
                                                reply_type, TRUE,
                                                &reply_port, &reply_soright);
  
                          assert(kr == KERN_SUCCESS);
                          assert(reply_port == dest_port);
                          assert(entry->ie_bits & MACH_PORT_TYPE_RECEIVE);
                          assert(reply_soright == IP_NULL);
                  } else if ((dest_type == MACH_MSG_TYPE_COPY_SEND) &&
                             (reply_type == MACH_MSG_TYPE_COPY_SEND)) {
                          /*
                           *      To make this atomic, just do one copy-send,
                           *      and dup the send right we get out.
                           */
  
                          kr = ipc_right_copyin(space, name, entry,
                                                dest_type, FALSE,
                                                &dest_port, &dest_soright);
                          if (kr != KERN_SUCCESS)
                                  goto invalid_dest;
  
                          assert(entry->ie_bits & MACH_PORT_TYPE_SEND);
                          assert(dest_soright == IP_NULL);
  
                          /*
                           *      It's OK if the port we got is dead now,
                           *      so reply_port is IP_DEAD, because the msg
                           *      won't go anywhere anyway.
                           */
  
                          reply_port = (ipc_object_t)
                                  ipc_port_copy_send((ipc_port_t) dest_port);
                          reply_soright = IP_NULL;
                  } else if ((dest_type == MACH_MSG_TYPE_MOVE_SEND) &&
                             (reply_type == MACH_MSG_TYPE_MOVE_SEND)) {
                          /*
                           *      This is an easy case.  Just use our
                           *      handy-dandy special-purpose copyin call
                           *      to get two send rights for the price of one.
                           */
  
                          kr = ipc_right_copyin_two(space, name, entry,
                                                    &dest_port, &dest_soright);
                          if (kr != KERN_SUCCESS)
                                  goto invalid_dest;
  
                          /* the entry might need to be deallocated */
  
                          if (IE_BITS_TYPE(entry->ie_bits)
                                                  == MACH_PORT_TYPE_NONE)
                                  ipc_entry_dealloc(space, name, entry);
  
                          reply_port = dest_port;
                          reply_soright = IP_NULL;
                  } else {
                          ipc_port_t soright;
  
                          assert(((dest_type == MACH_MSG_TYPE_COPY_SEND) &&
                                  (reply_type == MACH_MSG_TYPE_MOVE_SEND)) ||
                                 ((dest_type == MACH_MSG_TYPE_MOVE_SEND) &&
                                  (reply_type == MACH_MSG_TYPE_COPY_SEND)));
  
                          /*
                           *      To make this atomic, just do a move-send,
                           *      and dup the send right we get out.
                           */
  
                          kr = ipc_right_copyin(space, name, entry,
                                                MACH_MSG_TYPE_MOVE_SEND, FALSE,
                                                &dest_port, &soright);
                          if (kr != KERN_SUCCESS)
                                  goto invalid_dest;
  
                          /* the entry might need to be deallocated */
  
                          if (IE_BITS_TYPE(entry->ie_bits)
                                                  == MACH_PORT_TYPE_NONE)
                                  ipc_entry_dealloc(space, name, entry);
  
                          /*
                           *      It's OK if the port we got is dead now,
                           *      so reply_port is IP_DEAD, because the msg
                           *      won't go anywhere anyway.
                           */
  
                          reply_port = (ipc_object_t)
                                  ipc_port_copy_send((ipc_port_t) dest_port);
  
                          if (dest_type == MACH_MSG_TYPE_MOVE_SEND) {
                                  dest_soright = soright;
                                  reply_soright = IP_NULL;
                          } else {
                                  dest_soright = IP_NULL;
                                  reply_soright = soright;
                          }
                  }
          } else if (!MACH_PORT_VALID(reply_name)) {
                  ipc_entry_t entry;
  
                  /*
                   *      No reply port!  This is an easy case
                   *      to make atomic.  Just copyin the destination.
                   */
  
                  entry = ipc_entry_lookup(space, dest_name);
                  if (entry == IE_NULL)
                          goto invalid_dest;
  
                  kr = ipc_right_copyin(space, dest_name, entry,
                                        dest_type, FALSE,
                                        &dest_port, &dest_soright);
                  if (kr != KERN_SUCCESS)
                          goto invalid_dest;
  
                  /* the entry might need to be deallocated */
  
                  if (IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)
                          ipc_entry_dealloc(space, dest_name, entry);
  
                  reply_port = (ipc_object_t) reply_name;
                  reply_soright = IP_NULL;
          } else {
                  ipc_entry_t dest_entry, reply_entry;
                  ipc_port_t saved_reply;
  
                  /*
                   *      This is the tough case to make atomic.
                   *      The difficult problem is serializing with port death.
                   *      At the time we copyin dest_port, it must be alive.
                   *      If reply_port is alive when we copyin it, then
                   *      we are OK, because we serialize before the death
                   *      of both ports.  Assume reply_port is dead at copyin.
                   *      Then if dest_port dies/died after reply_port died,
                   *      we are OK, because we serialize between the death
                   *      of the two ports.  So the bad case is when dest_port
                   *      dies after its copyin, reply_port dies before its
                   *      copyin, and dest_port dies before reply_port.  Then
                   *      the copyins operated as if dest_port was alive
                   *      and reply_port was dead, which shouldn't have happened
                   *      because they died in the other order.
                   *
                   *      We handle the bad case by undoing the copyins
                   *      (which is only possible because the ports are dead)
                   *      and failing with MACH_SEND_INVALID_DEST, serializing
                   *      after the death of the ports.
                   *
                   *      Note that it is easy for a user task to tell if
                   *      a copyin happened before or after a port died.
                   *      For example, suppose both dest and reply are
                   *      send-once rights (types are both move-sonce) and
                   *      both rights have dead-name requests registered.
                   *      If a port dies before copyin, a dead-name notification
                   *      is generated and the dead name's urefs are incremented,
                   *      and if the copyin happens first, a port-deleted
                   *      notification is generated.
                   *
                   *      Note that although the entries are different,
                   *      dest_port and reply_port might still be the same.
                   */
  
                  dest_entry = ipc_entry_lookup(space, dest_name);
                  if (dest_entry == IE_NULL)
                          goto invalid_dest;
  
                  reply_entry = ipc_entry_lookup(space, reply_name);
                  if (reply_entry == IE_NULL)
                          goto invalid_reply;
  
                  assert(dest_entry != reply_entry); /* names are not equal */
                  assert(reply_type != 0); /* because reply_name not null */
  
                  if (!ipc_right_copyin_check(space, reply_name, reply_entry,
                                              reply_type))
                          goto invalid_reply;
  
                  kr = ipc_right_copyin(space, dest_name, dest_entry,
                                        dest_type, FALSE,
                                        &dest_port, &dest_soright);
                  if (kr != KERN_SUCCESS)
                          goto invalid_dest;
  
                  assert(IO_VALID(dest_port));
  
                  saved_reply = (ipc_port_t) reply_entry->ie_object;
                  /* might be IP_NULL, if this is a dead name */
                  if (saved_reply != IP_NULL)
                          ipc_port_reference(saved_reply);
  
                  kr = ipc_right_copyin(space, reply_name, reply_entry,
                                        reply_type, TRUE,
                                        &reply_port, &reply_soright);
  #if     MACH_IPC_COMPAT
                  if (kr != KERN_SUCCESS) {
                          assert(kr == KERN_INVALID_NAME);
  
                          /*
                           *      Oops.  This must have been a compat entry
                           *      and the port died after the check above.
                           *      We should back out the copyin of dest_port,
                           *      and report MACH_SEND_INVALID_REPLY, but
                           *      if dest_port is alive we can't always do that.
                           *      Punt and pretend we got IO_DEAD, skipping
                           *      further hairy atomicity problems.
                           */
  
                          reply_port = IO_DEAD;
                          reply_soright = IP_NULL;
                          goto skip_reply_checks;
                  }
  #else   /* MACH_IPC_COMPAT */
                  assert(kr == KERN_SUCCESS);
  #endif  /* MACH_IPC_COMPAT */
  
                  if ((saved_reply != IP_NULL) && (reply_port == IO_DEAD)) {
                          ipc_port_t dest = (ipc_port_t) dest_port;
                          ipc_port_timestamp_t timestamp;
                          boolean_t must_undo;
  
                          /*
                           *      The reply port died before copyin.
                           *      Check if dest port died before reply.
                           */
  
                          ip_lock(saved_reply);
                          assert(!ip_active(saved_reply));
                          timestamp = saved_reply->ip_timestamp;
                          ip_unlock(saved_reply);
  
                          ip_lock(dest);
                          must_undo = (!ip_active(dest) &&
                                       IP_TIMESTAMP_ORDER(dest->ip_timestamp,
                                                          timestamp));
                          ip_unlock(dest);
  
                          if (must_undo) {
                                  /*
                                   *      Our worst nightmares are realized.
                                   *      Both destination and reply ports
                                   *      are dead, but in the wrong order,
                                   *      so we must undo the copyins and
                                   *      possibly generate a dead-name notif.
                                   */
  
                                  ipc_right_copyin_undo(
                                                  space, dest_name, dest_entry,
                                                  dest_type, dest_port,
                                                  dest_soright);
                                  /* dest_entry may be deallocated now */
  
                                  ipc_right_copyin_undo(
                                                  space, reply_name, reply_entry,
                                                  reply_type, reply_port,
                                                  reply_soright);
                                  /* reply_entry may be deallocated now */
  
                                  is_write_unlock(space);
  
                                  if (dest_soright != IP_NULL)
                                          ipc_notify_dead_name(dest_soright,
                                                               dest_name);
                                  assert(reply_soright == IP_NULL);
  
                                  ipc_port_release(saved_reply);
                                  return MACH_SEND_INVALID_DEST;
                          }
                  }
  
                  /* the entries might need to be deallocated */
  
                  if (IE_BITS_TYPE(reply_entry->ie_bits) == MACH_PORT_TYPE_NONE)
                          ipc_entry_dealloc(space, reply_name, reply_entry);
  
  #if     MACH_IPC_COMPAT
              skip_reply_checks:
                  /*
                   *      We jump here if the reply entry was a compat entry
                   *      and the port died on us.  In this case, the copyin
                   *      code already deallocated reply_entry.
                   */
  #endif  /* MACH_IPC_COMPAT */
  
                  if (IE_BITS_TYPE(dest_entry->ie_bits) == MACH_PORT_TYPE_NONE)
                          ipc_entry_dealloc(space, dest_name, dest_entry);
  
                  if (saved_reply != IP_NULL)
                          ipc_port_release(saved_reply);
          }
  
          /*
           *      At this point, dest_port, reply_port,
           *      dest_soright, reply_soright are all initialized.
           *      Any defunct entries have been deallocated.
           *      The space is still write-locked, and we need to
           *      make the MACH_SEND_CANCEL check.  The notify_port pointer
           *      is still usable, because the copyin code above won't ever
           *      deallocate a receive right, so its entry still exists
           *      and holds a ref.  Note notify_port might even equal
           *      dest_port or reply_port.
           */
  
          if ((notify != MACH_PORT_NULL) &&
              (dest_soright == notify_port)) {
                  ipc_port_release_sonce(dest_soright);
                  dest_soright = IP_NULL;
          }
  
          is_write_unlock(space);
  
          if (dest_soright != IP_NULL)
                  ipc_notify_port_deleted(dest_soright, dest_name);
  
          if (reply_soright != IP_NULL)
                  ipc_notify_port_deleted(reply_soright, reply_name);
  
          dest_type = ipc_object_copyin_type(dest_type);
          reply_type = ipc_object_copyin_type(reply_type);
  
          msg->msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                            MACH_MSGH_BITS(dest_type, reply_type));
          msg->msgh_remote_port = (mach_port_t) dest_port;
          msg->msgh_local_port = (mach_port_t) reply_port;
      }
  
          return MACH_MSG_SUCCESS;
  
      invalid_dest:
          is_write_unlock(space);
          return MACH_SEND_INVALID_DEST;
  
      invalid_reply:
          is_write_unlock(space);
          return MACH_SEND_INVALID_REPLY;
  }
  
  /*
   *      Routine:        ipc_kmsg_copyin
   *      Purpose:
   *              "Copy-in" port rights and out-of-line memory
   *              in the message.
   *
   *              In all failure cases, the message is left holding
   *              no rights or memory.  However, the message buffer
   *              is not deallocated.  If successful, the message
   *              contains a valid destination port.
   *      Conditions:
   *              Nothing locked.
   *      Returns:
   *              MACH_MSG_SUCCESS        Successful copyin.
   *              MACH_SEND_INVALID_HEADER
   *                      Illegal value in the message header bits.
   *              MACH_SEND_INVALID_NOTIFY        Bad notify port.
   *              MACH_SEND_INVALID_DEST  Can't copyin destination port.
   *              MACH_SEND_INVALID_REPLY Can't copyin reply port.
   *              MACH_SEND_INVALID_MEMORY        Can't grab out-of-line memory.
   *              MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
   *              MACH_SEND_INVALID_TYPE  Bad type specification.
   *              MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
   */
  
  mach_msg_return_t
  ipc_kmsg_copyin(
          ipc_kmsg_t      kmsg,
          ipc_space_t     space,
          vm_map_t        map,
          mach_port_t     notify)
  {
          ipc_object_t dest;
          vm_offset_t saddr, eaddr;
          boolean_t complex;
          mach_msg_return_t mr;
          boolean_t use_page_lists, steal_pages;
  
          mr = ipc_kmsg_copyin_header(&kmsg->ikm_header, space, notify);
          if (mr != MACH_MSG_SUCCESS)
                  return mr;
  
          if ((kmsg->ikm_header.msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0)
                  return MACH_MSG_SUCCESS;
  
          dest = (ipc_object_t) kmsg->ikm_header.msgh_remote_port;
          complex = FALSE;
          use_page_lists = ipc_kobject_vm_page_list(ip_kotype((ipc_port_t)dest));
          steal_pages = ipc_kobject_vm_page_steal(ip_kotype((ipc_port_t)dest));
  
  #if     NORMA_IPC
          if (IP_NORMA_IS_PROXY((ipc_port_t) dest)) {
                  use_page_lists = TRUE;
                  steal_pages = TRUE;
          }
  #endif  /* NORMA_IPC */
  
          saddr = (vm_offset_t) (&kmsg->ikm_header + 1);
          eaddr = (vm_offset_t) &kmsg->ikm_header + kmsg->ikm_header.msgh_size;
  
          while (saddr < eaddr) {
                  vm_offset_t taddr = saddr;
                  mach_msg_type_long_t *type;
                  mach_msg_type_name_t name;
                  mach_msg_type_size_t size;
                  mach_msg_type_number_t number;
                  boolean_t is_inline, longform, dealloc, is_port;
                  vm_offset_t data;
                  vm_size_t length;
                  kern_return_t kr;
  
                  type = (mach_msg_type_long_t *) saddr;
  
                  if (((eaddr - saddr) < sizeof(mach_msg_type_t)) ||
                      ((longform = ((mach_msg_type_t*)type)->msgt_longform) &&
                       ((eaddr - saddr) < sizeof(mach_msg_type_long_t)))) {
                          ipc_kmsg_clean_partial(kmsg, taddr, FALSE, 0);
                          return MACH_SEND_MSG_TOO_SMALL;
                  }
  
                  is_inline = ((mach_msg_type_t*)type)->msgt_inline;
                  dealloc = ((mach_msg_type_t*)type)->msgt_deallocate;
                  if (longform) {
                          /* This must be aligned */
                          if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                              (is_misaligned(type))) {
                                  saddr = ptr_align(saddr);
                                  continue;
                          }
                          name = type->msgtl_name;
                          size = type->msgtl_size;
                          number = type->msgtl_number;
                          saddr += sizeof(mach_msg_type_long_t);
                  } else {
                          name = ((mach_msg_type_t*)type)->msgt_name;
                          size = ((mach_msg_type_t*)type)->msgt_size;
                          number = ((mach_msg_type_t*)type)->msgt_number;
                          saddr += sizeof(mach_msg_type_t);
                  }
  
                  is_port = MACH_MSG_TYPE_PORT_ANY(name);
  
                  if ((is_port && (size != PORT_T_SIZE_IN_BITS)) ||
                      (longform && ((type->msgtl_header.msgt_name != 0) ||
                                    (type->msgtl_header.msgt_size != 0) ||
                                    (type->msgtl_header.msgt_number != 0))) ||
                      (((mach_msg_type_t*)type)->msgt_unused != 0) ||
                      (dealloc && is_inline)) {
                          ipc_kmsg_clean_partial(kmsg, taddr, FALSE, 0);
                          return MACH_SEND_INVALID_TYPE;
                  }
  
                  /* padding (ptrs and ports) ? */
                  if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                      ((size >> 3) == sizeof(natural_t)))
                          saddr = ptr_align(saddr);
  
                  /* calculate length of data in bytes, rounding up */
  
                  length = ((number * size) + 7) >> 3;
  
                  if (is_inline) {
                          vm_size_t amount;
  
                          /* inline data sizes round up to int boundaries */
  
                          amount = (length + 3) &~ 3;
                          if ((eaddr - saddr) < amount) {
                                  ipc_kmsg_clean_partial(kmsg, taddr, FALSE, 0);
                                  return MACH_SEND_MSG_TOO_SMALL;
                          }
  
                          data = saddr;
                          saddr += amount;
                  } else {
                          vm_offset_t addr;
  
                          if (sizeof(vm_offset_t) > sizeof(mach_msg_type_t))
                                  saddr = ptr_align(saddr);
                          
                          if ((eaddr - saddr) < sizeof(vm_offset_t)) {
                                  ipc_kmsg_clean_partial(kmsg, taddr, FALSE, 0);
                                  return MACH_SEND_MSG_TOO_SMALL;
                          }
  
                          /* grab the out-of-line data */
  
                          addr = * (vm_offset_t *) saddr;
  
                          if (length == 0)
                                  data = 0;
                          else if (is_port) {
                                  data = kalloc(length);
                                  if (data == 0)
                                          goto invalid_memory;
  
                                  if (copyinmap(map, (char *) addr,
                                                (char *) data, length) ||
                                      (dealloc &&
                                       (vm_deallocate(map, addr, length) !=
                                                          KERN_SUCCESS))) {
                                          kfree(data, length);
                                          goto invalid_memory;
                                  }
                          } else {
                                  vm_map_copy_t copy;
  
                                  if (use_page_lists) {
                                          kr = vm_map_copyin_page_list(map,
                                                  addr, length, dealloc,
                                                  steal_pages, &copy, FALSE);
                                  } else {
                                          kr = vm_map_copyin(map, addr, length,
                                                             dealloc, &copy);
                                  }
                                  if (kr != KERN_SUCCESS) {
                                      invalid_memory:
                                          ipc_kmsg_clean_partial(kmsg, taddr,
                                                                 FALSE, 0);
                                          return MACH_SEND_INVALID_MEMORY;
                                  }
  
                                  data = (vm_offset_t) copy;
                          }
  
                          * (vm_offset_t *) saddr = data;
                          saddr += sizeof(vm_offset_t);
                          complex = TRUE;
                  }
  
                  if (is_port) {
                          mach_msg_type_name_t newname =
                                          ipc_object_copyin_type(name);
                          ipc_object_t *objects = (ipc_object_t *) data;
                          mach_msg_type_number_t i;
  
                          if (longform)
                                  type->msgtl_name = newname;
                          else
                                  ((mach_msg_type_t*)type)->msgt_name = newname;
  
                          for (i = 0; i < number; i++) {
                                  mach_port_t port = (mach_port_t) objects[i];
                                  ipc_object_t object;
  
                                  if (!MACH_PORT_VALID(port))
                                          continue;
  
                                  kr = ipc_object_copyin(space, port,
                                                         name, &object);
                                  if (kr != KERN_SUCCESS) {
                                          ipc_kmsg_clean_partial(kmsg, taddr,
                                                                 TRUE, i);
                                          return MACH_SEND_INVALID_RIGHT;
                                  }
  
                                  if ((newname == MACH_MSG_TYPE_PORT_RECEIVE) &&
                                      ipc_port_check_circularity(
                                                          (ipc_port_t) object,
                                                          (ipc_port_t) dest))
                                          kmsg->ikm_header.msgh_bits |=
                                                  MACH_MSGH_BITS_CIRCULAR;
  
                                  objects[i] = object;
                          }
  
                          complex = TRUE;
                  }
          }
  
          if (!complex)
                  kmsg->ikm_header.msgh_bits &= ~MACH_MSGH_BITS_COMPLEX;
  
          return MACH_MSG_SUCCESS;
  }
  
  /*
   *      Routine:        ipc_kmsg_copyin_from_kernel
   *      Purpose:
   *              "Copy-in" port rights and out-of-line memory
   *              in a message sent from the kernel.
   *
   *              Because the message comes from the kernel,
   *              the implementation assumes there are no errors
   *              or peculiarities in the message.
   *
   *              Returns TRUE if queueing the message
   *              would result in a circularity.
   *      Conditions:
   *              Nothing locked.
   */
  
  void
  ipc_kmsg_copyin_from_kernel(
          ipc_kmsg_t kmsg)
  {
          mach_msg_bits_t bits = kmsg->ikm_header.msgh_bits;
          mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits);
          mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits);
          ipc_object_t remote = (ipc_object_t) kmsg->ikm_header.msgh_remote_port;
          ipc_object_t local = (ipc_object_t) kmsg->ikm_header.msgh_local_port;
          vm_offset_t saddr, eaddr;
  
          /* translate the destination and reply ports */
  
          ipc_object_copyin_from_kernel(remote, rname);
          if (IO_VALID(local))
                  ipc_object_copyin_from_kernel(local, lname);
  
          /*
           *      The common case is a complex message with no reply port,
           *      because that is what the memory_object interface uses.
           */
  
          if (bits == (MACH_MSGH_BITS_COMPLEX |
                       MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) {
                  bits = (MACH_MSGH_BITS_COMPLEX |
                          MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0));
  
                  kmsg->ikm_header.msgh_bits = bits;
          } else {
                  bits = (MACH_MSGH_BITS_OTHER(bits) |
                          MACH_MSGH_BITS(ipc_object_copyin_type(rname),
                                         ipc_object_copyin_type(lname)));
  
                  kmsg->ikm_header.msgh_bits = bits;
                  if ((bits & MACH_MSGH_BITS_COMPLEX) == 0)
                          return;
          }
  
          saddr = (vm_offset_t) (&kmsg->ikm_header + 1);
          eaddr = (vm_offset_t) &kmsg->ikm_header + kmsg->ikm_header.msgh_size;
  
          while (saddr < eaddr) {
                  mach_msg_type_long_t *type;
                  mach_msg_type_name_t name;
                  mach_msg_type_size_t size;
                  mach_msg_type_number_t number;
                  boolean_t is_inline, longform, is_port;
                  vm_offset_t data;
                  vm_size_t length;
  
                  type = (mach_msg_type_long_t *) saddr;
                  is_inline = ((mach_msg_type_t*)type)->msgt_inline;
                  longform = ((mach_msg_type_t*)type)->msgt_longform;
                  /* type->msgtl_header.msgt_deallocate not used */
                  if (longform) {
                          /* This must be aligned */
                          if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                              (is_misaligned(type))) {
                                  saddr = ptr_align(saddr);
                                  continue;
                          }
                          name = type->msgtl_name;
                          size = type->msgtl_size;
                          number = type->msgtl_number;
                          saddr += sizeof(mach_msg_type_long_t);
                  } else {
                          name = ((mach_msg_type_t*)type)->msgt_name;
                          size = ((mach_msg_type_t*)type)->msgt_size;
                          number = ((mach_msg_type_t*)type)->msgt_number;
                          saddr += sizeof(mach_msg_type_t);
                  }
  
                  /* padding (ptrs and ports) ? */
                  if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                      ((size >> 3) == sizeof(natural_t)))
                          saddr = ptr_align(saddr);
  
                  /* calculate length of data in bytes, rounding up */
  
                  length = ((number * size) + 7) >> 3;
  
                  is_port = MACH_MSG_TYPE_PORT_ANY(name);
  
                  if (is_inline) {
                          /* inline data sizes round up to int boundaries */
  
                          data = saddr;
                          saddr += (length + 3) &~ 3;
                  } else {
                          /*
                           *      The sender should supply ready-made memory
                           *      for us, so we don't need to do anything.
                           */
  
                          data = * (vm_offset_t *) saddr;
                          saddr += sizeof(vm_offset_t);
                  }
  
                  if (is_port) {
                          mach_msg_type_name_t newname =
                                          ipc_object_copyin_type(name);
                          ipc_object_t *objects = (ipc_object_t *) data;
                          mach_msg_type_number_t i;
  
                          if (longform)
                                  type->msgtl_name = newname;
                          else
                                  ((mach_msg_type_t*)type)->msgt_name = newname;
                          for (i = 0; i < number; i++) {
                                  ipc_object_t object = objects[i];
  
                                  if (!IO_VALID(object))
                                          continue;
  
                                  ipc_object_copyin_from_kernel(object, name);
  
                                  if ((newname == MACH_MSG_TYPE_PORT_RECEIVE) &&
                                      ipc_port_check_circularity(
                                                          (ipc_port_t) object,
                                                          (ipc_port_t) remote))
                                          kmsg->ikm_header.msgh_bits |=
                                                  MACH_MSGH_BITS_CIRCULAR;
                          }
                  }
          }
  }
  
  /*
   *      Routine:        ipc_kmsg_copyout_header
   *      Purpose:
   *              "Copy-out" port rights in the header of a message.
   *              Operates atomically; if it doesn't succeed the
   *              message header and the space are left untouched.
   *              If it does succeed the remote/local port fields
   *              contain port names instead of object pointers,
   *              and the bits field is updated.
   *
   *              The notify argument implements the MACH_RCV_NOTIFY option.
   *              If it is not MACH_PORT_NULL, it should name a receive right.
   *              If the process of receiving the reply port creates a
   *              new right in the receiving task, then the new right is
   *              automatically registered for a dead-name notification,
   *              with the notify port supplying the send-once right.
   *      Conditions:
   *              Nothing locked.
   *      Returns:
   *              MACH_MSG_SUCCESS        Copied out port rights.
   *              MACH_RCV_INVALID_NOTIFY 
   *                      Notify is non-null and doesn't name a receive right.
   *                      (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.)
   *              MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
   *                      The space is dead.
   *              MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE
   *                      No room in space for another name.
   *              MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
   *                      Couldn't allocate memory for the reply port.
   *              MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL
   *                      Couldn't allocate memory for the dead-name request.
   */
  
  mach_msg_return_t
  ipc_kmsg_copyout_header(
          mach_msg_header_t *msg,
          ipc_space_t     space,
          mach_port_t     notify)
  {
          mach_msg_bits_t mbits = msg->msgh_bits;
          ipc_port_t dest = (ipc_port_t) msg->msgh_remote_port;
  
          assert(IP_VALID(dest));
  
          /* first check for common cases */
  
          if (notify == MACH_PORT_NULL) switch (MACH_MSGH_BITS_PORTS(mbits)) {
              case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0): {
                  mach_port_t dest_name;
                  ipc_port_t nsrequest;
  
                  /* receiving an asynchronous message */
  
                  ip_lock(dest);
                  if (!ip_active(dest)) {
                          ip_unlock(dest);
                          break;
                  }
  
                  /* optimized ipc_object_copyout_dest */
  
                  assert(dest->ip_srights > 0);
                  ip_release(dest);
  
                  if (dest->ip_receiver == space)
                          dest_name = dest->ip_receiver_name;
                  else
                          dest_name = MACH_PORT_NULL;
  
                  if ((--dest->ip_srights == 0) &&
                      ((nsrequest = dest->ip_nsrequest) != IP_NULL)) {
                          mach_port_mscount_t mscount;
  
                          dest->ip_nsrequest = IP_NULL;
                          mscount = dest->ip_mscount;
                          ip_unlock(dest);
  
                          ipc_notify_no_senders(nsrequest, mscount);
                  } else
                          ip_unlock(dest);
  
                  msg->msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                                    MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND));
                  msg->msgh_local_port = dest_name;
                  msg->msgh_remote_port = MACH_PORT_NULL;
                  return MACH_MSG_SUCCESS;
              }
  
              case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
                                  MACH_MSG_TYPE_PORT_SEND_ONCE): {
                  ipc_entry_t table;
                  mach_port_index_t index;
                  ipc_entry_t entry;
                  ipc_port_t reply = (ipc_port_t) msg->msgh_local_port;
                  mach_port_t dest_name, reply_name;
                  ipc_port_t nsrequest;
  
                  /* receiving a request message */
  
                  if (!IP_VALID(reply))
                          break;
  
                  is_write_lock(space);
                  if (!space->is_active ||
                      ((index = (table = space->is_table)->ie_next) == 0)) {
                          is_write_unlock(space);
                          break;
                  }
  
                  /*
                   *      To do an atomic copyout, need simultaneous
                   *      locks on both ports and the space.  If
                   *      dest == reply, and simple locking is
                   *      enabled, then we will abort.  Otherwise it's
                   *      OK to unlock twice.
                   */
  
                  ip_lock(dest);
                  if (!ip_active(dest) || !ip_lock_try(reply)) {
                          ip_unlock(dest);
                          is_write_unlock(space);
                          break;
                  }
  
                  if (!ip_active(reply)) {
                          ip_unlock(reply);
                          ip_unlock(dest);
                          is_write_unlock(space);
                          break;
                  }
  
                  assert(reply->ip_sorights > 0);
                  ip_unlock(reply);
  
                  /* optimized ipc_entry_get */
  
                  entry = &table[index];
                  table->ie_next = entry->ie_next;
                  entry->ie_request = 0;
  
              {
                  mach_port_gen_t gen;
  
                  assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
                  gen = entry->ie_bits + IE_BITS_GEN_ONE;
  
                  reply_name = MACH_PORT_MAKE(index, gen);
  
                  /* optimized ipc_right_copyout */
  
                  entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
              }
  
                  assert(MACH_PORT_VALID(reply_name));
                  entry->ie_object = (ipc_object_t) reply;
                  is_write_unlock(space);
  
                  /* optimized ipc_object_copyout_dest */
  
                  assert(dest->ip_srights > 0);
                  ip_release(dest);
  
                  if (dest->ip_receiver == space)
                          dest_name = dest->ip_receiver_name;
                  else
                          dest_name = MACH_PORT_NULL;
  
                  if ((--dest->ip_srights == 0) &&
                      ((nsrequest = dest->ip_nsrequest) != IP_NULL)) {
                          mach_port_mscount_t mscount;
  
                          dest->ip_nsrequest = IP_NULL;
                          mscount = dest->ip_mscount;
                          ip_unlock(dest);
  
                          ipc_notify_no_senders(nsrequest, mscount);
                  } else
                          ip_unlock(dest);
  
                  msg->msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                                    MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
                                                   MACH_MSG_TYPE_PORT_SEND));
                  msg->msgh_local_port = dest_name;
                  msg->msgh_remote_port = reply_name;
                  return MACH_MSG_SUCCESS;
              }
  
              case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
                  mach_port_t dest_name;
  
                  /* receiving a reply message */
  
                  ip_lock(dest);
                  if (!ip_active(dest)) {
                          ip_unlock(dest);
                          break;
                  }
  
                  /* optimized ipc_object_copyout_dest */
  
                  assert(dest->ip_sorights > 0);
  
                  if (dest->ip_receiver == space) {
                          ip_release(dest);
                          dest->ip_sorights--;
                          dest_name = dest->ip_receiver_name;
                          ip_unlock(dest);
                  } else {
                          ip_unlock(dest);
  
                          ipc_notify_send_once(dest);
                          dest_name = MACH_PORT_NULL;
                  }
  
                  msg->msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                          MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE));
                  msg->msgh_local_port = dest_name;
                  msg->msgh_remote_port = MACH_PORT_NULL;
                  return MACH_MSG_SUCCESS;
              }
  
              default:
                  /* don't bother optimizing */
                  break;
          }
  
      {
          mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
          mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
          ipc_port_t reply = (ipc_port_t) msg->msgh_local_port;
          mach_port_t dest_name, reply_name;
  
          if (IP_VALID(reply)) {
                  ipc_port_t notify_port;
                  ipc_entry_t entry;
                  kern_return_t kr;
  
                  /*
                   *      Handling notify (for MACH_RCV_NOTIFY) is tricky.
                   *      The problem is atomically making a send-once right
                   *      from the notify port and installing it for a
                   *      dead-name request in the new entry, because this
                   *      requires two port locks (on the notify port and
                   *      the reply port).  However, we can safely make
                   *      and consume send-once rights for the notify port
                   *      as long as we hold the space locked.  This isn't
                   *      an atomicity problem, because the only way
                   *      to detect that a send-once right has been created
                   *      and then consumed if it wasn't needed is by getting
                   *      at the receive right to look at ip_sorights, and
                   *      because the space is write-locked status calls can't
                   *      lookup the notify port receive right.  When we make
                   *      the send-once right, we lock the notify port,
                   *      so any status calls in progress will be done.
                   */
  
                  is_write_lock(space);
  
                  for (;;) {
                          ipc_port_request_index_t request;
  
                          if (!space->is_active) {
                                  is_write_unlock(space);
                                  return (MACH_RCV_HEADER_ERROR|
                                          MACH_MSG_IPC_SPACE);
                          }
  
                          if (notify != MACH_PORT_NULL) {
                                  notify_port = ipc_port_lookup_notify(space,
                                                                       notify);
                                  if (notify_port == IP_NULL) {
                                          is_write_unlock(space);
                                          return MACH_RCV_INVALID_NOTIFY;
                                  }
                          } else
                                  notify_port = IP_NULL;
  
                          if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
                              ipc_right_reverse(space, (ipc_object_t) reply,
                                                &reply_name, &entry)) {
                                  /* reply port is locked and active */
  
                                  /*
                                   *      We don't need the notify_port
                                   *      send-once right, but we can't release
                                   *      it here because reply port is locked.
                                   *      Wait until after the copyout to
                                   *      release the notify port right.
                                   */
  
                                  assert(entry->ie_bits &
                                                  MACH_PORT_TYPE_SEND_RECEIVE);
                                  break;
                          }
  
                          ip_lock(reply);
                          if (!ip_active(reply)) {
                                  ip_release(reply);
                                  ip_check_unlock(reply);
  
                                  if (notify_port != IP_NULL)
                                          ipc_port_release_sonce(notify_port);
  
                                  ip_lock(dest);
                                  is_write_unlock(space);
  
                                  reply = IP_DEAD;
                                  reply_name = MACH_PORT_DEAD;
                                  goto copyout_dest;
                          }
  
                          kr = ipc_entry_get(space, &reply_name, &entry);
                          if (kr != KERN_SUCCESS) {
                                  ip_unlock(reply);
  
                                  if (notify_port != IP_NULL)
                                          ipc_port_release_sonce(notify_port);
  
                                  /* space is locked */
                                  kr = ipc_entry_grow_table(space);
                                  if (kr != KERN_SUCCESS) {
                                          /* space is unlocked */
  
                                          if (kr == KERN_RESOURCE_SHORTAGE)
                                                  return (MACH_RCV_HEADER_ERROR|
                                                          MACH_MSG_IPC_KERNEL);
                                          else
                                                  return (MACH_RCV_HEADER_ERROR|
                                                          MACH_MSG_IPC_SPACE);
                                  }
                                  /* space is locked again; start over */
  
                                  continue;
                          }
  
                          assert(IE_BITS_TYPE(entry->ie_bits)
                                                  == MACH_PORT_TYPE_NONE);
                          assert(entry->ie_object == IO_NULL);
  
                          if (notify_port == IP_NULL) {
                                  /* not making a dead-name request */
  
                                  entry->ie_object = (ipc_object_t) reply;
                                  break;
                          }
  
                          kr = ipc_port_dnrequest(reply, reply_name,
                                                  notify_port, &request);
                          if (kr != KERN_SUCCESS) {
                                  ip_unlock(reply);
  
                                  ipc_port_release_sonce(notify_port);
  
                                  ipc_entry_dealloc(space, reply_name, entry);
                                  is_write_unlock(space);
  
                                  ip_lock(reply);
                                  if (!ip_active(reply)) {
                                          /* will fail next time around loop */
  
                                          ip_unlock(reply);
                                          is_write_lock(space);
                                          continue;
                                  }
  
                                  kr = ipc_port_dngrow(reply);
                                  /* port is unlocked */
                                  if (kr != KERN_SUCCESS)
                                          return (MACH_RCV_HEADER_ERROR|
                                                  MACH_MSG_IPC_KERNEL);
  
                                  is_write_lock(space);
                                  continue;
                          }
  
                          notify_port = IP_NULL; /* don't release right below */
  
                          entry->ie_object = (ipc_object_t) reply;
                          entry->ie_request = request;
                          break;
                  }
  
                  /* space and reply port are locked and active */
  
                  ip_reference(reply);    /* hold onto the reply port */
  
                  kr = ipc_right_copyout(space, reply_name, entry,
                                         reply_type, TRUE, (ipc_object_t) reply);
                  /* reply port is unlocked */
                  assert(kr == KERN_SUCCESS);
  
                  if (notify_port != IP_NULL)
                          ipc_port_release_sonce(notify_port);
  
                  ip_lock(dest);
                  is_write_unlock(space);
          } else {
                  /*
                   *      No reply port!  This is an easy case.
                   *      We only need to have the space locked
                   *      when checking notify and when locking
                   *      the destination (to ensure atomicity).
                   */
  
                  is_read_lock(space);
                  if (!space->is_active) {
                          is_read_unlock(space);
                          return MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE;
                  }
  
                  if (notify != MACH_PORT_NULL) {
                          ipc_entry_t entry;
  
                          /* must check notify even though it won't be used */
  
                          if (((entry = ipc_entry_lookup(space, notify))
                                                                  == IE_NULL) ||
                              ((entry->ie_bits & MACH_PORT_TYPE_RECEIVE) == 0)) {
                                  is_read_unlock(space);
                                  return MACH_RCV_INVALID_NOTIFY;
                          }
                  }
  
                  ip_lock(dest);
                  is_read_unlock(space);
  
                  reply_name = (mach_port_t) reply;
          }
  
          /*
           *      At this point, the space is unlocked and the destination
           *      port is locked.  (Lock taken while space was locked.)
           *      reply_name is taken care of; we still need dest_name.
           *      We still hold a ref for reply (if it is valid).
           *
           *      If the space holds receive rights for the destination,
           *      we return its name for the right.  Otherwise the task
           *      managed to destroy or give away the receive right between
           *      receiving the message and this copyout.  If the destination
           *      is dead, return MACH_PORT_DEAD, and if the receive right
           *      exists somewhere else (another space, in transit)
           *      return MACH_PORT_NULL.
           *
           *      Making this copyout operation atomic with the previous
           *      copyout of the reply port is a bit tricky.  If there was
           *      no real reply port (it wasn't IP_VALID) then this isn't
           *      an issue.  If the reply port was dead at copyout time,
           *      then we are OK, because if dest is dead we serialize
           *      after the death of both ports and if dest is alive
           *      we serialize after reply died but before dest's (later) death.
           *      So assume reply was alive when we copied it out.  If dest
           *      is alive, then we are OK because we serialize before
           *      the ports' deaths.  So assume dest is dead when we look at it.
           *      If reply dies/died after dest, then we are OK because
           *      we serialize after dest died but before reply dies.
           *      So the hard case is when reply is alive at copyout,
           *      dest is dead at copyout, and reply died before dest died.
           *      In this case pretend that dest is still alive, so
           *      we serialize while both ports are alive.
           *
           *      Because the space lock is held across the copyout of reply
           *      and locking dest, the receive right for dest can't move
           *      in or out of the space while the copyouts happen, so
           *      that isn't an atomicity problem.  In the last hard case
           *      above, this implies that when dest is dead that the
           *      space couldn't have had receive rights for dest at
           *      the time reply was copied-out, so when we pretend
           *      that dest is still alive, we can return MACH_PORT_NULL.
           *
           *      If dest == reply, then we have to make it look like
           *      either both copyouts happened before the port died,
           *      or both happened after the port died.  This special
           *      case works naturally if the timestamp comparison
           *      is done correctly.
           */
  
      copyout_dest:
  
          if (ip_active(dest)) {
                  ipc_object_copyout_dest(space, (ipc_object_t) dest,
                                          dest_type, &dest_name);
                  /* dest is unlocked */
          } else {
                  ipc_port_timestamp_t timestamp;
  
                  timestamp = dest->ip_timestamp;
                  ip_release(dest);
                  ip_check_unlock(dest);
  
                  if (IP_VALID(reply)) {
                          ip_lock(reply);
                          if (ip_active(reply) ||
                              IP_TIMESTAMP_ORDER(timestamp,
                                                 reply->ip_timestamp))
                                  dest_name = MACH_PORT_DEAD;
                          else
                                  dest_name = MACH_PORT_NULL;
                          ip_unlock(reply);
                  } else
                          dest_name = MACH_PORT_DEAD;
          }
  
          if (IP_VALID(reply))
                  ipc_port_release(reply);
  
          msg->msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                            MACH_MSGH_BITS(reply_type, dest_type));
          msg->msgh_local_port = dest_name;
          msg->msgh_remote_port = reply_name;
      }
  
          return MACH_MSG_SUCCESS;
  }
  
  /*
   *      Routine:        ipc_kmsg_copyout_object
   *      Purpose:
   *              Copy-out a port right.  Always returns a name,
   *              even for unsuccessful return codes.  Always
   *              consumes the supplied object.
   *      Conditions:
   *              Nothing locked.
   *      Returns:
   *              MACH_MSG_SUCCESS        The space acquired the right
   *                      (name is valid) or the object is dead (MACH_PORT_DEAD).
   *              MACH_MSG_IPC_SPACE      No room in space for the right,
   *                      or the space is dead.  (Name is MACH_PORT_NULL.)
   *              MACH_MSG_IPC_KERNEL     Kernel resource shortage.
   *                      (Name is MACH_PORT_NULL.)
   */
  
  mach_msg_return_t
  ipc_kmsg_copyout_object(
          ipc_space_t     space,
          ipc_object_t    object,
          mach_msg_type_name_t msgt_name,
          mach_port_t     *namep)
  {
          if (!IO_VALID(object)) {
                  *namep = (mach_port_t) object;
                  return MACH_MSG_SUCCESS;
          }
  
          /*
           *      Attempt quick copyout of send rights.  We optimize for a
           *      live port for which the receiver holds send (and not
           *      receive) rights in his local table.
           */
  
          if (msgt_name != MACH_MSG_TYPE_PORT_SEND)
                  goto slow_copyout;
  
      {
          register ipc_port_t port = (ipc_port_t) object;
          ipc_entry_t entry;
  
          is_write_lock(space);
          if (!space->is_active) {
                  is_write_unlock(space);
                  goto slow_copyout;
          }
  
          ip_lock(port);
          if (!ip_active(port) ||
              !ipc_hash_local_lookup(space, (ipc_object_t) port,
                                     namep, &entry)) {
                  ip_unlock(port);
                  is_write_unlock(space);
                  goto slow_copyout;
          }
  
          /*
           *      Copyout the send right, incrementing urefs
           *      unless it would overflow, and consume the right.
           */
  
          assert(port->ip_srights > 1);
          port->ip_srights--;
          ip_release(port);
          ip_unlock(port);
  
          assert(entry->ie_bits & MACH_PORT_TYPE_SEND);
          assert(IE_BITS_UREFS(entry->ie_bits) > 0);
          assert(IE_BITS_UREFS(entry->ie_bits) < MACH_PORT_UREFS_MAX);
  
      {
          register ipc_entry_bits_t bits = entry->ie_bits + 1;
  
          if (IE_BITS_UREFS(bits) < MACH_PORT_UREFS_MAX)
                  entry->ie_bits = bits;
      }
  
          is_write_unlock(space);
          return MACH_MSG_SUCCESS;
      }
  
      slow_copyout: {
          kern_return_t kr;
  
          kr = ipc_object_copyout(space, object, msgt_name, TRUE, namep);
          if (kr != KERN_SUCCESS) {
                  ipc_object_destroy(object, msgt_name);
  
                  if (kr == KERN_INVALID_CAPABILITY)
                          *namep = MACH_PORT_DEAD;
                  else {
                          *namep = MACH_PORT_NULL;
  
                          if (kr == KERN_RESOURCE_SHORTAGE)
                                  return MACH_MSG_IPC_KERNEL;
                          else
                                  return MACH_MSG_IPC_SPACE;
                  }
          }
  
          return MACH_MSG_SUCCESS;
      }
  }
  
  /*
   *      Routine:        ipc_kmsg_copyout_body
   *      Purpose:
   *              "Copy-out" port rights and out-of-line memory
   *              in the body of a message.
   *
   *              The error codes are a combination of special bits.
   *              The copyout proceeds despite errors.
   *      Conditions:
   *              Nothing locked.
   *      Returns:
   *              MACH_MSG_SUCCESS        Successfull copyout.
   *              MACH_MSG_IPC_SPACE      No room for port right in name space.
   *              MACH_MSG_VM_SPACE       No room for memory in address space.
   *              MACH_MSG_IPC_KERNEL     Resource shortage handling port right.
   *              MACH_MSG_VM_KERNEL      Resource shortage handling memory.
   */
  
  mach_msg_return_t
  ipc_kmsg_copyout_body(
          vm_offset_t     saddr,
          vm_offset_t     eaddr,
          ipc_space_t     space,
          vm_map_t        map)
  {
          mach_msg_return_t mr = MACH_MSG_SUCCESS;
          kern_return_t kr;
  
          while (saddr < eaddr) {
                  vm_offset_t taddr = saddr;
                  mach_msg_type_long_t *type;
                  mach_msg_type_name_t name;
                  mach_msg_type_size_t size;
                  mach_msg_type_number_t number;
                  boolean_t is_inline, longform, is_port;
                  vm_size_t length;
                  vm_offset_t addr;
  
                  type = (mach_msg_type_long_t *) saddr;
                  is_inline = ((mach_msg_type_t*)type)->msgt_inline;
                  longform = ((mach_msg_type_t*)type)->msgt_longform;
                  if (longform) {
                          /* This must be aligned */
                          if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                              (is_misaligned(type))) {
                                  saddr = ptr_align(saddr);
                                  continue;
                          }
                          name = type->msgtl_name;
                          size = type->msgtl_size;
                          number = type->msgtl_number;
                          saddr += sizeof(mach_msg_type_long_t);
                  } else {
                          name = ((mach_msg_type_t*)type)->msgt_name;
                          size = ((mach_msg_type_t*)type)->msgt_size;
                          number = ((mach_msg_type_t*)type)->msgt_number;
                          saddr += sizeof(mach_msg_type_t);
                  }
  
                  /* padding (ptrs and ports) ? */
                  if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                      ((size >> 3) == sizeof(natural_t)))
                          saddr = ptr_align(saddr);
  
                  /* calculate length of data in bytes, rounding up */
  
                  length = ((number * size) + 7) >> 3;
  
                  is_port = MACH_MSG_TYPE_PORT_ANY(name);
  
                  if (is_port) {
                          mach_port_t *objects;
                          mach_msg_type_number_t i;
  
                          if (!is_inline && (length != 0)) {
                                  /* first allocate memory in the map */
  
                                  kr = vm_allocate(map, &addr, length, TRUE);
                                  if (kr != KERN_SUCCESS) {
                                          ipc_kmsg_clean_body(taddr, saddr);
                                          goto vm_copyout_failure;
                                  }
                          }
  
                          objects = (mach_port_t *)
                                  (is_inline ? saddr : * (vm_offset_t *) saddr);
  
                          /* copyout port rights carried in the message */
  
                          for (i = 0; i < number; i++) {
                                  ipc_object_t object =
                                          (ipc_object_t) objects[i];
  
                                  mr |= ipc_kmsg_copyout_object(space, object,
                                                          name, &objects[i]);
                          }
                  }
  
                  if (is_inline) {
                          /* inline data sizes round up to int boundaries */
  
                          ((mach_msg_type_t*)type)->msgt_deallocate = FALSE;
                          saddr += (length + 3) &~ 3;
                  } else {
                          vm_offset_t data;
  
                          if (sizeof(vm_offset_t) > sizeof(mach_msg_type_t))
                                  saddr = ptr_align(saddr);
  
                          data = * (vm_offset_t *) saddr;
  
                          /* copyout memory carried in the message */
  
                          if (length == 0) {
                                  assert(data == 0);
                                  addr = 0;
                          } else if (is_port) {
                                  /* copyout to memory allocated above */
  
                                  (void) copyoutmap(map, (char *) data,
                                                    (char *) addr, length);
                                  kfree(data, length);
                          } else {
                                  vm_map_copy_t copy = (vm_map_copy_t) data;
  
                                  kr = vm_map_copyout(map, &addr, copy);
                                  if (kr != KERN_SUCCESS) {
                                          vm_map_copy_discard(copy);
  
                                      vm_copyout_failure:
  
                                          addr = 0;
                                          if (longform)
                                                  type->msgtl_size = 0;
                                          else
                                                  ((mach_msg_type_t*)type)->msgt_size = 0;
  
                                          if (kr == KERN_RESOURCE_SHORTAGE)
                                                  mr |= MACH_MSG_VM_KERNEL;
                                          else
                                                  mr |= MACH_MSG_VM_SPACE;
                                  }
                          }
  
                          ((mach_msg_type_t*)type)->msgt_deallocate = TRUE;
                          * (vm_offset_t *) saddr = addr;
                          saddr += sizeof(vm_offset_t);
                  }
          }
  
          return mr;
  }
  
  /*
   *      Routine:        ipc_kmsg_copyout
   *      Purpose:
   *              "Copy-out" port rights and out-of-line memory
   *              in the message.
   *      Conditions:
   *              Nothing locked.
   *      Returns:
   *              MACH_MSG_SUCCESS        Copied out all rights and memory.
   *              MACH_RCV_INVALID_NOTIFY Bad notify port.
   *                      Rights and memory in the message are intact.
   *              MACH_RCV_HEADER_ERROR + special bits
   *                      Rights and memory in the message are intact.
   *              MACH_RCV_BODY_ERROR + special bits
   *                      The message header was successfully copied out.
   *                      As much of the body was handled as possible.
   */
  
  mach_msg_return_t
  ipc_kmsg_copyout(
          ipc_kmsg_t      kmsg,
          ipc_space_t     space,
          vm_map_t        map,
          mach_port_t     notify)
  {
          mach_msg_bits_t mbits = kmsg->ikm_header.msgh_bits;
          mach_msg_return_t mr;
  
          mr = ipc_kmsg_copyout_header(&kmsg->ikm_header, space, notify);
          if (mr != MACH_MSG_SUCCESS)
                  return mr;
  
          if (mbits & MACH_MSGH_BITS_COMPLEX) {
                  vm_offset_t saddr, eaddr;
  
                  saddr = (vm_offset_t) (&kmsg->ikm_header + 1);
                  eaddr = (vm_offset_t) &kmsg->ikm_header +
                                  kmsg->ikm_header.msgh_size;
  
                  mr = ipc_kmsg_copyout_body(saddr, eaddr, space, map);
                  if (mr != MACH_MSG_SUCCESS)
                          mr |= MACH_RCV_BODY_ERROR;
          }
  
          return mr;
  }
  
  /*
   *      Routine:        ipc_kmsg_copyout_pseudo
   *      Purpose:
   *              Does a pseudo-copyout of the message.
   *              This is like a regular copyout, except
   *              that the ports in the header are handled
   *              as if they are in the body.  They aren't reversed.
   *
   *              The error codes are a combination of special bits.
   *              The copyout proceeds despite errors.
   *      Conditions:
   *              Nothing locked.
   *      Returns:
   *              MACH_MSG_SUCCESS        Successful copyout.
   *              MACH_MSG_IPC_SPACE      No room for port right in name space.
   *              MACH_MSG_VM_SPACE       No room for memory in address space.
   *              MACH_MSG_IPC_KERNEL     Resource shortage handling port right.
   *              MACH_MSG_VM_KERNEL      Resource shortage handling memory.
   */
  
  mach_msg_return_t
  ipc_kmsg_copyout_pseudo(
          ipc_kmsg_t      kmsg,
          ipc_space_t     space,
          vm_map_t        map)
  {
          mach_msg_bits_t mbits = kmsg->ikm_header.msgh_bits;
          ipc_object_t dest = (ipc_object_t) kmsg->ikm_header.msgh_remote_port;
          ipc_object_t reply = (ipc_object_t) kmsg->ikm_header.msgh_local_port;
          mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
          mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
          mach_port_t dest_name, reply_name;
          mach_msg_return_t mr;
  
          assert(IO_VALID(dest));
  
          mr = (ipc_kmsg_copyout_object(space, dest, dest_type, &dest_name) |
                ipc_kmsg_copyout_object(space, reply, reply_type, &reply_name));
  
          kmsg->ikm_header.msgh_bits = mbits &~ MACH_MSGH_BITS_CIRCULAR;
          kmsg->ikm_header.msgh_remote_port = dest_name;
          kmsg->ikm_header.msgh_local_port = reply_name;
  
          if (mbits & MACH_MSGH_BITS_COMPLEX) {
                  vm_offset_t saddr, eaddr;
  
                  saddr = (vm_offset_t) (&kmsg->ikm_header + 1);
                  eaddr = (vm_offset_t) &kmsg->ikm_header +
                                  kmsg->ikm_header.msgh_size;
  
                  mr |= ipc_kmsg_copyout_body(saddr, eaddr, space, map);
          }
  
          return mr;
  }
  
  /*
   *      Routine:        ipc_kmsg_copyout_dest
   *      Purpose:
   *              Copies out the destination port in the message.
   *              Destroys all other rights and memory in the message.
   *      Conditions:
   *              Nothing locked.
   */
  
  void
  ipc_kmsg_copyout_dest(
          ipc_kmsg_t      kmsg,
          ipc_space_t     space)
  {
          mach_msg_bits_t mbits = kmsg->ikm_header.msgh_bits;
          ipc_object_t dest = (ipc_object_t) kmsg->ikm_header.msgh_remote_port;
          ipc_object_t reply = (ipc_object_t) kmsg->ikm_header.msgh_local_port;
          mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
          mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
          mach_port_t dest_name, reply_name;
  
          assert(IO_VALID(dest));
  
          io_lock(dest);
          if (io_active(dest)) {
                  ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
                  /* dest is unlocked */
          } else {
                  io_release(dest);
                  io_check_unlock(dest);
                  dest_name = MACH_PORT_DEAD;
          }
  
          if (IO_VALID(reply)) {
                  ipc_object_destroy(reply, reply_type);
                  reply_name = MACH_PORT_NULL;
          } else
                  reply_name = (mach_port_t) reply;
  
          kmsg->ikm_header.msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                                        MACH_MSGH_BITS(reply_type, dest_type));
          kmsg->ikm_header.msgh_local_port = dest_name;
          kmsg->ikm_header.msgh_remote_port = reply_name;
  
          if (mbits & MACH_MSGH_BITS_COMPLEX) {
                  vm_offset_t saddr, eaddr;
  
                  saddr = (vm_offset_t) (&kmsg->ikm_header + 1);
                  eaddr = (vm_offset_t) &kmsg->ikm_header +
                                  kmsg->ikm_header.msgh_size;
  
                  ipc_kmsg_clean_body(saddr, eaddr);
          }
  }
  
  #if     NORMA_IPC || NORMA_VM
  /*
   *      Routine:        ipc_kmsg_copyout_to_kernel
   *      Purpose:
   *              Copies out the destination and reply ports in the message.
   *              Leaves all other rights and memory in the message alone.
   *      Conditions:
   *              Nothing locked.
   *
   *      Derived from ipc_kmsg_copyout_dest.
   *      Use by mach_msg_rpc_from_kernel (which used to use copyout_dest).
   *      We really do want to save rights and memory.
   */
  
  void
  ipc_kmsg_copyout_to_kernel(
          ipc_kmsg_t      kmsg,
          ipc_space_t     space)
  {
          mach_msg_bits_t mbits = kmsg->ikm_header.msgh_bits;
          ipc_object_t dest = (ipc_object_t) kmsg->ikm_header.msgh_remote_port;
          ipc_object_t reply = (ipc_object_t) kmsg->ikm_header.msgh_local_port;
          mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
          mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
          mach_port_t dest_name, reply_name;
  
          assert(IO_VALID(dest));
  
          io_lock(dest);
          if (io_active(dest)) {
                  ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
                  /* dest is unlocked */
          } else {
                  io_release(dest);
                  io_check_unlock(dest);
                  dest_name = MACH_PORT_DEAD;
          }
  
          reply_name = (mach_port_t) reply;
  
          kmsg->ikm_header.msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
                                        MACH_MSGH_BITS(reply_type, dest_type));
          kmsg->ikm_header.msgh_local_port = dest_name;
          kmsg->ikm_header.msgh_remote_port = reply_name;
  }
  #endif  /* NORMA_IPC || NORMA_VM */
  
  #if     MACH_IPC_COMPAT
  
  /*
   *      Routine:        ipc_kmsg_copyin_compat
   *      Purpose:
   *              "Copy-in" port rights and out-of-line memory
   *              in the message.
   *
   *              In all failure cases, the message is left holding
   *              no rights or memory.  However, the message buffer
   *              is not deallocated.  If successful, the message
   *              contains a valid destination port.
   *      Conditions:
   *              Nothing locked.
   *      Returns:
   *              MACH_MSG_SUCCESS        Successful copyin.
   *              MACH_SEND_INVALID_DEST  Can't copyin destination port.
   *              MACH_SEND_INVALID_REPLY Can't copyin reply port.
   *              MACH_SEND_INVALID_MEMORY        Can't grab out-of-line memory.
   *              MACH_SEND_INVALID_RIGHT Can't copyin port right in body.
   *              MACH_SEND_INVALID_TYPE  Bad type specification.
   *              MACH_SEND_MSG_TOO_SMALL Body is too small for types/data.
   */
  
  mach_msg_return_t
  ipc_kmsg_copyin_compat(
          ipc_kmsg_t kmsg,
          ipc_space_t space,
          vm_map_t map)
  {
          msg_header_t msg;
          mach_port_t dest_name;
          mach_port_t reply_name;
          ipc_object_t dest, reply;
          mach_msg_type_name_t dest_type, reply_type;
          vm_offset_t saddr, eaddr;
          boolean_t complex;
          kern_return_t kr;
          boolean_t use_page_lists, steal_pages;
  
          msg = * (msg_header_t *) &kmsg->ikm_header;
          dest_name = (mach_port_t) msg.msg_remote_port;
          reply_name = (mach_port_t) msg.msg_local_port;
  
          /* translate the destination and reply ports */
  
          kr = ipc_object_copyin_header(space, dest_name, &dest, &dest_type);
          if (kr != KERN_SUCCESS)
                  return MACH_SEND_INVALID_DEST;
  
          if (reply_name == MACH_PORT_NULL) {
                  reply = IO_NULL;
                  reply_type = 0;
          } else {
                  kr = ipc_object_copyin_header(space, reply_name,
                                                &reply, &reply_type);
                  if (kr != KERN_SUCCESS) {
                          ipc_object_destroy(dest, dest_type);
                          return MACH_SEND_INVALID_REPLY;
                  }
          }
  
          kmsg->ikm_header.msgh_bits = MACH_MSGH_BITS(dest_type, reply_type);
          kmsg->ikm_header.msgh_size = (mach_msg_size_t) msg.msg_size;
          kmsg->ikm_header.msgh_remote_port = (mach_port_t) dest;
          kmsg->ikm_header.msgh_local_port = (mach_port_t) reply;
          kmsg->ikm_header.msgh_seqno = (mach_msg_kind_t) msg.msg_type;
          kmsg->ikm_header.msgh_id = (mach_msg_id_t) msg.msg_id;
  
          if (msg.msg_simple)
                  return MACH_MSG_SUCCESS;
  
          complex = FALSE;
          use_page_lists = ipc_kobject_vm_page_list(ip_kotype((ipc_port_t)dest));
          steal_pages = ipc_kobject_vm_page_steal(ip_kotype((ipc_port_t)dest));
  
  #if     NORMA_IPC
          if (IP_NORMA_IS_PROXY((ipc_port_t) dest)) {
                  use_page_lists = TRUE;
                  steal_pages = TRUE;
          }
  #endif  /* NORMA_IPC */
  
          saddr = (vm_offset_t) (&kmsg->ikm_header + 1);
          eaddr = (vm_offset_t) &kmsg->ikm_header + kmsg->ikm_header.msgh_size;
  
          while (saddr < eaddr) {
                  vm_offset_t taddr = saddr;
                  mach_msg_type_long_t *type;
                  mach_msg_type_name_t name;
                  mach_msg_type_size_t size;
                  mach_msg_type_number_t number;
                  boolean_t is_inline, longform, dealloc, is_port;
                  vm_offset_t data;
                  vm_size_t length;
  
                  type = (mach_msg_type_long_t *) saddr;
  
                  if (((eaddr - saddr) < sizeof(mach_msg_type_t)) ||
                      ((longform = ((mach_msg_type_t*)type)->msgt_longform) &&
                       ((eaddr - saddr) < sizeof(mach_msg_type_long_t)))) {
                          ipc_kmsg_clean_partial(kmsg, taddr, FALSE, 0);
                          return MACH_SEND_MSG_TOO_SMALL;
                  }
  
                  is_inline = ((mach_msg_type_t*)type)->msgt_inline;
                  dealloc = ((mach_msg_type_t*)type)->msgt_deallocate;
                  if (longform) {
                          /* This must be aligned */
                          if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                              (is_misaligned(type))) {
                                  saddr = ptr_align(saddr);
                                  continue;
                          }
                          name = type->msgtl_name;
                          size = type->msgtl_size;
                          number = type->msgtl_number;
                          saddr += sizeof(mach_msg_type_long_t);
                  } else {
                          name = ((mach_msg_type_t*)type)->msgt_name;
                          size = ((mach_msg_type_t*)type)->msgt_size;
                          number = ((mach_msg_type_t*)type)->msgt_number;
                          saddr += sizeof(mach_msg_type_t);
                  }
  
                  is_port = MSG_TYPE_PORT_ANY(name);
  
                  if (is_port && (size != PORT_T_SIZE_IN_BITS)) {
                          ipc_kmsg_clean_partial(kmsg, taddr, FALSE, 0);
                          return MACH_SEND_INVALID_TYPE;
                  }
  
                  /*
                   *      New IPC says these should be zero, but old IPC
                   *      tasks often leave them with random values.  So
                   *      we have to clear them.
                   */
  
                  ((mach_msg_type_t*)type)->msgt_unused = 0;
                  if (longform) {
                          type->msgtl_header.msgt_name = 0;
                          type->msgtl_header.msgt_size = 0;
                          type->msgtl_header.msgt_number = 0;
                  }
  
                  /* padding (ptrs and ports) ? */
                  if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                      ((size >> 3) == sizeof(natural_t)))
                          saddr = ptr_align(saddr);
  
                  /* calculate length of data in bytes, rounding up */
  
                  length = ((number * size) + 7) >> 3;
  
                  if (is_inline) {
                          vm_size_t amount;
  
                          /* inline data sizes round up to int boundaries */
  
                          amount = (length + 3) &~ 3;
                          if ((eaddr - saddr) < amount) {
                                  ipc_kmsg_clean_partial(kmsg, taddr, FALSE, 0);
                                  return MACH_SEND_MSG_TOO_SMALL;
                          }
  
                          data = saddr;
                          saddr += amount;
                  } else {
                          vm_offset_t addr;
  
                          if ((eaddr - saddr) < sizeof(vm_offset_t)) {
                                  ipc_kmsg_clean_partial(kmsg, taddr, FALSE, 0);
                                  return MACH_SEND_MSG_TOO_SMALL;
                          }
  
                          /* grab the out-of-line data */
  
                          addr = * (vm_offset_t *) saddr;
  
                          if (length == 0)
                                  data = 0;
                          else if (is_port) {
                                  data = kalloc(length);
                                  if (data == 0)
                                          goto invalid_memory;
  
                                  if (copyinmap(map, (char *) addr,
                                                (char *) data, length) ||
                                      (dealloc &&
                                       (vm_deallocate(map, addr, length) !=
                                                          KERN_SUCCESS))) {
                                          kfree(data, length);
                                          goto invalid_memory;
                                  }
                          } else {
                                  vm_map_copy_t copy;
  
                                  if (use_page_lists) {
                                          kr = vm_map_copyin_page_list(map,
                                                  addr, length, dealloc,
                                                  steal_pages, &copy, FALSE);
                                  } else {
                                          kr = vm_map_copyin(map, addr, length,
                                                             dealloc,
                                                             &copy);
                                  }
                                  if (kr != KERN_SUCCESS) {
                                      invalid_memory:
                                          ipc_kmsg_clean_partial(kmsg, taddr,
                                                                 FALSE, 0);
                                          return MACH_SEND_INVALID_MEMORY;
                                  }
  
                                  data = (vm_offset_t) copy;
                          }
  
                          * (vm_offset_t *) saddr = data;
                          saddr += sizeof(vm_offset_t);
                          complex = TRUE;
                  }
  
                  if (is_port) {
                          mach_msg_type_name_t newname =
                                          ipc_object_copyin_type(name);
                          ipc_object_t *objects = (ipc_object_t *) data;
                          mach_msg_type_number_t i;
  
                          if (longform)
                                  type->msgtl_name = newname;
                          else
                                  ((mach_msg_type_t*)type)->msgt_name = newname;
  
                          for (i = 0; i < number; i++) {
                                  mach_port_t port = (mach_port_t) objects[i];
                                  ipc_object_t object;
  
                                  if (!MACH_PORT_VALID(port))
                                          continue;
  
                                  kr = ipc_object_copyin_compat(space, port,
                                                  name, dealloc, &object);
                                  if (kr != KERN_SUCCESS) {
                                          ipc_kmsg_clean_partial(kmsg, taddr,
                                                                 TRUE, i);
                                          return MACH_SEND_INVALID_RIGHT;
                                  }
  
                                  if ((newname == MACH_MSG_TYPE_PORT_RECEIVE) &&
                                      ipc_port_check_circularity(
                                                          (ipc_port_t) object,
                                                          (ipc_port_t) dest))
                                          kmsg->ikm_header.msgh_bits |=
                                                  MACH_MSGH_BITS_CIRCULAR;
  
                                  objects[i] = object;
                          }
  
                          complex = TRUE;
                  }
          }
  
          if (complex)
                  kmsg->ikm_header.msgh_bits |= MACH_MSGH_BITS_COMPLEX;
  
          return MACH_MSG_SUCCESS;
  }
  
  /*
   *      Routine:        ipc_kmsg_copyout_compat
   *      Purpose:
   *              "Copy-out" port rights and out-of-line memory
   *              in the message, producing an old IPC message.
   *
   *              Doesn't bother to handle the header atomically.
   *              Skips over errors.  Problem ports produce MACH_PORT_NULL
   *              (MACH_PORT_DEAD is never produced), and problem memory
   *              produces a zero address.
   *      Conditions:
   *              Nothing locked.
   *      Returns:
   *              MACH_MSG_SUCCESS        Copied out rights and memory.
   */
  
  mach_msg_return_t
  ipc_kmsg_copyout_compat(
          ipc_kmsg_t kmsg,
          ipc_space_t space,
          vm_map_t map)
  {
          msg_header_t msg;
          mach_msg_bits_t mbits = kmsg->ikm_header.msgh_bits;
          ipc_object_t dest = (ipc_object_t) kmsg->ikm_header.msgh_remote_port;
          ipc_object_t reply = (ipc_object_t) kmsg->ikm_header.msgh_local_port;
          mach_port_t dest_name, reply_name;
          vm_offset_t saddr, eaddr;
          kern_return_t kr;
  
          assert(IO_VALID(dest));
  
          io_lock(dest);
          if (io_active(dest)) {
                  mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits);
  
                  ipc_object_copyout_dest(space, dest, dest_type, &dest_name);
                  /* dest is unlocked */
          } else {
                  io_release(dest);
                  io_check_unlock(dest);
                  dest_name = MACH_PORT_NULL;
          }
  
          if (IO_VALID(reply)) {
                  mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits);
  
                  kr = ipc_object_copyout_compat(space, reply, reply_type,
                                                 &reply_name);
                  if (kr != KERN_SUCCESS) {
                          ipc_object_destroy(reply, reply_type);
                          reply_name = MACH_PORT_NULL;
                  }
          } else
                  reply_name = MACH_PORT_NULL;
  
          msg.msg_unused = 0;
          msg.msg_simple = (mbits & MACH_MSGH_BITS_COMPLEX) ? FALSE : TRUE;
          msg.msg_size = (msg_size_t) kmsg->ikm_header.msgh_size;
          msg.msg_type = (integer_t) kmsg->ikm_header.msgh_seqno;
          msg.msg_local_port = (port_name_t) dest_name;
          msg.msg_remote_port = (port_name_t) reply_name;
          msg.msg_id = (integer_t) kmsg->ikm_header.msgh_id;
          * (msg_header_t *) &kmsg->ikm_header = msg;
  
          if (msg.msg_simple)
                  return MACH_MSG_SUCCESS;
  
          saddr = (vm_offset_t) (&kmsg->ikm_header + 1);
          eaddr = (vm_offset_t) &kmsg->ikm_header + kmsg->ikm_header.msgh_size;
  
          while (saddr < eaddr) {
                  vm_offset_t taddr = saddr;
                  mach_msg_type_long_t *type;
                  mach_msg_type_name_t name;
                  mach_msg_type_size_t size;
                  mach_msg_type_number_t number;
                  boolean_t is_inline, longform, is_port;
                  vm_size_t length;
                  vm_offset_t addr;
  
                  type = (mach_msg_type_long_t *) saddr;
                  is_inline = ((mach_msg_type_t*)type)->msgt_inline;
                  longform = ((mach_msg_type_t*)type)->msgt_longform;
                  if (longform) {
                          /* This must be aligned */
                          if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                              (is_misaligned(type))) {
                                  saddr = ptr_align(saddr);
                                  continue;
                          }
                          name = type->msgtl_name;
                          size = type->msgtl_size;
                          number = type->msgtl_number;
                          saddr += sizeof(mach_msg_type_long_t);
                  } else {
                          name = ((mach_msg_type_t*)type)->msgt_name;
                          size = ((mach_msg_type_t*)type)->msgt_size;
                          number = ((mach_msg_type_t*)type)->msgt_number;
                          saddr += sizeof(mach_msg_type_t);
                  }
  
                  /* padding (ptrs and ports) ? */
                  if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                      ((size >> 3) == sizeof(natural_t)))
                          saddr = ptr_align(saddr);
  
                  /* calculate length of data in bytes, rounding up */
  
                  length = ((number * size) + 7) >> 3;
  
                  is_port = MACH_MSG_TYPE_PORT_ANY(name);
  
                  if (is_port) {
                          mach_port_t *objects;
                          mach_msg_type_number_t i;
                          mach_msg_type_name_t newname;
  
                          if (!is_inline && (length != 0)) {
                                  /* first allocate memory in the map */
  
                                  kr = vm_allocate(map, &addr, length, TRUE);
                                  if (kr != KERN_SUCCESS) {
                                          ipc_kmsg_clean_body(taddr, saddr);
                                          goto vm_copyout_failure;
                                  }
                          }
  
                          newname = ipc_object_copyout_type_compat(name);
                          if (longform)
                                  type->msgtl_name = newname;
                          else
                                  ((mach_msg_type_t*)type)->msgt_name = newname;
  
                          objects = (mach_port_t *)
                                  (is_inline ? saddr : * (vm_offset_t *) saddr);
  
                          /* copyout port rights carried in the message */
  
                          for (i = 0; i < number; i++) {
                                  ipc_object_t object =
                                          (ipc_object_t) objects[i];
  
                                  if (!IO_VALID(object)) {
                                          objects[i] = MACH_PORT_NULL;
                                          continue;
                                  }
  
                                  kr = ipc_object_copyout_compat(space, object,
                                                          name, &objects[i]);
                                  if (kr != KERN_SUCCESS) {
                                          ipc_object_destroy(object, name);
                                          objects[i] = MACH_PORT_NULL;
                                  }
                          }
                  }
  
                  if (is_inline) {
                          /* inline data sizes round up to int boundaries */
  
                          saddr += (length + 3) &~ 3;
                  } else {
                          vm_offset_t data = * (vm_offset_t *) saddr;
  
                          /* copyout memory carried in the message */
  
                          if (length == 0) {
                                  assert(data == 0);
                                  addr = 0;
                          } else if (is_port) {
                                  /* copyout to memory allocated above */
  
                                  (void) copyoutmap(map, (char *) data,
                                                    (char *) addr, length);
                                  kfree(data, length);
                          } else {
                                  vm_map_copy_t copy = (vm_map_copy_t) data;
  
                                  kr = vm_map_copyout(map, &addr, copy);
                                  if (kr != KERN_SUCCESS) {
                                          vm_map_copy_discard(copy);
  
                                      vm_copyout_failure:
  
                                          addr = 0;
                                  }
                          }
  
                          * (vm_offset_t *) saddr = addr;
                          saddr += sizeof(vm_offset_t);
                  }
          }
  
          return MACH_MSG_SUCCESS;
  }
  
  #endif  /* MACH_IPC_COMPAT */
  
  #include <mach_kdb.h>
  #if     MACH_KDB
  #include <ddb/db_output.h>      /* db_printf */
  
  char *
  ipc_type_name(
          int       type_name,
          boolean_t received)
  {
          switch (type_name) {
                  case MACH_MSG_TYPE_BOOLEAN:
                  return "boolean";
                  
                  case MACH_MSG_TYPE_INTEGER_16:
                  return "short";
                  
                  case MACH_MSG_TYPE_INTEGER_32:
                  return "int32";
  
                  case MACH_MSG_TYPE_INTEGER_64:
                  return "int64";
                  
                  case MACH_MSG_TYPE_CHAR:
                  return "char";
                  
                  case MACH_MSG_TYPE_BYTE:
                  return "byte";
                  
                  case MACH_MSG_TYPE_REAL:
                  return "real";
                  
                  case MACH_MSG_TYPE_STRING:
                  return "string";
                  
                  case MACH_MSG_TYPE_PORT_NAME:
                  return "port_name";
                  
                  case MACH_MSG_TYPE_MOVE_RECEIVE:
                  if (received) {
                          return "port_receive";
                  } else {
                          return "move_receive";
                  }
                  
                  case MACH_MSG_TYPE_MOVE_SEND:
                  if (received) {
                          return "port_send";
                  } else {
                          return "move_send";
                  }
                  
                  case MACH_MSG_TYPE_MOVE_SEND_ONCE:
                  if (received) {
                          return "port_send_once";
                  } else {
                          return "move_send_once";
                  }
                  
                  case MACH_MSG_TYPE_COPY_SEND:
                  return "copy_send";
                  
                  case MACH_MSG_TYPE_MAKE_SEND:
                  return "make_send";
                  
                  case MACH_MSG_TYPE_MAKE_SEND_ONCE:
                  return "make_send_once";
                  
                  default:
                  return (char *) 0;
          }
  }
                  
  void
  ipc_print_type_name(
          int type_name)
  {
          char *name = ipc_type_name(type_name, TRUE);
          if (name) {
                  db_printf("%s", name);
          } else {
                  db_printf("type%d", type_name);
          }
  }
  
  /*
   * ipc_kmsg_print       [ debug ]
   */
  void
  ipc_kmsg_print(
          ipc_kmsg_t kmsg)
  {
          db_printf("kmsg=0x%x\n", kmsg);
          db_printf("ikm_next=0x%x,prev=0x%x,size=%d,marequest=0x%x",
                    kmsg->ikm_next,
                    kmsg->ikm_prev,
                    kmsg->ikm_size,
                    kmsg->ikm_marequest);
  #if     NORMA_IPC
          db_printf(",page=0x%x,copy=0x%x\n",
                    kmsg->ikm_page,
                    kmsg->ikm_copy);
  #else   /* NORMA_IPC */
          db_printf("\n");
  #endif  /* NORMA_IPC */
          ipc_msg_print(&kmsg->ikm_header);
  }
  
  /*
   * ipc_msg_print        [ debug ]
   */
  void
  ipc_msg_print(
          mach_msg_header_t *msgh)
  {
          vm_offset_t saddr, eaddr;
  
          db_printf("msgh_bits=0x%x: ", msgh->msgh_bits);
          if (msgh->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
                  db_printf("complex,");
          }
          if (msgh->msgh_bits & MACH_MSGH_BITS_CIRCULAR) {
                  db_printf("circular,");
          }
          if (msgh->msgh_bits & MACH_MSGH_BITS_COMPLEX_PORTS) {
                  db_printf("complex_ports,");
          }
          if (msgh->msgh_bits & MACH_MSGH_BITS_COMPLEX_DATA) {
                  db_printf("complex_data,");
          }
          if (msgh->msgh_bits & MACH_MSGH_BITS_MIGRATED) {
                  db_printf("migrated,");
          }
          if (msgh->msgh_bits & MACH_MSGH_BITS_UNUSED) {
                  db_printf("unused=0x%x,",
                            msgh->msgh_bits & MACH_MSGH_BITS_UNUSED);
          }
          db_printf("l=0x%x,r=0x%x\n",
                    MACH_MSGH_BITS_LOCAL(msgh->msgh_bits),
                    MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
  
          db_printf("msgh_id=%d,size=%d,seqno=%d,",
                    msgh->msgh_id,
                    msgh->msgh_size,
                    msgh->msgh_seqno);
  
          if (msgh->msgh_remote_port) {
                  db_printf("remote=0x%x(", msgh->msgh_remote_port);
                  ipc_print_type_name(MACH_MSGH_BITS_REMOTE(msgh->msgh_bits));
                  db_printf("),");
          } else {
                  db_printf("remote=null,\n");
          }
  
          if (msgh->msgh_local_port) {
                  db_printf("local=0x%x(", msgh->msgh_local_port);
                  ipc_print_type_name(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits));
                  db_printf(")\n");
          } else {
                  db_printf("local=null\n");
          }
  
          saddr = (vm_offset_t) (msgh + 1);
          eaddr = (vm_offset_t) msgh + msgh->msgh_size;
  
          while (saddr < eaddr) {
                  mach_msg_type_long_t *type;
                  mach_msg_type_name_t name;
                  mach_msg_type_size_t size;
                  mach_msg_type_number_t number;
                  boolean_t is_inline, longform, dealloc, is_port;
                  vm_size_t length;
  
                  type = (mach_msg_type_long_t *) saddr;
  
                  if (((eaddr - saddr) < sizeof(mach_msg_type_t)) ||
                      ((longform = ((mach_msg_type_t*)type)->msgt_longform) &&
                       ((eaddr - saddr) < sizeof(mach_msg_type_long_t)))) {
                          db_printf("*** msg too small\n");
                          return;
                  }
  
                  is_inline = ((mach_msg_type_t*)type)->msgt_inline;
                  dealloc = ((mach_msg_type_t*)type)->msgt_deallocate;
                  if (longform) {
                          /* This must be aligned */
                          if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                              (is_misaligned(type))) {
                                  saddr = ptr_align(saddr);
                                  continue;
                          }
                          name = type->msgtl_name;
                          size = type->msgtl_size;
                          number = type->msgtl_number;
                          saddr += sizeof(mach_msg_type_long_t);
                  } else {
                          name = ((mach_msg_type_t*)type)->msgt_name;
                          size = ((mach_msg_type_t*)type)->msgt_size;
                          number = ((mach_msg_type_t*)type)->msgt_number;
                          saddr += sizeof(mach_msg_type_t);
                  }
  
                  db_printf("-- type=");
                  ipc_print_type_name(name);
                  if (! is_inline) {
                          db_printf(",ool");
                  }
                  if (dealloc) {
                          db_printf(",dealloc");
                  }
                  if (longform) {
                          db_printf(",longform");
                  }
                  db_printf(",size=%d,number=%d,addr=0x%x\n",
                         size,
                         number,
                         saddr);
  
                  is_port = MACH_MSG_TYPE_PORT_ANY(name);
  
                  if ((is_port && (size != PORT_T_SIZE_IN_BITS)) ||
                      (longform && ((type->msgtl_header.msgt_name != 0) ||
                                    (type->msgtl_header.msgt_size != 0) ||
                                    (type->msgtl_header.msgt_number != 0))) ||
                      (((mach_msg_type_t*)type)->msgt_unused != 0) ||
                      (dealloc && is_inline)) {
                          db_printf("*** invalid type\n");
                          return;
                  }
  
                  /* padding (ptrs and ports) ? */
                  if ((sizeof(natural_t) > sizeof(mach_msg_type_t)) &&
                      ((size >> 3) == sizeof(natural_t)))
                          saddr = ptr_align(saddr);
  
                  /* calculate length of data in bytes, rounding up */
  
                  length = ((number * size) + 7) >> 3;
  
                  if (is_inline) {
                          vm_size_t amount;
                          int i, numwords;
  
                          /* inline data sizes round up to int boundaries */
                          amount = (length + 3) &~ 3;
                          if ((eaddr - saddr) < amount) {
                                  db_printf("*** too small\n");
                                  return;
                          }
                          numwords = amount / sizeof(int);
                          if (numwords > 8) {
                                  numwords = 8;
                          }
                          for (i = 0; i < numwords; i++) {
                                  db_printf("0x%x\n", ((int *) saddr)[i]);
                          }
                          if (numwords < amount / sizeof(int)) {
                                  db_printf("...\n");
                          }
                          saddr += amount;
                  } else {
                          if ((eaddr - saddr) < sizeof(vm_offset_t)) {
                                  db_printf("*** too small\n");
                                  return;
                          }
                          db_printf("0x%x\n", * (vm_offset_t *) saddr);
                          saddr += sizeof(vm_offset_t);
                  }
          }
  }
  #endif  /* MACH_KDB */